saboteur 0.8.0 → 0.8.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +115 -0
- data/README.md +34 -6
- data/lib/saboteur/attribute_finder.rb +113 -0
- data/lib/saboteur/cli/presets.rb +41 -1
- data/lib/saboteur/cli/run_budget_options.rb +3 -1
- data/lib/saboteur/cli/source_set.rb +0 -18
- data/lib/saboteur/cli.rb +4 -2
- data/lib/saboteur/config.rb +4 -1
- data/lib/saboteur/config_file.rb +12 -8
- data/lib/saboteur/custom_map.rb +12 -1
- data/lib/saboteur/dirty_tree_warning.rb +5 -3
- data/lib/saboteur/doctor.rb +4 -0
- data/lib/saboteur/engine/baseline_guard.rb +76 -10
- data/lib/saboteur/engine/boot.rb +7 -3
- data/lib/saboteur/engine/bytecode_equivalence.rb +17 -1
- data/lib/saboteur/engine/candidate_selection.rb +1 -0
- data/lib/saboteur/engine/coverage_store.rb +25 -3
- data/lib/saboteur/engine/diagnostics.rb +1 -1
- data/lib/saboteur/engine/gated_run.rb +3 -1
- data/lib/saboteur/engine/load_coverage.rb +18 -0
- data/lib/saboteur/engine/pipelines.rb +8 -0
- data/lib/saboteur/engine/run_budget.rb +5 -2
- data/lib/saboteur/engine/sandbox_policy.rb +21 -3
- data/lib/saboteur/engine/scheduler.rb +32 -24
- data/lib/saboteur/engine/selection_announcement.rb +8 -3
- data/lib/saboteur/engine/subject_selection.rb +115 -31
- data/lib/saboteur/engine/test_guess.rb +106 -16
- data/lib/saboteur/engine.rb +3 -2
- data/lib/saboteur/execution/external_runner.rb +18 -0
- data/lib/saboteur/execution/fiddle_loader.rb +34 -0
- data/lib/saboteur/execution/marker_sweep.rb +2 -1
- data/lib/saboteur/execution/process_table.rb +7 -3
- data/lib/saboteur/execution/reevaluator.rb +7 -0
- data/lib/saboteur/execution/sandbox.rb +10 -4
- data/lib/saboteur/execution/seatbelt.rb +10 -3
- data/lib/saboteur/execution/supervisor.rb +2 -1
- data/lib/saboteur/execution/test_command_shape.rb +149 -0
- data/lib/saboteur/frameworks/sqlite_memory_database.rb +32 -2
- data/lib/saboteur/integration/rspec.rb +99 -11
- data/lib/saboteur/mutation_source/argument_swap.rb +54 -0
- data/lib/saboteur/mutation_source/builtin.rb +3 -1
- data/lib/saboteur/mutation_source/hazard_guard.rb +94 -13
- data/lib/saboteur/mutation_source/operator_examples.rb +1 -0
- data/lib/saboteur/mutation_source/operator_registry.rb +1 -0
- data/lib/saboteur/preview.rb +5 -0
- data/lib/saboteur/reporter/agent_incomplete.rb +20 -5
- data/lib/saboteur/reporter/terminal.rb +16 -1
- data/lib/saboteur/selection/coverage_map.rb +17 -2
- data/lib/saboteur/version.rb +1 -1
- data/lib/saboteur.rb +1 -0
- metadata +5 -15
|
@@ -24,7 +24,7 @@ module Saboteur
|
|
|
24
24
|
# coverage build when the cache is cold, so the first timed pass can
|
|
25
25
|
# price the rest (#260).
|
|
26
26
|
def initialize(config:, integration:, hooks:, supervisor:, diagnostics:, full_passes: 2, between_passes: nil,
|
|
27
|
-
timeout_cap: nil)
|
|
27
|
+
timeout_cap: nil, run_started: nil)
|
|
28
28
|
@config = config
|
|
29
29
|
@integration = integration
|
|
30
30
|
@hooks = hooks
|
|
@@ -33,6 +33,7 @@ module Saboteur
|
|
|
33
33
|
@full_passes = full_passes
|
|
34
34
|
@between_passes = between_passes
|
|
35
35
|
@timeout_cap = timeout_cap
|
|
36
|
+
@run_started = run_started
|
|
36
37
|
@completed_passes = 0
|
|
37
38
|
end
|
|
38
39
|
|
|
@@ -51,10 +52,7 @@ module Saboteur
|
|
|
51
52
|
started = Process.clock_gettime(Process::CLOCK_MONOTONIC)
|
|
52
53
|
data = check(number)
|
|
53
54
|
announce_first_pass_cost(number, started)
|
|
54
|
-
unless data["passed"] && data["failed"].empty?
|
|
55
|
-
refuse(number,
|
|
56
|
-
with_hint("the suite failed: #{Array(data['failed']).join(', ')}", data))
|
|
57
|
-
end
|
|
55
|
+
refuse_suite_failure(number, data) unless data["passed"] && data["failed"].empty?
|
|
58
56
|
executed = data["executed"]
|
|
59
57
|
refuse(number, "executed tests do not match the loaded suite") unless
|
|
60
58
|
executed.sort == expected.sort
|
|
@@ -67,6 +65,16 @@ module Saboteur
|
|
|
67
65
|
|
|
68
66
|
private
|
|
69
67
|
|
|
68
|
+
def refuse_failed_suite(number, data)
|
|
69
|
+
outside = data["failed"].empty?
|
|
70
|
+
reason = if outside
|
|
71
|
+
"the suite errored outside any example; see the log tail"
|
|
72
|
+
else
|
|
73
|
+
"the suite failed: #{data['failed'].join(', ')}"
|
|
74
|
+
end
|
|
75
|
+
refuse(number, with_hint(reason, data), tail: outside)
|
|
76
|
+
end
|
|
77
|
+
|
|
70
78
|
# A spent --max-duration budget skips the remaining passes - including
|
|
71
79
|
# the first, when boot alone already spent it (#245).
|
|
72
80
|
def budget_spent?(_index)
|
|
@@ -95,9 +103,10 @@ module Saboteur
|
|
|
95
103
|
|
|
96
104
|
coverage_pending = @full_passes > @config["baseline_passes"]
|
|
97
105
|
remaining = @full_passes - number - (coverage_pending ? 1 : 0)
|
|
106
|
+
elapsed = (Process.clock_gettime(Process::CLOCK_MONOTONIC) - started).round
|
|
107
|
+
warn_budget_projection(elapsed, remaining + (coverage_pending ? 1 : 0))
|
|
98
108
|
return if remaining.zero? && !coverage_pending
|
|
99
109
|
|
|
100
|
-
elapsed = (Process.clock_gettime(Process::CLOCK_MONOTONIC) - started).round
|
|
101
110
|
work = []
|
|
102
111
|
work << "the remaining #{remaining} full #{remaining == 1 ? 'pass' : 'passes'}" if remaining.positive?
|
|
103
112
|
work << "the coverage map build" if coverage_pending
|
|
@@ -106,6 +115,28 @@ module Saboteur
|
|
|
106
115
|
"#{work.join(' and ')} at that rate")
|
|
107
116
|
end
|
|
108
117
|
|
|
118
|
+
# The first timed pass also prices the run against --max-duration:
|
|
119
|
+
# boot and this pass are spent, the remaining passes and a cold
|
|
120
|
+
# coverage build cost about that much again at that rate, and only
|
|
121
|
+
# what is left feeds mutants. Warn while a Ctrl-C is still cheap -
|
|
122
|
+
# the refusal only fires once the coverage build is unaffordable
|
|
123
|
+
# outright (#637).
|
|
124
|
+
def warn_budget_projection(pass_elapsed, remaining_passes)
|
|
125
|
+
budget = @config["max_duration"]
|
|
126
|
+
return unless budget && @run_started
|
|
127
|
+
|
|
128
|
+
spent = Process.clock_gettime(Process::CLOCK_MONOTONIC) - @run_started
|
|
129
|
+
projected = spent + (pass_elapsed * remaining_passes)
|
|
130
|
+
mutants_get = budget - projected
|
|
131
|
+
return if mutants_get >= pass_elapsed
|
|
132
|
+
|
|
133
|
+
verdict = mutants_get <= 0 ? "exceeds" : "barely fits"
|
|
134
|
+
@diagnostics.warn("projected baseline+coverage ~#{projected.round}s #{verdict} " \
|
|
135
|
+
"--max-duration #{budget.round}s; mutants would get " \
|
|
136
|
+
"~#{format('%g', [mutants_get, 0].max.round(1))}s. Raise --max-duration, pass fewer " \
|
|
137
|
+
"--baseline-passes, or narrow the scope")
|
|
138
|
+
end
|
|
139
|
+
|
|
109
140
|
# An admitted pass still gets only the time left in the run budget:
|
|
110
141
|
# without the cap it would overrun a spent --max-duration by its
|
|
111
142
|
# whole coverage_timeout.
|
|
@@ -209,6 +240,41 @@ module Saboteur
|
|
|
209
240
|
writer.close
|
|
210
241
|
end
|
|
211
242
|
|
|
243
|
+
# A failed unchanged suite never runs mutants: warn first when the
|
|
244
|
+
# failure carries the Minitest signature (#626), then refuse.
|
|
245
|
+
def refuse_suite_failure(number, data)
|
|
246
|
+
warn_minitest_signature(number)
|
|
247
|
+
refuse_failed_suite(number, data)
|
|
248
|
+
end
|
|
249
|
+
|
|
250
|
+
# Minitest::Spec suites in *_spec.rb clothing load under the RSpec
|
|
251
|
+
# integration and then fail the baseline with NoMethodError for
|
|
252
|
+
# Minitest-only methods (#626): the refusal still reads "fix the
|
|
253
|
+
# suite", so warn what the signature actually points at. It stays a
|
|
254
|
+
# warning: an RSpec suite can require minitest for a helper and
|
|
255
|
+
# fail for its own reason, and a heuristic refusal would
|
|
256
|
+
# false-positive there.
|
|
257
|
+
def warn_minitest_signature(number)
|
|
258
|
+
lines = log_lines(number)
|
|
259
|
+
errors = lines.grep(/NoMethodError/)
|
|
260
|
+
return if errors.empty? || !spec_helper_requires_minitest?
|
|
261
|
+
|
|
262
|
+
symptom = (lines.find { |line| line.include?("undefined method") } || errors.first).strip
|
|
263
|
+
@diagnostics.warn("the baseline failure looks like a Minitest suite running under RSpec, not a " \
|
|
264
|
+
"broken RSpec suite: the log shows #{errors.size} " \
|
|
265
|
+
"NoMethodError#{'s' if errors.size > 1} (for example: #{symptom.slice(0, EXCERPT_WIDTH)}) " \
|
|
266
|
+
"and spec/spec_helper.rb requires minitest. saboteur is RSpec-only, so a " \
|
|
267
|
+
"Minitest suite cannot be mutation tested with it.")
|
|
268
|
+
end
|
|
269
|
+
|
|
270
|
+
# The framework check behind the warning: spec_helper is the
|
|
271
|
+
# suite's own declaration of what it runs on.
|
|
272
|
+
def spec_helper_requires_minitest?
|
|
273
|
+
File.read(File.join(@config.root, "spec", "spec_helper.rb")).match?(/^\s*require\s+["']minitest/)
|
|
274
|
+
rescue SystemCallError
|
|
275
|
+
false
|
|
276
|
+
end
|
|
277
|
+
|
|
212
278
|
# A child that recognises its own failure shape (an empty sqlite
|
|
213
279
|
# :memory: database after the fork, #387) says how to fix it.
|
|
214
280
|
def with_hint(reason, data)
|
|
@@ -230,9 +296,9 @@ module Saboteur
|
|
|
230
296
|
"the suite passed but left #{count} child #{count == 1 ? 'process' : 'processes'} behind: #{detail}"
|
|
231
297
|
end
|
|
232
298
|
|
|
233
|
-
def refuse(number, reason, excerpt: true)
|
|
299
|
+
def refuse(number, reason, excerpt: true, tail: false)
|
|
234
300
|
raise Error, "baseline check #{number} failed at seed #{@config['seed']}: #{reason}. " \
|
|
235
|
-
"#{failure_excerpt(number) if excerpt}See the full log at #{log_path(number)}. " \
|
|
301
|
+
"#{failure_excerpt(number, tail: tail) if excerpt}See the full log at #{log_path(number)}. " \
|
|
236
302
|
"Fix the suite or its setup, then run saboteur again with the same seed. " \
|
|
237
303
|
"No mutants were started."
|
|
238
304
|
end
|
|
@@ -240,11 +306,11 @@ module Saboteur
|
|
|
240
306
|
# The first lines of the failure, inline, so the cause is readable
|
|
241
307
|
# without opening the log: RSpec's "Failures:" section when the log has
|
|
242
308
|
# one, otherwise the last lines of the log.
|
|
243
|
-
def failure_excerpt(number)
|
|
309
|
+
def failure_excerpt(number, tail: false)
|
|
244
310
|
lines = log_lines(number)
|
|
245
311
|
return "" if lines.empty?
|
|
246
312
|
|
|
247
|
-
start = lines.index { |line| line.start_with?("Failures:") }
|
|
313
|
+
start = lines.index { |line| line.start_with?("Failures:") } unless tail
|
|
248
314
|
shown = start ? lines[start, EXCERPT_LINES] : lines.last(EXCERPT_LINES)
|
|
249
315
|
"The log says:\n#{shown.map { |line| " #{line.slice(0, EXCERPT_WIDTH)}" }.join("\n")}\n"
|
|
250
316
|
end
|
data/lib/saboteur/engine/boot.rb
CHANGED
|
@@ -66,7 +66,9 @@ module Saboteur
|
|
|
66
66
|
@rails = rails = Frameworks::Rails.new(@config, @hooks) if @config["rails"]
|
|
67
67
|
rails&.before_boot
|
|
68
68
|
@hooks.run(:before_boot)
|
|
69
|
-
add_spec_to_load_path
|
|
69
|
+
add_spec_to_load_path
|
|
70
|
+
# Helpers may configure RSpec before the integration loads specs.
|
|
71
|
+
Integration::RSpec.new(@config).require_framework if @config["integration"] == "rspec"
|
|
70
72
|
@config["require"].each { |file| require File.expand_path(file, @root) }
|
|
71
73
|
integration = Integration.fetch(@config["integration"]).new(@config)
|
|
72
74
|
integration.setup
|
|
@@ -78,8 +80,10 @@ module Saboteur
|
|
|
78
80
|
private
|
|
79
81
|
|
|
80
82
|
# rspec puts spec/ on the load path, so a boot file under spec/ (a
|
|
81
|
-
# dummy app's rails_helper,
|
|
82
|
-
#
|
|
83
|
+
# dummy app's rails_helper, or a non-Rails gem's spec_helper that
|
|
84
|
+
# requires its siblings) can require its siblings. Match that for
|
|
85
|
+
# every project - a non-Rails gem does not add spec/ itself (#615) -
|
|
86
|
+
# or the same require list fails to load.
|
|
83
87
|
def add_spec_to_load_path
|
|
84
88
|
path = File.join(@root, "spec")
|
|
85
89
|
$LOAD_PATH.unshift(path) unless $LOAD_PATH.include?(path)
|
|
@@ -42,11 +42,27 @@ module Saboteur
|
|
|
42
42
|
|
|
43
43
|
def fingerprint(text, frozen)
|
|
44
44
|
options = { frozen_string_literal: frozen }
|
|
45
|
-
|
|
45
|
+
iseq = warnings_silenced do
|
|
46
|
+
RubyVM::InstructionSequence.compile(text, "saboteur/bytecode-equivalence", nil, 1, options)
|
|
47
|
+
end
|
|
48
|
+
strip_positions(iseq.to_a)
|
|
46
49
|
rescue SyntaxError, StandardError
|
|
47
50
|
nil
|
|
48
51
|
end
|
|
49
52
|
|
|
53
|
+
# Mutants compile one by one, and generated bodies (body_nil,
|
|
54
|
+
# statement_removal) trip Ruby's own void-context warnings, so a host
|
|
55
|
+
# suite running with -w gets one line per mutant flooding stderr
|
|
56
|
+
# (#592, #656). The check is internal: silence the compiler. A mutant
|
|
57
|
+
# that does not parse still fails the compile and runs.
|
|
58
|
+
def warnings_silenced
|
|
59
|
+
saved = $VERBOSE
|
|
60
|
+
$VERBOSE = nil
|
|
61
|
+
yield
|
|
62
|
+
ensure
|
|
63
|
+
$VERBOSE = saved
|
|
64
|
+
end
|
|
65
|
+
|
|
50
66
|
# Structural equality that never coerces: an Integer is not a Float,
|
|
51
67
|
# and +0.0 is not -0.0 (1.0 / +0.0 and 1.0 / -0.0 are different
|
|
52
68
|
# infinities), which eql? alone would not tell apart.
|
|
@@ -66,6 +66,7 @@ module Saboteur
|
|
|
66
66
|
# Load-only code is not "no coverage": it ran, just never inside a
|
|
67
67
|
# test, and no forked test can re-run it (see CoverageMap::RAN_AT_LOAD).
|
|
68
68
|
def skip_reason_for(subject)
|
|
69
|
+
return Selection::CoverageMap::PRELOADED if @map.preloaded?(subject.file)
|
|
69
70
|
return "no coverage" unless @map.ran_at_load?(subject.file, subject.coverage_lines)
|
|
70
71
|
|
|
71
72
|
Selection::CoverageMap::RAN_AT_LOAD
|
|
@@ -40,10 +40,11 @@ module Saboteur
|
|
|
40
40
|
|
|
41
41
|
def fetch(files)
|
|
42
42
|
path = cache_path(files)
|
|
43
|
+
preloaded = LoadCoverage.preloaded_uninstrumented(files)
|
|
43
44
|
cached = cached_map(files)
|
|
44
|
-
return refuse_empty_map(cached, files) if cached
|
|
45
|
+
return refuse_empty_map(cached.with_preloaded(preloaded), files, preloaded) if cached
|
|
45
46
|
|
|
46
|
-
map = refuse_empty_map(build(files), files)
|
|
47
|
+
map = refuse_empty_map(build(files).with_preloaded(preloaded), files, preloaded)
|
|
47
48
|
map.save(path)
|
|
48
49
|
File.write(File.join(@config.path("cache_dir"), "preview.json"),
|
|
49
50
|
JSON.generate(path: path, fingerprint: RunFingerprint.new(config: @config).snapshot.digest))
|
|
@@ -157,9 +158,20 @@ module Saboteur
|
|
|
157
158
|
# these sources, so the map passes with a warning. Applies to a cached
|
|
158
159
|
# map too, so a map cached before this check cannot keep a run green,
|
|
159
160
|
# and the guard runs before saving, so an empty map is never cached.
|
|
160
|
-
def refuse_empty_map(map, files)
|
|
161
|
+
def refuse_empty_map(map, files, preloaded)
|
|
161
162
|
return map if files.empty? || !map.empty_for?(files)
|
|
162
163
|
|
|
164
|
+
# A file loaded before the executable's Coverage.start can never be
|
|
165
|
+
# instrumented, so emptiness there is a setup defect, not untested
|
|
166
|
+
# code: name it apart from the generic no-coverage refusal.
|
|
167
|
+
preloaded_empty = preloaded & files.map { |file| Selection::CoverageMap.canonical(file) }
|
|
168
|
+
if preloaded_empty.any?
|
|
169
|
+
message = preloaded_message(preloaded_empty)
|
|
170
|
+
raise Error, message if @config["tests"].empty?
|
|
171
|
+
|
|
172
|
+
@diagnostics.warn(message)
|
|
173
|
+
return map
|
|
174
|
+
end
|
|
163
175
|
if @config["tests"].empty?
|
|
164
176
|
raise Error, "Coverage recorded nothing in #{files.size} included files during a whole-suite " \
|
|
165
177
|
"build. No test in the suite may exercise these files, or the setup hid the app " \
|
|
@@ -170,6 +182,16 @@ module Saboteur
|
|
|
170
182
|
map
|
|
171
183
|
end
|
|
172
184
|
|
|
185
|
+
def preloaded_message(files)
|
|
186
|
+
names = files.first(5).map { |file| file.delete_prefix("#{@root}/") }
|
|
187
|
+
more = files.size > 5 ? ", ..." : ""
|
|
188
|
+
"#{names.join(', ')}#{more} loaded before saboteur started coverage (a require ran before the " \
|
|
189
|
+
"executable's Coverage.start - commonly the gemspec requiring the library while bundler/setup " \
|
|
190
|
+
"evaluates it, or RUBYOPT -r), so no test can ever cover #{files.size > 1 ? 'them' : 'it'}. " \
|
|
191
|
+
"Let the test suite load these " \
|
|
192
|
+
"files; a gemspec should require only its version file."
|
|
193
|
+
end
|
|
194
|
+
|
|
173
195
|
# Bootsnap is named only when it is loaded: blaming a library the app
|
|
174
196
|
# does not use sends the reader to the wrong place.
|
|
175
197
|
def bootsnap_note
|
|
@@ -74,7 +74,7 @@ module Saboteur
|
|
|
74
74
|
DiscoveryFailure = Data.define(:file, :reason)
|
|
75
75
|
|
|
76
76
|
attr_reader :warnings, :invalid_mutations, :generation_events, :omissions, :discovery_failures,
|
|
77
|
-
:hazard_suppressions, :inline_suppressions
|
|
77
|
+
:hazard_suppressions, :inline_suppressions, :verbose
|
|
78
78
|
|
|
79
79
|
def initialize(err, verbose: false)
|
|
80
80
|
@err = err
|
|
@@ -26,7 +26,9 @@ module Saboteur
|
|
|
26
26
|
# a gated run keep the hard stop.
|
|
27
27
|
def stop_or_warn_unmapped(guess)
|
|
28
28
|
return if guess.unmatched.empty?
|
|
29
|
-
|
|
29
|
+
if !config["since"] || guess.specs.empty? || gated?
|
|
30
|
+
raise ConfigError, guess.unmatched_message(verbose: @diagnostics.verbose)
|
|
31
|
+
end
|
|
30
32
|
|
|
31
33
|
guess.unmatched.each { |file| @diagnostics.announce(guess.unmatched_warning(file)) }
|
|
32
34
|
end
|
|
@@ -32,6 +32,24 @@ module Saboteur
|
|
|
32
32
|
end
|
|
33
33
|
end
|
|
34
34
|
|
|
35
|
+
# Include-listed files that are already loaded yet carry no coverage
|
|
36
|
+
# instrumentation: something required them before the executable's
|
|
37
|
+
# Coverage.start (a gemspec requiring the library while bundler/setup
|
|
38
|
+
# evaluates it, or RUBYOPT -r), so no test can ever be attributed to
|
|
39
|
+
# them. Read in the parent, where the snapshot still reflects the boot.
|
|
40
|
+
def self.preloaded_uninstrumented(files, loaded: $LOADED_FEATURES)
|
|
41
|
+
return [] unless Coverage.state == :suspended
|
|
42
|
+
|
|
43
|
+
instrumented = CoverageReads.result(stop: false, clear: false)
|
|
44
|
+
.each_key
|
|
45
|
+
.to_set { |path| Selection::CoverageMap.canonical(path) }
|
|
46
|
+
loaded_set = loaded.to_set { |path| Selection::CoverageMap.canonical(path) }
|
|
47
|
+
files.map { |file| Selection::CoverageMap.canonical(file) }
|
|
48
|
+
.uniq
|
|
49
|
+
.reject { |file| instrumented.include?(file) }
|
|
50
|
+
.select { |file| loaded_set.include?(file) }
|
|
51
|
+
end
|
|
52
|
+
|
|
35
53
|
# A one-line or endless definition executes its def line even when its
|
|
36
54
|
# body is never called. If any sibling on that physical line is uncalled,
|
|
37
55
|
# discard its ambiguous line evidence rather than crediting every body.
|
|
@@ -1,6 +1,8 @@
|
|
|
1
1
|
# SPDX-License-Identifier: MIT
|
|
2
2
|
# frozen_string_literal: true
|
|
3
3
|
|
|
4
|
+
require_relative "../execution/test_command_shape"
|
|
5
|
+
|
|
4
6
|
module Saboteur
|
|
5
7
|
class Engine
|
|
6
8
|
# The two run pipelines: in-process (boot, coverage, workers) and the
|
|
@@ -43,6 +45,12 @@ module Saboteur
|
|
|
43
45
|
def call_external(ignore_list, started)
|
|
44
46
|
@diagnostics.inform("external test command: #{config['test_command']} " \
|
|
45
47
|
"(serial, no coverage, unsandboxed, mutants applied in a disposable copy)")
|
|
48
|
+
if Execution::TestCommandShape.multi_file_ruby?(config["test_command"])
|
|
49
|
+
@diagnostics.warn("the test command hands several test files to one ruby: ruby executes only " \
|
|
50
|
+
"the first, so the rest never run and their mutants silently survive (#611) - " \
|
|
51
|
+
"use the framework's multi-file form (bin/rails test f1 f2, rspec f1 f2) or a " \
|
|
52
|
+
"shell loop (for f in ...; do ruby \"$f\" || exit 1; done)")
|
|
53
|
+
end
|
|
46
54
|
files = config.included_files
|
|
47
55
|
subjects, scope = select_subjects(files)
|
|
48
56
|
results = subjects.empty? ? [] : mutate_external(subjects, scope, ignore_list)
|
|
@@ -35,8 +35,11 @@ module Saboteur
|
|
|
35
35
|
|
|
36
36
|
# Absolute monotonic deadline shared with the schedulers (forked
|
|
37
37
|
# workers inherit it): a subject's remaining mutants stop once it
|
|
38
|
-
# passes
|
|
39
|
-
#
|
|
38
|
+
# passes. New mutation work never overshoots more than the fork in
|
|
39
|
+
# flight; a subject the deadline cut mid-list still confirms the
|
|
40
|
+
# kills it already measured (#588), and those checks and retries
|
|
41
|
+
# fork uncapped - the overshoot is bounded by the subject's
|
|
42
|
+
# distinct killing sequences times their full timeout.
|
|
40
43
|
def budget_deadline
|
|
41
44
|
config["max_duration"] && (@run_started + config["max_duration"])
|
|
42
45
|
end
|
|
@@ -39,8 +39,7 @@ module Saboteur
|
|
|
39
39
|
# same line twice (D's review, #178).
|
|
40
40
|
unless @sandbox_note_given
|
|
41
41
|
@sandbox_note_given = true
|
|
42
|
-
@diagnostics.warn(
|
|
43
|
-
"and canary only")
|
|
42
|
+
@diagnostics.warn(degraded_note(abi))
|
|
44
43
|
end
|
|
45
44
|
nil
|
|
46
45
|
end
|
|
@@ -55,11 +54,30 @@ module Saboteur
|
|
|
55
54
|
end
|
|
56
55
|
|
|
57
56
|
def self.sandbox_gap(abi)
|
|
58
|
-
|
|
57
|
+
if abi.zero?
|
|
58
|
+
# Landlock is Linux-only: on macOS there is no kernel to upgrade,
|
|
59
|
+
# so name the platform instead of a Linux version (#667).
|
|
60
|
+
return "macOS has no Landlock" if Execution::Seatbelt.darwin?
|
|
61
|
+
|
|
62
|
+
return "this kernel lacks Landlock (needs Linux 6.2+)"
|
|
63
|
+
end
|
|
59
64
|
|
|
60
65
|
"this kernel's Landlock ABI is #{abi} but ABI 3 (Linux 6.2+) is needed to restrict truncate()"
|
|
61
66
|
end
|
|
62
67
|
|
|
68
|
+
# The degraded-run note: on macOS the gap is permanent, so point at
|
|
69
|
+
# the Seatbelt opt-in; on Linux the kernel wording stands alone
|
|
70
|
+
# (#667). Severity stays a warning - degrading confinement is never
|
|
71
|
+
# routine.
|
|
72
|
+
def degraded_note(abi)
|
|
73
|
+
gap = self.class.sandbox_gap(abi)
|
|
74
|
+
tail = "children run with the hazard guard and canary only"
|
|
75
|
+
return "sandbox is on but #{gap}; #{tail}" unless Execution::Seatbelt.darwin?
|
|
76
|
+
|
|
77
|
+
"sandbox is on but #{gap}; #{tail} - opt into the macOS sandbox with c.sandbox = \"seatbelt\" " \
|
|
78
|
+
"(docs/guides/macos-seatbelt-sandbox.md), or set c.sandbox = false to silence this"
|
|
79
|
+
end
|
|
80
|
+
|
|
63
81
|
def restrict_hook
|
|
64
82
|
writable = sandbox_writable_dirs
|
|
65
83
|
->(keep_fds) { Execution::Sandbox.restrict!(writable: writable, keep_fds: keep_fds) }
|
|
@@ -64,7 +64,7 @@ module Saboteur
|
|
|
64
64
|
results = run_candidates(candidates) do |mutation, text|
|
|
65
65
|
reuse(plan, mutation, text) || execute(plan, mutation, text)
|
|
66
66
|
end
|
|
67
|
-
checked(plan, results)
|
|
67
|
+
checked(plan, results, truncated: results.size < candidates.size)
|
|
68
68
|
end
|
|
69
69
|
|
|
70
70
|
# The per-mutant timeout: timeout_factor times what the tests took
|
|
@@ -81,14 +81,21 @@ module Saboteur
|
|
|
81
81
|
|
|
82
82
|
# Verdicts that did not hold up are warned about once per subject;
|
|
83
83
|
# only checked verdicts are stored for --incremental.
|
|
84
|
-
# A spent budget admits no check
|
|
85
|
-
#
|
|
86
|
-
#
|
|
87
|
-
#
|
|
88
|
-
# reports
|
|
89
|
-
|
|
90
|
-
|
|
91
|
-
|
|
84
|
+
# A spent budget admits no check for a subject that finished its
|
|
85
|
+
# mutants: forking the unchanged run with no time left would turn
|
|
86
|
+
# genuine kills into fabricated errors and cache them. Fresh kills
|
|
87
|
+
# stand unverified instead - outside the score, never cached - and
|
|
88
|
+
# the distribution marks the subject so the run reports
|
|
89
|
+
# budget-partial.
|
|
90
|
+
# A subject the budget stopped mid-list is different (#588): its
|
|
91
|
+
# completed mutants were already measured, so their checks still
|
|
92
|
+
# run, uncapped - the budget stops new mutation work, not the
|
|
93
|
+
# confirmation of work already done. The overshoot stays bounded by
|
|
94
|
+
# the subject's distinct killing sequences.
|
|
95
|
+
def checked(plan, results, truncated: false)
|
|
96
|
+
return results.map { |result| unverifiable(result) } if budget_spent? && !truncated
|
|
97
|
+
|
|
98
|
+
@confirming_truncation = truncated
|
|
92
99
|
subject = plan.subject
|
|
93
100
|
check = VerdictCheck.new(run_unchanged: unchanged_runner(subject, plan),
|
|
94
101
|
baseline_timeout: @config["baseline_timeout"])
|
|
@@ -101,6 +108,8 @@ module Saboteur
|
|
|
101
108
|
@diagnostics.warn("#{subject.id}: #{overturned} verdict(s) failed with the method unchanged")
|
|
102
109
|
end
|
|
103
110
|
final
|
|
111
|
+
ensure
|
|
112
|
+
@confirming_truncation = false
|
|
104
113
|
end
|
|
105
114
|
|
|
106
115
|
# A kill rejected because its covering examples fail with the method
|
|
@@ -111,10 +120,12 @@ module Saboteur
|
|
|
111
120
|
# mutant's run on the same examples. Otherwise the error stays.
|
|
112
121
|
#
|
|
113
122
|
# Cost bound: at most two extra forks per rejected kill, the unchanged
|
|
114
|
-
# prefix run and one mutant run.
|
|
115
|
-
# other fork: neither starts once the budget is spent, and
|
|
116
|
-
# more than the time left. A retry the budget stops is
|
|
117
|
-
# an error or a kill.
|
|
123
|
+
# prefix run and one mutant run. Ordinarily both spend --max-duration
|
|
124
|
+
# like every other fork: neither starts once the budget is spent, and
|
|
125
|
+
# neither gets more than the time left. A retry the budget stops is
|
|
126
|
+
# unverified, not an error or a kill. The exception is a truncated
|
|
127
|
+
# subject (#588): its completed kills still retry, at the full
|
|
128
|
+
# timeout. Keep this bound if the retry ever grows.
|
|
118
129
|
def retried_in_defined_order(plan, result)
|
|
119
130
|
return result unless result.rejected_kill? && !result.reused? && result.covered_by.any?
|
|
120
131
|
|
|
@@ -129,10 +140,10 @@ module Saboteur
|
|
|
129
140
|
end
|
|
130
141
|
|
|
131
142
|
def rerun_in_prefix(plan, result, tests)
|
|
132
|
-
return result.as_unverified(BUDGET_RETRY_REASON) if budget_spent?
|
|
143
|
+
return result.as_unverified(BUDGET_RETRY_REASON) if budget_spent? && !@confirming_truncation
|
|
133
144
|
|
|
134
145
|
full = timeout_for(tests, plan.slowdown)
|
|
135
|
-
capped = budget_capped(full)
|
|
146
|
+
capped = @confirming_truncation ? full : budget_capped(full)
|
|
136
147
|
outcome = @runner.run(subject: plan.subject, text: result.mutated_source, test_ids: tests,
|
|
137
148
|
timeout: capped, mutation: result.mutation, eval_prefix: eval_prefix(plan))
|
|
138
149
|
return result.as_unverified(BUDGET_RETRY_REASON) if outcome.status == :timeout && capped < full
|
|
@@ -150,11 +161,11 @@ module Saboteur
|
|
|
150
161
|
# the check never got its answer (#296).
|
|
151
162
|
def unchanged_runner(subject, plan)
|
|
152
163
|
lambda do |tests, timeout|
|
|
153
|
-
if budget_spent?
|
|
164
|
+
if budget_spent? && !@confirming_truncation
|
|
154
165
|
Execution::Runner::Outcome.new(status: :unverified, runtime: 0.0,
|
|
155
166
|
message: "the --max-duration budget is spent")
|
|
156
167
|
else
|
|
157
|
-
capped = budget_capped(timeout)
|
|
168
|
+
capped = @confirming_truncation ? timeout : budget_capped(timeout)
|
|
158
169
|
outcome = @runner.run(subject: subject, text: subject.source, test_ids: tests, timeout: capped,
|
|
159
170
|
eval_prefix: eval_prefix(plan))
|
|
160
171
|
note_leftovers(subject, "verdict check", outcome)
|
|
@@ -238,17 +249,14 @@ module Saboteur
|
|
|
238
249
|
[@config["timeout_factor"] * runtime, @config["timeout_min"]].max
|
|
239
250
|
end
|
|
240
251
|
|
|
241
|
-
# What a subject no test ran gets.
|
|
242
|
-
#
|
|
243
|
-
# it, so it gets its own status, out of the score, with a hint.
|
|
252
|
+
# What a subject no test ran gets. Load-only code is not "no coverage":
|
|
253
|
+
# it ran, but no forked test can re-run it, so it gets its own status.
|
|
244
254
|
def unloadable(candidates, subject, line_numbers = nil)
|
|
255
|
+
return skip_all(candidates, subject, Selection::CoverageMap::PRELOADED) if @map.preloaded?(subject.file)
|
|
245
256
|
return skip_all(candidates, subject, "no coverage") unless
|
|
246
257
|
@map.ran_at_load?(subject.file, line_numbers || subject.coverage_lines)
|
|
247
258
|
|
|
248
|
-
candidates
|
|
249
|
-
MutantResult.new(subject: subject, mutation: mutation, status: :ran_at_load, mutated_source: text,
|
|
250
|
-
reason: Selection::CoverageMap::RAN_AT_LOAD)
|
|
251
|
-
end
|
|
259
|
+
load_time_results(candidates, subject)
|
|
252
260
|
end
|
|
253
261
|
|
|
254
262
|
# A def-time macro's only callers ran while the app loaded: the
|
|
@@ -10,10 +10,15 @@ module Saboteur
|
|
|
10
10
|
module SelectionAnnouncement
|
|
11
11
|
private
|
|
12
12
|
|
|
13
|
+
# A --since diff with no changed source file (a spec-only or docs-only
|
|
14
|
+
# change) yields zero subjects, so the suite never runs: say there is
|
|
15
|
+
# nothing to test rather than promise a whole-suite run that does not
|
|
16
|
+
# happen (#607).
|
|
13
17
|
def selection_message(guess)
|
|
14
|
-
if guess.changed.zero?
|
|
15
|
-
|
|
16
|
-
|
|
18
|
+
if guess.changed.zero? && config["since"]
|
|
19
|
+
"inferred specs: no changed source file to infer a spec for, so there is nothing to test"
|
|
20
|
+
elsif guess.changed.zero?
|
|
21
|
+
"inferred specs: no source file to infer a spec for, so the whole suite runs"
|
|
17
22
|
elsif guess.specs.empty?
|
|
18
23
|
guess.no_specs_message(since: config["since"])
|
|
19
24
|
else
|