saboteur 0.11.0 → 0.11.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +86 -0
- data/README.md +2 -2
- data/lib/saboteur/baseline/command.rb +46 -2
- data/lib/saboteur/baseline/comparison.rb +6 -1
- data/lib/saboteur/baseline/completeness.rb +21 -0
- data/lib/saboteur/baseline/evidence.rb +11 -3
- data/lib/saboteur/baseline/loader.rb +18 -5
- data/lib/saboteur/baseline/manifest.rb +13 -4
- data/lib/saboteur/baseline/run_evidence.rb +3 -1
- data/lib/saboteur/baseline/workflow.rb +57 -9
- data/lib/saboteur/baseline_health.rb +4 -1
- data/lib/saboteur/cli/dirty_tree_notice.rb +7 -1
- data/lib/saboteur/cli/doctor_command.rb +10 -1
- data/lib/saboteur/cli/exit_status.rb +10 -5
- data/lib/saboteur/cli/explain_command.rb +51 -0
- data/lib/saboteur/cli/explicit_sources.rb +1 -1
- data/lib/saboteur/cli/failure_output.rb +2 -2
- data/lib/saboteur/cli/flaky_exit.rb +23 -0
- data/lib/saboteur/cli/presets.rb +11 -1
- data/lib/saboteur/cli/run_command.rb +17 -1
- data/lib/saboteur/cli/run_output.rb +10 -0
- data/lib/saboteur/cli/run_reporting.rb +13 -2
- data/lib/saboteur/cli/survivors_command.rb +82 -18
- data/lib/saboteur/cli.rb +3 -0
- data/lib/saboteur/config.rb +7 -2
- data/lib/saboteur/crash_report.rb +3 -2
- data/lib/saboteur/doctor.rb +11 -2
- data/lib/saboteur/engine/boot.rb +4 -2
- data/lib/saboteur/engine/coverage_store.rb +54 -3
- data/lib/saboteur/engine/distribution.rb +6 -2
- data/lib/saboteur/engine/pipelines.rb +14 -2
- data/lib/saboteur/engine/result_cache.rb +4 -2
- data/lib/saboteur/engine/scheduler.rb +23 -21
- data/lib/saboteur/engine/selection_announcement.rb +5 -4
- data/lib/saboteur/engine/subject_selection.rb +11 -1
- data/lib/saboteur/engine/survivor_confirmation.rb +1 -1
- data/lib/saboteur/engine/test_guess.rb +1 -1
- data/lib/saboteur/engine/thread_warnings.rb +9 -4
- data/lib/saboteur/engine.rb +57 -9
- data/lib/saboteur/execution/fiddle_loader.rb +29 -2
- data/lib/saboteur/help.rb +13 -1
- data/lib/saboteur/ignore_list.rb +36 -3
- data/lib/saboteur/install.rb +18 -3
- data/lib/saboteur/mutation_source/builtin.rb +1 -0
- data/lib/saboteur/mutation_source/operator_explain.rb +181 -0
- data/lib/saboteur/preview.rb +6 -1
- data/lib/saboteur/reporter/agent_commands.rb +29 -7
- data/lib/saboteur/reporter/agent_incomplete.rb +9 -2
- data/lib/saboteur/reporter/agent_summary.rb +2 -1
- data/lib/saboteur/reporter/baseline_tail.rb +9 -7
- data/lib/saboteur/reporter/gap_kinds.rb +80 -0
- data/lib/saboteur/reporter/html.rb +160 -10
- data/lib/saboteur/reporter/next_steps.rb +9 -7
- data/lib/saboteur/reporter/notice.rb +24 -0
- data/lib/saboteur/reporter/reuse_note.rb +37 -0
- data/lib/saboteur/reporter/review_comments.rb +17 -12
- data/lib/saboteur/reporter/score_table.rb +8 -7
- data/lib/saboteur/reporter/stryker.rb +11 -1
- data/lib/saboteur/reporter/survivor_classification.rb +26 -8
- data/lib/saboteur/reporter/survivor_hints.rb +3 -2
- data/lib/saboteur/reporter/survivor_list.rb +58 -4
- data/lib/saboteur/reporter/survivor_ranking.rb +13 -5
- data/lib/saboteur/reporter/terminal.rb +32 -33
- data/lib/saboteur/reporter/wording.rb +37 -0
- data/lib/saboteur/result.rb +39 -8
- data/lib/saboteur/selection/git_diff.rb +19 -1
- data/lib/saboteur/survivor_guidance.rb +33 -0
- data/lib/saboteur/text.rb +6 -0
- data/lib/saboteur/version.rb +1 -1
- metadata +10 -2
|
@@ -77,6 +77,28 @@ module Saboteur
|
|
|
77
77
|
deadline(recorded * slowdown)
|
|
78
78
|
end
|
|
79
79
|
|
|
80
|
+
# Survivor confirmation (#1063) drives the runner and the coverage map
|
|
81
|
+
# from outside after the run.
|
|
82
|
+
attr_reader :runner, :map
|
|
83
|
+
|
|
84
|
+
# The eval prefix a post-run confirmation run needs for this subject
|
|
85
|
+
# (#1063): the same wrapper detection load_problem applies, reduced to
|
|
86
|
+
# the sorbet sig prefix a survivor could have run with.
|
|
87
|
+
def confirmation_prefix(subject)
|
|
88
|
+
return nil unless @verify_loaded
|
|
89
|
+
return nil if @reevaluator.whole_file?(subject)
|
|
90
|
+
|
|
91
|
+
owner = @reevaluator.resolve_owner(subject)
|
|
92
|
+
target = subject.singleton ? owner.singleton_class : owner
|
|
93
|
+
method = own_method(target, subject.name)
|
|
94
|
+
return nil unless method
|
|
95
|
+
|
|
96
|
+
file, = method.source_location
|
|
97
|
+
Execution::Wrapper.around(method, file) == :sorbet ? subject.sig_source : nil
|
|
98
|
+
rescue Error
|
|
99
|
+
nil
|
|
100
|
+
end
|
|
101
|
+
|
|
80
102
|
private
|
|
81
103
|
|
|
82
104
|
# Verdicts that did not hold up are warned about once per subject;
|
|
@@ -232,26 +254,6 @@ module Saboteur
|
|
|
232
254
|
result
|
|
233
255
|
end
|
|
234
256
|
|
|
235
|
-
attr_reader :runner, :map
|
|
236
|
-
|
|
237
|
-
# The eval prefix a post-run confirmation run needs for this subject
|
|
238
|
-
# (#1063): the same wrapper detection load_problem applies, reduced to
|
|
239
|
-
# the sorbet sig prefix a survivor could have run with.
|
|
240
|
-
def confirmation_prefix(subject)
|
|
241
|
-
return nil unless @verify_loaded
|
|
242
|
-
return nil if @reevaluator.whole_file?(subject)
|
|
243
|
-
|
|
244
|
-
owner = @reevaluator.resolve_owner(subject)
|
|
245
|
-
target = subject.singleton ? owner.singleton_class : owner
|
|
246
|
-
method = own_method(target, subject.name)
|
|
247
|
-
return nil unless method
|
|
248
|
-
|
|
249
|
-
file, = method.source_location
|
|
250
|
-
Execution::Wrapper.around(method, file) == :sorbet ? subject.sig_source : nil
|
|
251
|
-
rescue Error
|
|
252
|
-
nil
|
|
253
|
-
end
|
|
254
|
-
|
|
255
257
|
# The tests that ran the edited code (a coarse edit spans the whole
|
|
256
258
|
# body, so it gets every test of the method), prior killers first, then
|
|
257
259
|
# fastest-first.
|
|
@@ -350,7 +352,7 @@ module Saboteur
|
|
|
350
352
|
def flaky_message(outcome, rerun)
|
|
351
353
|
decisive = (outcome.failed_ids | rerun.failed_ids).first(3)
|
|
352
354
|
named = decisive.empty? ? "" : " (decisive tests: #{decisive.join(', ')})"
|
|
353
|
-
"
|
|
355
|
+
"#{MutantResult::FLAKY_PREFIX}: first #{outcome.status}, then #{rerun.status} on a reordered confirmation " \
|
|
354
356
|
"run#{named} - the mutant is unverified, not #{outcome.status}"
|
|
355
357
|
end
|
|
356
358
|
|
|
@@ -11,12 +11,13 @@ module Saboteur
|
|
|
11
11
|
private
|
|
12
12
|
|
|
13
13
|
# A --since diff with no changed source file (a spec-only or docs-only
|
|
14
|
-
# change) yields zero subjects, so the suite never runs: say
|
|
15
|
-
#
|
|
16
|
-
# happen (#607).
|
|
14
|
+
# change) yields zero subjects, so the suite never runs: say why no
|
|
15
|
+
# spec was inferred rather than promise a whole-suite run that does not
|
|
16
|
+
# happen (#607). The conclusion itself is stated once, by the
|
|
17
|
+
# "--since <ref>: nothing to test - <reason>" outcome line (#1232).
|
|
17
18
|
def selection_message(guess)
|
|
18
19
|
if guess.changed.zero? && config["since"]
|
|
19
|
-
"inferred specs: no changed source file to infer a spec for
|
|
20
|
+
"inferred specs: no changed source file to infer a spec for"
|
|
20
21
|
elsif guess.changed.zero?
|
|
21
22
|
"inferred specs: no source file to infer a spec for, so the whole suite runs"
|
|
22
23
|
elsif guess.specs.empty?
|
|
@@ -82,7 +82,7 @@ module Saboteur
|
|
|
82
82
|
"the resolved range has no changes"
|
|
83
83
|
elsif !@raw_changed_files.intersect?(@files)
|
|
84
84
|
"#{count} changed file#{'s' unless count == 1}, none in mutation scope " \
|
|
85
|
-
"(docs
|
|
85
|
+
"(specs, docs, excluded, or non-Ruby files)"
|
|
86
86
|
else
|
|
87
87
|
"no subject covers the changed lines (layout-only edits, deletions, " \
|
|
88
88
|
"or intentionally untested code)"
|
|
@@ -169,6 +169,11 @@ module Saboteur
|
|
|
169
169
|
return missed.map { |pattern| missed_pattern_message(pattern) }.join("; ")
|
|
170
170
|
end
|
|
171
171
|
|
|
172
|
+
if @config["since"]
|
|
173
|
+
return "--subject #{missed.join(', ')} matches no subject in the --since #{@config['since']} " \
|
|
174
|
+
"diff scope (the scope limits the search to changed files; drop --since or pass --all)"
|
|
175
|
+
end
|
|
176
|
+
|
|
172
177
|
"--subject #{missed.join(', ')} matches no subject (patterns are names like " \
|
|
173
178
|
"Klass, Klass#meth or Klass.meth, not globs)"
|
|
174
179
|
end
|
|
@@ -180,6 +185,11 @@ module Saboteur
|
|
|
180
185
|
macro = attribute_macro_for(pattern)
|
|
181
186
|
return "--subject #{pattern} is defined by #{macro} and has no body to mutate" if macro
|
|
182
187
|
|
|
188
|
+
if @config["since"]
|
|
189
|
+
return "--subject #{pattern} matches no subject in the --since #{@config['since']} diff scope " \
|
|
190
|
+
"(the scope limits the search to changed files)"
|
|
191
|
+
end
|
|
192
|
+
|
|
183
193
|
"fix the --subject pattern \"#{pattern}\": it matches no subject (patterns are names like " \
|
|
184
194
|
"Klass, Klass#meth or Klass.meth, not globs)"
|
|
185
195
|
end
|
|
@@ -44,7 +44,7 @@ module Saboteur
|
|
|
44
44
|
return unconfirmed(result, "the --max-duration budget is spent") if @budget_exceeded.call
|
|
45
45
|
|
|
46
46
|
tests = unrun_covering_tests(result)
|
|
47
|
-
return result if tests.empty?
|
|
47
|
+
return unconfirmed(result, "no covering tests") if tests.empty?
|
|
48
48
|
|
|
49
49
|
outcome = run_confirmation(result, tests)
|
|
50
50
|
@remaining -= outcome.runtime || 0
|
|
@@ -81,7 +81,7 @@ module Saboteur
|
|
|
81
81
|
# a file whose own entry broke heard that in the warning (#640).
|
|
82
82
|
mapped = without_map_entry.any? &&
|
|
83
83
|
" If the layout defeats the naming convention, map #{names(without_map_entry)} by hand in " \
|
|
84
|
-
"config/saboteur_custom_map.rb
|
|
84
|
+
"config/saboteur_custom_map.rb (no entry covers it yet)."
|
|
85
85
|
"no spec found for #{pairs.join(', ')}, so nothing ran.#{changed} Re-run the same command with " \
|
|
86
86
|
"one --test per existing spec path, or append --all to use the whole suite without " \
|
|
87
87
|
"changing the source scope.#{mapped}"
|
|
@@ -17,10 +17,12 @@ module Saboteur
|
|
|
17
17
|
# fork_mode, whose default is :both, i.e. attach (RUBY_DEBUG_FORK_MODE;
|
|
18
18
|
# ruby/debug's config.rb and SESSION.activate on_fork: true in
|
|
19
19
|
# session.rb).
|
|
20
|
+
REAPER = "AR Pool Reaper"
|
|
21
|
+
|
|
20
22
|
KNOWN_FORK_THREADS = {
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
23
|
+
REAPER => "ActiveRecord's connection-pool reaper: expected harmless when children " \
|
|
24
|
+
"reconnect the database after fork (rails mode does) - it only maintains " \
|
|
25
|
+
"the parent's pool",
|
|
24
26
|
"DEBUGGER__::SESSION@server" => "the debug gem's remote-console server: what it does in " \
|
|
25
27
|
"children depends on RUBY_DEBUG_FORK_MODE - with parent no " \
|
|
26
28
|
"debugger runs in children; with child or both (the gem's " \
|
|
@@ -30,8 +32,11 @@ module Saboteur
|
|
|
30
32
|
"RUBY_DEBUG_OPEN or require \"debug\" without :open"
|
|
31
33
|
}.freeze
|
|
32
34
|
|
|
33
|
-
|
|
35
|
+
# In rails mode the children reconnect the database, so the pool
|
|
36
|
+
# reaper is not worth a warning on every run (#1229).
|
|
37
|
+
def self.warn(diagnostics, rails: false)
|
|
34
38
|
others = Thread.list - [Thread.current]
|
|
39
|
+
others = others.reject { |thread| thread.name == REAPER } if rails
|
|
35
40
|
return if others.empty?
|
|
36
41
|
|
|
37
42
|
unknown = false
|
data/lib/saboteur/engine.rb
CHANGED
|
@@ -4,6 +4,7 @@
|
|
|
4
4
|
require "fileutils"
|
|
5
5
|
require "json"
|
|
6
6
|
require_relative "replay"
|
|
7
|
+
require_relative "text"
|
|
7
8
|
require_relative "mutation_source/hazard_guard"
|
|
8
9
|
require_relative "baseline/run_evidence"
|
|
9
10
|
require_relative "engine/early_stop_recording"
|
|
@@ -30,6 +31,11 @@ module Saboteur
|
|
|
30
31
|
include GatedRun
|
|
31
32
|
include SelectionAnnouncement
|
|
32
33
|
|
|
34
|
+
# Confirmation only guards a partial coverage map, so it is silent unless
|
|
35
|
+
# the user opted into partial_coverage = use (#1228).
|
|
36
|
+
CONFIRMATION_SKIPPED = "survivor confirmation skipped: partial_coverage = use applies only to a cached " \
|
|
37
|
+
"whole-suite run (no --test, one coverage part)"
|
|
38
|
+
|
|
33
39
|
# Everything besides sources and tests that can change a verdict.
|
|
34
40
|
VERDICT_INPUTS = %w[integration require rspec_args rails reeval sources operators exclude_operators arid
|
|
35
41
|
timeout_factor timeout_min
|
|
@@ -51,7 +57,7 @@ module Saboteur
|
|
|
51
57
|
@replay = Replay.new(digest: digest) if digest
|
|
52
58
|
if capture_evidence && (digest || config["since"] || config["incremental"] || config["resume"] || config["shard"])
|
|
53
59
|
raise ConfigError,
|
|
54
|
-
"baseline evidence requires fresh whole-scope execution: no since, replay, shard or incremental"
|
|
60
|
+
"baseline evidence requires fresh whole-scope execution: no since, replay, shard, resume or incremental"
|
|
55
61
|
end
|
|
56
62
|
|
|
57
63
|
@capture_evidence = capture_evidence
|
|
@@ -111,6 +117,24 @@ module Saboteur
|
|
|
111
117
|
warnings + map.duplicate_keys(path).map { |key| "duplicate custom map key #{key}: Ruby keeps the last" }
|
|
112
118
|
end
|
|
113
119
|
|
|
120
|
+
# The map this run selected tests from: survivor presentation compares
|
|
121
|
+
# it against the cached whole-suite map (review C1).
|
|
122
|
+
def run_coverage_map
|
|
123
|
+
@coverage_store&.fetched_map
|
|
124
|
+
end
|
|
125
|
+
|
|
126
|
+
# The cached whole-suite map on this tree, for survivor presentation
|
|
127
|
+
# in scoped runs; nil when caching is off or no full build ran (review C1).
|
|
128
|
+
def whole_suite_coverage_map
|
|
129
|
+
@coverage_store&.whole_suite_map
|
|
130
|
+
end
|
|
131
|
+
|
|
132
|
+
# False when caching is off, where no coverage build can ever leave a
|
|
133
|
+
# whole-suite map to consult, so advising one would be wrong (review C1).
|
|
134
|
+
def whole_suite_map_lookup?
|
|
135
|
+
config["cache"] ? true : false
|
|
136
|
+
end
|
|
137
|
+
|
|
114
138
|
private
|
|
115
139
|
|
|
116
140
|
# Inferred test selection, the default since 0.7.0: scope the run to the
|
|
@@ -174,7 +198,7 @@ module Saboteur
|
|
|
174
198
|
hazard_suppressions: @diagnostics.hazard_suppressions, early_stop: @early_stop,
|
|
175
199
|
matrix: config["matrix"], inline_suppressions: @diagnostics.inline_suppressions,
|
|
176
200
|
operator_policy: config.operator_policy, discovery_failures: @diagnostics.discovery_failures,
|
|
177
|
-
test_command: config["test_command"],
|
|
201
|
+
test_command: config["test_command"], reuse_label: reuse_label, since: config["since"],
|
|
178
202
|
shard: @shard, memory: @memory_report, referenced: @guessed_referenced || {})
|
|
179
203
|
end
|
|
180
204
|
|
|
@@ -240,7 +264,11 @@ module Saboteur
|
|
|
240
264
|
end
|
|
241
265
|
|
|
242
266
|
def confirm_survivors(results, scheduler, cache, ignore_list)
|
|
243
|
-
|
|
267
|
+
unless CoveragePartial.use?(config, config["coverage_parts"])
|
|
268
|
+
live = results.any? { |result| result.survived? && !ignore_list.accepts?(result) }
|
|
269
|
+
@diagnostics.inform(CONFIRMATION_SKIPPED) if live && config["partial_coverage"] == "use"
|
|
270
|
+
return results
|
|
271
|
+
end
|
|
244
272
|
|
|
245
273
|
SurvivorConfirmation.new(config: config, map: scheduler.map, scheduler: scheduler, diagnostics: @diagnostics,
|
|
246
274
|
budget_exceeded: method(:budget_exceeded?), cache: cache)
|
|
@@ -303,16 +331,27 @@ module Saboteur
|
|
|
303
331
|
tests: map.tests.size, cached: @map_cached)
|
|
304
332
|
@diagnostics.quiet_inference(@quiet_spec_count)
|
|
305
333
|
else
|
|
306
|
-
@diagnostics.inform("#{subjects.size
|
|
334
|
+
@diagnostics.inform("#{Text.counted(subjects.size, 'subject')}, " \
|
|
335
|
+
"#{Text.counted(map.tests.size, 'test')} in coverage map")
|
|
307
336
|
end
|
|
308
|
-
ThreadWarnings.warn(@diagnostics)
|
|
337
|
+
ThreadWarnings.warn(@diagnostics, rails: config["rails"])
|
|
338
|
+
end
|
|
339
|
+
|
|
340
|
+
# This run's map is the whole-suite map only when nothing scoped it:
|
|
341
|
+
# no --since, no subjects, and tests either unset or inferred over every
|
|
342
|
+
# included file (never an explicit --test list) (review C1).
|
|
343
|
+
def whole_suite_run?
|
|
344
|
+
config["since"].nil? && config["subjects"].empty? &&
|
|
345
|
+
(config["tests"].empty? || !@guessed_tests.nil?)
|
|
309
346
|
end
|
|
310
347
|
|
|
311
348
|
def coverage_store(integration, load_lines, load_methods = {})
|
|
312
|
-
|
|
313
|
-
|
|
314
|
-
|
|
315
|
-
|
|
349
|
+
@coverage_store ||=
|
|
350
|
+
CoverageStore.new(config: config, integration: integration, hooks: hooks, load_lines:, load_methods:,
|
|
351
|
+
whole_suite_run: whole_suite_run?,
|
|
352
|
+
supervisor: supervisor(memory_factor: 4), diagnostics: @diagnostics,
|
|
353
|
+
timeout_cap: method(:budget_capped_timeout),
|
|
354
|
+
parts: config["coverage_parts"])
|
|
316
355
|
end
|
|
317
356
|
|
|
318
357
|
def check_baseline(integration, full_passes)
|
|
@@ -368,6 +407,15 @@ module Saboteur
|
|
|
368
407
|
coverage: map, fallback_files: source_files(files))
|
|
369
408
|
end
|
|
370
409
|
|
|
410
|
+
# Which reuse mechanism is on, named as the flag the user typed, so the
|
|
411
|
+
# terminal's reuse note never cites a flag that was not passed. A replay
|
|
412
|
+
# runs one mutant fresh by design; it does not reuse, so it gets no label.
|
|
413
|
+
def reuse_label
|
|
414
|
+
return nil if @replay
|
|
415
|
+
|
|
416
|
+
config["incremental"] ? "--incremental" : ("--resume" if config["resume"])
|
|
417
|
+
end
|
|
418
|
+
|
|
371
419
|
def verdict_inputs
|
|
372
420
|
JSON.generate(VERDICT_INPUTS.to_h { |key| [key, config[key]] }.merge("operator_policy" => config.operator_policy))
|
|
373
421
|
end
|
|
@@ -24,11 +24,38 @@ module Saboteur
|
|
|
24
24
|
# free. A LoadError means the sandbox layers cannot work: REFUSE -
|
|
25
25
|
# the error propagates, it is never downgraded to a warning.
|
|
26
26
|
def self.require_fiddle!(with_import: false)
|
|
27
|
-
|
|
28
|
-
|
|
27
|
+
quietly do
|
|
28
|
+
require "fiddle"
|
|
29
|
+
require "fiddle/import" if with_import
|
|
30
|
+
end
|
|
29
31
|
rescue LoadError
|
|
30
32
|
raise Error, MESSAGE
|
|
31
33
|
end
|
|
34
|
+
|
|
35
|
+
# Ruby 3.4 prints "fiddle was loaded from the standard library, but
|
|
36
|
+
# will no longer be part of the default gems" on this require when the
|
|
37
|
+
# Gemfile does not name it. The notice is about the user's Gemfile, not
|
|
38
|
+
# a saboteur fault, and Ruby 4.0 refuses the load outright (MESSAGE
|
|
39
|
+
# says why), so only that one notice is dropped, only during this
|
|
40
|
+
# require (#1226).
|
|
41
|
+
NOTICE = "fiddle was loaded from the standard library"
|
|
42
|
+
|
|
43
|
+
module NoticeFilter
|
|
44
|
+
def warn(message, **)
|
|
45
|
+
return if Thread.current[:saboteur_fiddle_require] && message.include?(NOTICE)
|
|
46
|
+
|
|
47
|
+
super
|
|
48
|
+
end
|
|
49
|
+
end
|
|
50
|
+
Warning.singleton_class.prepend(NoticeFilter)
|
|
51
|
+
|
|
52
|
+
def self.quietly
|
|
53
|
+
previous = Thread.current[:saboteur_fiddle_require]
|
|
54
|
+
Thread.current[:saboteur_fiddle_require] = true
|
|
55
|
+
yield
|
|
56
|
+
ensure
|
|
57
|
+
Thread.current[:saboteur_fiddle_require] = previous
|
|
58
|
+
end
|
|
32
59
|
end
|
|
33
60
|
end
|
|
34
61
|
end
|
data/lib/saboteur/help.rb
CHANGED
|
@@ -121,7 +121,8 @@ module Saboteur
|
|
|
121
121
|
"accept verifies current inputs and publishes it. No scores or execution are suppressed. " \
|
|
122
122
|
"With --epsilon POINTS, check also fails when the score drops more than POINTS percentage " \
|
|
123
123
|
"points below the accepted baseline's score. " \
|
|
124
|
-
"Only whole or subject scope; no diff, replay, incremental reuse or
|
|
124
|
+
"Only whole or subject scope; no diff, replay, shard, resume, incremental reuse or " \
|
|
125
|
+
"score threshold.",
|
|
125
126
|
takes_options: true,
|
|
126
127
|
exit_codes: { 0 => "Clean check, proposal written or accepted.",
|
|
127
128
|
1 => "New, changed or unmatched survivors, or a score drop beyond --epsilon.",
|
|
@@ -151,6 +152,17 @@ module Saboteur
|
|
|
151
152
|
["Diff, replay command and ignore template for one mutant",
|
|
152
153
|
"saboteur survivors .saboteur/report.json --show 1a2b3c4d5e6f7a8b"]]
|
|
153
154
|
),
|
|
155
|
+
Command.new(
|
|
156
|
+
name: "explain",
|
|
157
|
+
summary: "Explain one mutation operator: definition, example and kill guidance",
|
|
158
|
+
description: "Prints what one operator does, a before/after example of the mutation, the test " \
|
|
159
|
+
"shapes that let it survive, and the assertion that kills it. Covers every operator " \
|
|
160
|
+
"the registry can emit.",
|
|
161
|
+
takes_options: false,
|
|
162
|
+
exit_codes: { 0 => "Explanation printed.",
|
|
163
|
+
2 => "Unknown or missing operator name; the valid names are listed." },
|
|
164
|
+
examples: [["Why did this mutant survive?", "saboteur explain operand_replacement"]]
|
|
165
|
+
),
|
|
154
166
|
Command.new(
|
|
155
167
|
name: "badge",
|
|
156
168
|
summary: "Print a shields.io endpoint badge over the opt-in score history",
|
data/lib/saboteur/ignore_list.rb
CHANGED
|
@@ -125,7 +125,7 @@ module Saboteur
|
|
|
125
125
|
# unscoped_digests (subject id => every digest generated before a diff
|
|
126
126
|
# scope cut) keeps a scoped run from calling a scoped-out entry stale
|
|
127
127
|
# (#284).
|
|
128
|
-
def apply(results, unscoped_digests: nil)
|
|
128
|
+
def apply(results, subjects: nil, unscoped_digests: nil, full_run: false)
|
|
129
129
|
seen = {}
|
|
130
130
|
results.each do |result|
|
|
131
131
|
key = [result.subject.id, result.digest]
|
|
@@ -139,7 +139,8 @@ module Saboteur
|
|
|
139
139
|
result.triage_reason = entry.reason
|
|
140
140
|
result.ignored = entry.reason if entry.state != "todo"
|
|
141
141
|
end
|
|
142
|
-
stale_warnings(results, seen, unscoped_digests)
|
|
142
|
+
duplicate_warnings + stale_warnings(results, seen, unscoped_digests) +
|
|
143
|
+
missing_subject_warnings(results, subjects, full_run: full_run)
|
|
143
144
|
end
|
|
144
145
|
|
|
145
146
|
private
|
|
@@ -148,7 +149,39 @@ module Saboteur
|
|
|
148
149
|
examined = results.to_set { |result| result.subject.id }
|
|
149
150
|
entries.select { |entry| examined.include?(entry.subject) && !seen[[entry.subject, entry.mutation]] }
|
|
150
151
|
.reject { |entry| unscoped_digests&.fetch(entry.subject, nil)&.include?(entry.mutation) }
|
|
151
|
-
.map { |entry|
|
|
152
|
+
.map { |entry| stale_message(entry) }
|
|
153
|
+
end
|
|
154
|
+
|
|
155
|
+
# Entries naming the same (subject, mutation) more than once: the index
|
|
156
|
+
# keeps the last one, which is invisible otherwise (#1207).
|
|
157
|
+
def duplicate_warnings
|
|
158
|
+
entries.group_by { |entry| [entry.subject, entry.mutation] }
|
|
159
|
+
.reject { |_, group| group.one? }
|
|
160
|
+
.map { |(subject, mutation), _| "duplicate ignore entry #{subject} #{mutation}: the last entry wins" }
|
|
161
|
+
end
|
|
162
|
+
|
|
163
|
+
# A full run examines every in-scope subject, so an entry whose subject
|
|
164
|
+
# never appears points at code that was renamed or removed - say so, but
|
|
165
|
+
# only then: a scoped (--since/--subject) run cannot tell "renamed" from
|
|
166
|
+
# "not in this diff" (#1206). The run's subject list is the source of
|
|
167
|
+
# truth when given: an examined subject with no results (uncovered, or
|
|
168
|
+
# nothing mutable left) is not missing.
|
|
169
|
+
def missing_subject_warnings(results, subjects, full_run:)
|
|
170
|
+
return [] unless full_run
|
|
171
|
+
|
|
172
|
+
examined = (subjects || results).to_set { |item| item.respond_to?(:id) ? item.id : item.subject.id }
|
|
173
|
+
entries.reject { |entry| examined.include?(entry.subject) }
|
|
174
|
+
.map { |entry| missing_subject_message(entry) }
|
|
175
|
+
end
|
|
176
|
+
|
|
177
|
+
def missing_subject_message(entry)
|
|
178
|
+
"stale ignore entry #{entry.subject} #{entry.mutation}: subject not found " \
|
|
179
|
+
"(renamed, removed, or its file is excluded?) - remove the entry or fix the subject name"
|
|
180
|
+
end
|
|
181
|
+
|
|
182
|
+
def stale_message(entry)
|
|
183
|
+
"stale ignore entry #{entry.subject} #{entry.mutation}: no such mutation " \
|
|
184
|
+
"- remove the entry or refresh its digest (saboteur survivors --show)"
|
|
152
185
|
end
|
|
153
186
|
end
|
|
154
187
|
end
|
data/lib/saboteur/install.rb
CHANGED
|
@@ -32,6 +32,9 @@ module Saboteur
|
|
|
32
32
|
end
|
|
33
33
|
RUBY
|
|
34
34
|
|
|
35
|
+
BASE_CLASSES = %w[app/models/application_record.rb app/controllers/application_controller.rb
|
|
36
|
+
app/jobs/application_job.rb app/mailers/application_mailer.rb].freeze
|
|
37
|
+
|
|
35
38
|
BIN = <<~RUBY
|
|
36
39
|
#!/usr/bin/env ruby
|
|
37
40
|
# frozen_string_literal: true
|
|
@@ -64,19 +67,31 @@ module Saboteur
|
|
|
64
67
|
|
|
65
68
|
# Returns the list of Results (:created, :overwritten, :skipped, :appended, :unchanged).
|
|
66
69
|
def call
|
|
67
|
-
[write("config/saboteur.rb",
|
|
70
|
+
[write("config/saboteur.rb", config_text), write("bin/saboteur", BIN, mode: 0o755), gitignore]
|
|
68
71
|
end
|
|
69
72
|
|
|
70
|
-
|
|
73
|
+
RUN_HINT = "Next: bin/saboteur doctor, then bin/saboteur run --since origin/main " \
|
|
74
|
+
"(keep SimpleCov off for the run)."
|
|
75
|
+
|
|
76
|
+
# Printed after install; the Gemfile is never edited. The run pointer is
|
|
77
|
+
# always due: the Gemfile block only when the gem is missing.
|
|
71
78
|
def gemfile_hint
|
|
72
79
|
gemfile = File.join(@root, "Gemfile")
|
|
73
|
-
return
|
|
80
|
+
return RUN_HINT if File.file?(gemfile) && Text.utf8(File.read(gemfile)).match?(/^\s*gem\s+["']saboteur["']/)
|
|
74
81
|
|
|
75
82
|
GEMFILE_HINT
|
|
76
83
|
end
|
|
77
84
|
|
|
78
85
|
private
|
|
79
86
|
|
|
87
|
+
# CONFIG without the base-class excludes this app does not have: a
|
|
88
|
+
# minimal Rails app has no ApplicationJob or ApplicationMailer, and an
|
|
89
|
+
# exclude that matches no file makes doctor and every run warn.
|
|
90
|
+
def config_text
|
|
91
|
+
BASE_CLASSES.reject { |path| File.exist?(File.join(@root, path)) }
|
|
92
|
+
.reduce(CONFIG) { |text, path| text.sub(%( "#{path}",\n), "") }
|
|
93
|
+
end
|
|
94
|
+
|
|
80
95
|
# Never writes or chmods through a symlink: not the file itself, and not a
|
|
81
96
|
# symlinked directory above it (bin -> elsewhere). Without force the file is
|
|
82
97
|
# skipped; with force the link is removed and replaced by a real file or
|
|
@@ -13,6 +13,7 @@ require_relative "pair_removal"
|
|
|
13
13
|
require_relative "element_removal"
|
|
14
14
|
require_relative "endpoint_replacement"
|
|
15
15
|
require_relative "operator_registry"
|
|
16
|
+
require_relative "operator_explain"
|
|
16
17
|
require_relative "range_replacement"
|
|
17
18
|
require_relative "operator_assignment"
|
|
18
19
|
require_relative "condition_negation"
|
|
@@ -0,0 +1,181 @@
|
|
|
1
|
+
# SPDX-License-Identifier: MIT
|
|
2
|
+
# frozen_string_literal: true
|
|
3
|
+
|
|
4
|
+
require_relative "operator_examples"
|
|
5
|
+
require_relative "operator_registry"
|
|
6
|
+
|
|
7
|
+
module Saboteur
|
|
8
|
+
module MutationSource
|
|
9
|
+
# Curated text behind `saboteur explain NAME`. The definition and the
|
|
10
|
+
# example stay shared with --list-operators (one wording source); the
|
|
11
|
+
# survival and kill hints below are the only text owned here. Every
|
|
12
|
+
# registry operator needs an entry: the coverage spec fails the build
|
|
13
|
+
# when one ships without it.
|
|
14
|
+
module OperatorExplain
|
|
15
|
+
Entry = Data.define(:name, :definition, :example, :survives, :kill)
|
|
16
|
+
|
|
17
|
+
GUIDANCE = {
|
|
18
|
+
"body_nil" => {
|
|
19
|
+
survives: "the method is called for side effects and no test asserts the return value.",
|
|
20
|
+
kill: "an assertion on the exact return value, or on a caller behaviour that depends on it."
|
|
21
|
+
},
|
|
22
|
+
"body_true" => {
|
|
23
|
+
survives: "tests never check the predicate outcome, or only take the path true already forces.",
|
|
24
|
+
kill: "an example whose input makes the predicate false, asserting that path is taken."
|
|
25
|
+
},
|
|
26
|
+
"body_false" => {
|
|
27
|
+
survives: "tests never check the predicate outcome, or only take the path false already forces.",
|
|
28
|
+
kill: "an example whose input makes the predicate true, asserting that path is taken."
|
|
29
|
+
},
|
|
30
|
+
"body_raise" => {
|
|
31
|
+
survives: "tests stub or skip the method, or wrap the call in a broad rescue.",
|
|
32
|
+
kill: "any test that calls the method and lets it complete; the raise fails that example."
|
|
33
|
+
},
|
|
34
|
+
"comparison" => {
|
|
35
|
+
survives: "tests never use values at the boundary the change moves, such as equal values for < vs <=.",
|
|
36
|
+
kill: "a boundary example: the equal case, or the two values just either side of the operator."
|
|
37
|
+
},
|
|
38
|
+
"arithmetic" => {
|
|
39
|
+
survives: "tests assert the result loosely (type, presence, sign) instead of its exact value.",
|
|
40
|
+
kill: "an assertion on the exact computed value with operands where the two operators differ."
|
|
41
|
+
},
|
|
42
|
+
"boolean_logic" => {
|
|
43
|
+
survives: "tests never exercise the dropped operand deciding the outcome on its own.",
|
|
44
|
+
kill: "one example per operand where that operand alone flips the result, asserting each outcome."
|
|
45
|
+
},
|
|
46
|
+
"condition_true" => {
|
|
47
|
+
survives: "tests only exercise the branch the forced condition takes.",
|
|
48
|
+
kill: "an example with a falsy input, asserting the else branch still runs."
|
|
49
|
+
},
|
|
50
|
+
"condition_false" => {
|
|
51
|
+
survives: "tests only exercise the branch the forced condition takes.",
|
|
52
|
+
kill: "an example with a truthy input, asserting the then branch still runs."
|
|
53
|
+
},
|
|
54
|
+
"negation_removal" => {
|
|
55
|
+
survives: "tests assert loosely, or never cover the input the negation inverts.",
|
|
56
|
+
kill: "assert the behaviour for both a truthy and a falsy input through the public API."
|
|
57
|
+
},
|
|
58
|
+
"literal_number" => {
|
|
59
|
+
survives: "the literal flows into nothing a test asserts, or every fixture value works for both.",
|
|
60
|
+
kill: "an assertion on the exact value the literal feeds (a total, a limit, an index)."
|
|
61
|
+
},
|
|
62
|
+
"or_assign" => {
|
|
63
|
+
survives: "tests only cover the nil or false case, where both forms assign.",
|
|
64
|
+
kill: "an example where the variable is already truthy and must keep its value."
|
|
65
|
+
},
|
|
66
|
+
"and_assign" => {
|
|
67
|
+
survives: "tests only cover the truthy case, where both forms assign.",
|
|
68
|
+
kill: "an example where the variable is falsy and must stay untouched."
|
|
69
|
+
},
|
|
70
|
+
"safe_navigation" => {
|
|
71
|
+
survives: "the receiver is never nil in any covering test.",
|
|
72
|
+
kill: "an example with a nil receiver asserting nil comes back instead of NoMethodError."
|
|
73
|
+
},
|
|
74
|
+
"statement_removal" => {
|
|
75
|
+
survives: "the removed statement's effect is never observed: an unasserted side effect or return.",
|
|
76
|
+
kill: "assert the side effect (a spy, a state change) or the value the statement produced."
|
|
77
|
+
},
|
|
78
|
+
"copy_replacement" => {
|
|
79
|
+
survives: "tests never mutate the returned object, so the aliasing goes unnoticed.",
|
|
80
|
+
kill: "mutate what the method returns and assert the original is unchanged (or assert a copy)."
|
|
81
|
+
},
|
|
82
|
+
"filter_replacement" => {
|
|
83
|
+
survives: "tests assert loosely, or the fixture collection is empty or all one kind.",
|
|
84
|
+
kill: "assert the exact surviving elements of a mixed collection, in order."
|
|
85
|
+
},
|
|
86
|
+
"literal_string" => {
|
|
87
|
+
survives: "no test asserts the string's content, only that something was returned.",
|
|
88
|
+
kill: "an exact string equality assertion on the value that carries the literal."
|
|
89
|
+
},
|
|
90
|
+
"literal_boolean" => {
|
|
91
|
+
survives: "the boolean is never asserted, or both values pass the covering assertions.",
|
|
92
|
+
kill: "an example whose expected behaviour flips with the boolean."
|
|
93
|
+
},
|
|
94
|
+
"argument_swap" => {
|
|
95
|
+
survives: "tests use equal or interchangeable arguments, or assert the result loosely.",
|
|
96
|
+
kill: "asymmetric arguments with an assertion on the exact result order."
|
|
97
|
+
},
|
|
98
|
+
"argument_removal" => {
|
|
99
|
+
survives: "the removed argument changes nothing the covering tests observe.",
|
|
100
|
+
kill: "a case where that argument changes the result, asserted exactly."
|
|
101
|
+
},
|
|
102
|
+
"pair_removal" => {
|
|
103
|
+
survives: "tests never read the removed key, directly or through a consumer.",
|
|
104
|
+
kill: "assert the value stored under the removed key, or the whole hash."
|
|
105
|
+
},
|
|
106
|
+
"element_removal" => {
|
|
107
|
+
survives: "tests never exercise the removed element of the array.",
|
|
108
|
+
kill: "an example per element whose behaviour changes when that element is missing."
|
|
109
|
+
},
|
|
110
|
+
"endpoint_replacement" => {
|
|
111
|
+
survives: "fixture collections have one element, or the wrong end is never asserted.",
|
|
112
|
+
kill: "a multi-element collection with an assertion on exactly which end comes back."
|
|
113
|
+
},
|
|
114
|
+
"chain_removal" => {
|
|
115
|
+
survives: "in every tested case the skipped call returns an equivalent value.",
|
|
116
|
+
kill: "a case where the removed call changes the value, asserted exactly."
|
|
117
|
+
},
|
|
118
|
+
"range" => {
|
|
119
|
+
survives: "tests never use the endpoint the exclusive range drops.",
|
|
120
|
+
kill: "an example whose input is the boundary value, asserting it is included."
|
|
121
|
+
},
|
|
122
|
+
"operator_assignment" => {
|
|
123
|
+
survives: "tests check the variable changed, not its exact new value.",
|
|
124
|
+
kill: "assert the exact value after the assignment, where += and -= disagree."
|
|
125
|
+
},
|
|
126
|
+
"condition_negation" => {
|
|
127
|
+
survives: "tests only cover the side of the predicate the negation keeps.",
|
|
128
|
+
kill: "assert both branches with inputs on each side of the predicate."
|
|
129
|
+
},
|
|
130
|
+
"method_swap" => {
|
|
131
|
+
survives: "tests assert loosely, or min and max coincide in every fixture.",
|
|
132
|
+
kill: "a fixture where the two methods differ, asserting the exact expected one."
|
|
133
|
+
},
|
|
134
|
+
"operand_replacement" => {
|
|
135
|
+
survives: "tests assert the result loosely (present, positive, non-nil) or never exercise " \
|
|
136
|
+
"the removed operand.",
|
|
137
|
+
kill: "an assertion on the exact value through the public API, with inputs where the " \
|
|
138
|
+
"removed operand changes the result."
|
|
139
|
+
},
|
|
140
|
+
"interpolation_removal" => {
|
|
141
|
+
survives: "no test asserts the full interpolated content of the string.",
|
|
142
|
+
kill: "an exact string assertion with a known value in the interpolation."
|
|
143
|
+
},
|
|
144
|
+
"jump_value" => {
|
|
145
|
+
survives: "callers ignore the returned value, or tests never take that return path.",
|
|
146
|
+
kill: "an example that takes the early return and asserts the exact value it hands back."
|
|
147
|
+
},
|
|
148
|
+
"rescue_removal" => {
|
|
149
|
+
survives: "tests never raise the error the handler rescues.",
|
|
150
|
+
kill: "an example raising that error, asserting the handler's fallback behaviour."
|
|
151
|
+
},
|
|
152
|
+
"ensure_removal" => {
|
|
153
|
+
survives: "the cleanup's effect is never observed on the tested paths.",
|
|
154
|
+
kill: "assert the cleanup ran (resource released, flag reset), including on the error path."
|
|
155
|
+
},
|
|
156
|
+
"when_removal" => {
|
|
157
|
+
survives: "tests never hit the removed branch of the case.",
|
|
158
|
+
kill: "one example per when branch, asserting its exact outcome."
|
|
159
|
+
},
|
|
160
|
+
"unary_removal" => {
|
|
161
|
+
survives: "fixtures use zero or symmetric values, or tests assert loosely.",
|
|
162
|
+
kill: "an assertion on the exact signed or complemented value with a non-trivial input."
|
|
163
|
+
},
|
|
164
|
+
"regex_replacement" => {
|
|
165
|
+
survives: "every test input matches under both the original and the loosened regex.",
|
|
166
|
+
kill: "an input the anchor or quantifier excludes, asserting it is rejected."
|
|
167
|
+
}
|
|
168
|
+
}.freeze
|
|
169
|
+
|
|
170
|
+
# nil for an unknown name; the command turns that into the exit-2 error.
|
|
171
|
+
def self.for(name)
|
|
172
|
+
return unless OperatorRegistry::DESCRIPTIONS.key?(name)
|
|
173
|
+
|
|
174
|
+
guidance = GUIDANCE.fetch(name)
|
|
175
|
+
Entry.new(name:, definition: OperatorRegistry::DESCRIPTIONS.fetch(name),
|
|
176
|
+
example: OperatorExamples::EXAMPLES.fetch(name),
|
|
177
|
+
survives: guidance.fetch(:survives), kill: guidance.fetch(:kill))
|
|
178
|
+
end
|
|
179
|
+
end
|
|
180
|
+
end
|
|
181
|
+
end
|