saboteur 0.11.0 → 0.11.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +51 -0
- data/README.md +1 -1
- data/lib/saboteur/baseline/command.rb +46 -2
- data/lib/saboteur/baseline/comparison.rb +6 -1
- data/lib/saboteur/baseline/completeness.rb +21 -0
- data/lib/saboteur/baseline/evidence.rb +11 -3
- data/lib/saboteur/baseline/loader.rb +18 -5
- data/lib/saboteur/baseline/manifest.rb +13 -4
- data/lib/saboteur/baseline/run_evidence.rb +3 -1
- data/lib/saboteur/baseline/workflow.rb +57 -9
- data/lib/saboteur/baseline_health.rb +4 -1
- data/lib/saboteur/cli/dirty_tree_notice.rb +7 -1
- data/lib/saboteur/cli/doctor_command.rb +10 -1
- data/lib/saboteur/cli/exit_status.rb +4 -3
- data/lib/saboteur/cli/explicit_sources.rb +1 -1
- data/lib/saboteur/cli/failure_output.rb +2 -2
- data/lib/saboteur/cli/presets.rb +11 -1
- data/lib/saboteur/cli/run_command.rb +4 -1
- data/lib/saboteur/cli/survivors_command.rb +81 -17
- data/lib/saboteur/crash_report.rb +3 -2
- data/lib/saboteur/doctor.rb +3 -1
- data/lib/saboteur/engine/boot.rb +4 -2
- data/lib/saboteur/engine/result_cache.rb +4 -2
- data/lib/saboteur/engine/scheduler.rb +22 -20
- data/lib/saboteur/engine/subject_selection.rb +1 -1
- data/lib/saboteur/engine/survivor_confirmation.rb +1 -1
- data/lib/saboteur/engine/test_guess.rb +1 -1
- data/lib/saboteur/engine/thread_warnings.rb +9 -4
- data/lib/saboteur/engine.rb +22 -4
- data/lib/saboteur/execution/fiddle_loader.rb +29 -2
- data/lib/saboteur/help.rb +2 -1
- data/lib/saboteur/ignore_list.rb +15 -2
- data/lib/saboteur/install.rb +18 -3
- data/lib/saboteur/preview.rb +6 -1
- data/lib/saboteur/reporter/agent_commands.rb +6 -1
- data/lib/saboteur/reporter/baseline_tail.rb +9 -7
- data/lib/saboteur/reporter/html.rb +141 -6
- data/lib/saboteur/reporter/next_steps.rb +9 -7
- data/lib/saboteur/reporter/notice.rb +24 -0
- data/lib/saboteur/reporter/reuse_note.rb +37 -0
- data/lib/saboteur/reporter/review_comments.rb +11 -7
- data/lib/saboteur/reporter/stryker.rb +11 -1
- data/lib/saboteur/reporter/survivor_list.rb +6 -2
- data/lib/saboteur/reporter/survivor_ranking.rb +13 -5
- data/lib/saboteur/reporter/terminal.rb +5 -4
- data/lib/saboteur/result.rb +29 -8
- data/lib/saboteur/selection/git_diff.rb +19 -1
- data/lib/saboteur/survivor_guidance.rb +33 -0
- data/lib/saboteur/version.rb +1 -1
- metadata +5 -2
|
@@ -3,6 +3,9 @@
|
|
|
3
3
|
|
|
4
4
|
require "json"
|
|
5
5
|
|
|
6
|
+
require_relative "../survivor_guidance"
|
|
7
|
+
require_relative "../reporter/survivor_hints"
|
|
8
|
+
|
|
6
9
|
module Saboteur
|
|
7
10
|
class CLI
|
|
8
11
|
# saboteur survivors <report.json> (#475): a read-only, line-oriented
|
|
@@ -11,6 +14,9 @@ module Saboteur
|
|
|
11
14
|
# one digest prints the diff, the exact replay command and the
|
|
12
15
|
# ignore-file template. It never edits the ignore file.
|
|
13
16
|
module SurvivorsCommand
|
|
17
|
+
# Where `saboteur run` writes the JSON report unless configured otherwise.
|
|
18
|
+
DEFAULT_REPORT = Config::DEFAULTS["json_report"]
|
|
19
|
+
|
|
14
20
|
private
|
|
15
21
|
|
|
16
22
|
def survivors_command
|
|
@@ -30,8 +36,7 @@ module Saboteur
|
|
|
30
36
|
end
|
|
31
37
|
options = {}
|
|
32
38
|
args = parser.parse!(@argv, into: options)
|
|
33
|
-
path = args.shift
|
|
34
|
-
"survivors expects a report path: saboteur survivors .saboteur/report.json"
|
|
39
|
+
path = args.shift || DEFAULT_REPORT
|
|
35
40
|
raise UsageError, "unexpected arguments: #{args.join(' ')}" if args.any?
|
|
36
41
|
|
|
37
42
|
report = load_survivor_report(path)
|
|
@@ -45,7 +50,10 @@ module Saboteur
|
|
|
45
50
|
end
|
|
46
51
|
|
|
47
52
|
def load_survivor_report(path)
|
|
48
|
-
|
|
53
|
+
unless File.file?(path)
|
|
54
|
+
hint = path == DEFAULT_REPORT ? "; run saboteur first, or pass a report path" : ""
|
|
55
|
+
raise UsageError, "report not found: #{path}#{hint}"
|
|
56
|
+
end
|
|
49
57
|
|
|
50
58
|
JSON.parse(File.read(path))
|
|
51
59
|
rescue JSON::ParserError => e
|
|
@@ -70,26 +78,77 @@ module Saboteur
|
|
|
70
78
|
end
|
|
71
79
|
|
|
72
80
|
def show(digest)
|
|
73
|
-
survivor =
|
|
74
|
-
raise(UsageError, "no survivor with digest #{digest} in this report")
|
|
81
|
+
survivor = unique_survivor(digest)
|
|
75
82
|
finding = findings_by_digest[survivor[:digest]] || {}
|
|
76
|
-
subject = finding["subject"]
|
|
83
|
+
subject = finding["subject"] || survivor[:subject]
|
|
84
|
+
seed = @report.dig("config", "seed")
|
|
77
85
|
[
|
|
78
|
-
|
|
79
|
-
"",
|
|
80
|
-
"- #{finding['original'] || original_line(survivor)}",
|
|
81
|
-
"+ #{finding['replacement'] || survivor[:replacement]}",
|
|
86
|
+
*detail_lines(survivor, finding),
|
|
82
87
|
"",
|
|
83
|
-
"
|
|
84
|
-
"
|
|
88
|
+
"replay one survivor: #{SurvivorGuidance.replay_command(survivor[:digest], seed: seed)}",
|
|
89
|
+
" #{SurvivorGuidance::REPLAY_CAVEAT}",
|
|
85
90
|
"",
|
|
86
|
-
|
|
87
|
-
|
|
91
|
+
SurvivorGuidance::IGNORE_INTRO,
|
|
92
|
+
" #{ignore_entry(survivor, subject)}"
|
|
88
93
|
].join("\n")
|
|
89
94
|
end
|
|
90
95
|
|
|
91
96
|
private
|
|
92
97
|
|
|
98
|
+
def detail_lines(survivor, finding)
|
|
99
|
+
[
|
|
100
|
+
"#{survivor[:path]}:#{survivor[:line]} #{survivor[:operator]} #{survivor[:digest]}",
|
|
101
|
+
*classification_line(survivor[:classification]),
|
|
102
|
+
"",
|
|
103
|
+
"- #{finding['original'] || original_line(survivor)}",
|
|
104
|
+
"+ #{finding['replacement'] || survivor[:replacement]}",
|
|
105
|
+
"why it matters: #{Reporter::SurvivorHints.why(survivor[:operator])}",
|
|
106
|
+
*covered_line(survivor),
|
|
107
|
+
*hint_line(survivor, survivor[:classification])
|
|
108
|
+
]
|
|
109
|
+
end
|
|
110
|
+
|
|
111
|
+
# The same wording the terminal survivor block uses (#1183).
|
|
112
|
+
def classification_line(label)
|
|
113
|
+
return [] unless label
|
|
114
|
+
|
|
115
|
+
reason = Reporter::SurvivorHints.label_reason(label)
|
|
116
|
+
["classification: #{label}#{" - #{reason}" if reason}"]
|
|
117
|
+
end
|
|
118
|
+
|
|
119
|
+
def covered_line(survivor)
|
|
120
|
+
names = survivor[:covered].map { |id| test_names[id] }.compact
|
|
121
|
+
return [] if names.empty?
|
|
122
|
+
|
|
123
|
+
rest = " (+#{names.size - 1} more)" if names.size >= 2
|
|
124
|
+
["covered by #{names.first}#{rest} - ran, still passed"]
|
|
125
|
+
end
|
|
126
|
+
|
|
127
|
+
def test_names
|
|
128
|
+
@test_names ||= @report.fetch("testFiles", {}).values
|
|
129
|
+
.flat_map { |file| file.fetch("tests", []) }
|
|
130
|
+
.to_h { |test| [test["id"], test["name"]] }
|
|
131
|
+
end
|
|
132
|
+
|
|
133
|
+
def hint_line(survivor, label)
|
|
134
|
+
return [] unless label
|
|
135
|
+
|
|
136
|
+
text = Reporter::SurvivorHints.for_label(survivor[:operator], label: label)
|
|
137
|
+
text ? ["hint: #{text}"] : []
|
|
138
|
+
end
|
|
139
|
+
|
|
140
|
+
# A prefix names one survivor or none: an ambiguous one is refused
|
|
141
|
+
# with the candidates, never resolved to the first match.
|
|
142
|
+
def unique_survivor(digest)
|
|
143
|
+
match, *others = all_survivors.select { |s| s[:digest].start_with?(digest) }
|
|
144
|
+
raise UsageError, "no survivor with digest #{digest} in this report" unless match
|
|
145
|
+
return match if others.empty?
|
|
146
|
+
|
|
147
|
+
digests = [match, *others].map { |m| m[:digest] }.sort
|
|
148
|
+
raise UsageError, "digest prefix #{digest} matches #{digests.size} survivors: " \
|
|
149
|
+
"#{digests.join(', ')}; use more characters"
|
|
150
|
+
end
|
|
151
|
+
|
|
93
152
|
def filtered(file, operator, triage)
|
|
94
153
|
survivors = all_survivors
|
|
95
154
|
survivors = survivors.select { |s| s[:path].include?(file) } if file
|
|
@@ -114,8 +173,13 @@ module Saboteur
|
|
|
114
173
|
end
|
|
115
174
|
|
|
116
175
|
def ignore_entry(survivor, subject)
|
|
117
|
-
|
|
118
|
-
|
|
176
|
+
block = if subject
|
|
177
|
+
SurvivorGuidance.ignore_block(survivor[:digest], subject)
|
|
178
|
+
else
|
|
179
|
+
SurvivorGuidance.ignore_block(survivor[:digest], "Klass#meth",
|
|
180
|
+
todo: "capped findings omit this digest - fill in the subject")
|
|
181
|
+
end
|
|
182
|
+
block.gsub("\n", "\n ")
|
|
119
183
|
end
|
|
120
184
|
|
|
121
185
|
def all_survivors
|
|
@@ -124,7 +188,7 @@ module Saboteur
|
|
|
124
188
|
entry.fetch("mutants", []).select { |m| m["status"] == "Survived" }.map do |m|
|
|
125
189
|
{ path: path, line: m.dig("location", "start", "line").to_i, operator: m["mutatorName"],
|
|
126
190
|
digest: m["id"], replacement: m["replacement"], classification: m["classification"],
|
|
127
|
-
covered: m["coveredBy"] || [], source: entry["source"] }
|
|
191
|
+
subject: m["subject"], covered: m["coveredBy"] || [], source: entry["source"] }
|
|
128
192
|
end
|
|
129
193
|
end
|
|
130
194
|
survivors.sort_by { |s| [s[:path], s[:line], s[:digest]] }
|
|
@@ -38,8 +38,9 @@ module Saboteur
|
|
|
38
38
|
line = "saboteur: #{summary} - please report it: #{ISSUES_URL}"
|
|
39
39
|
written = write_log(log_path, error)
|
|
40
40
|
line = "#{line}; full backtrace: #{display_path(written)}" if written
|
|
41
|
-
|
|
42
|
-
err.puts(
|
|
41
|
+
palette = Reporter::Palette.for(err, mode: color)
|
|
42
|
+
err.puts(palette.paint(:fail, line))
|
|
43
|
+
err.puts(palette.paint(:fail, VERDICT_LINE))
|
|
43
44
|
err.puts(full_text(error)) if verbose
|
|
44
45
|
out.print(agent_json(summary)) if agent
|
|
45
46
|
EXIT_CRASH
|
data/lib/saboteur/doctor.rb
CHANGED
|
@@ -195,7 +195,9 @@ module Saboteur
|
|
|
195
195
|
@out.puts "#{key}: #{file} (found)"
|
|
196
196
|
end
|
|
197
197
|
end
|
|
198
|
-
@config.unmatched_patterns.each
|
|
198
|
+
@config.unmatched_patterns.each do |pattern|
|
|
199
|
+
@out.puts "Warning: #{pattern} matches no file - point include at your code in config/saboteur.rb"
|
|
200
|
+
end
|
|
199
201
|
end
|
|
200
202
|
end
|
|
201
203
|
end
|
data/lib/saboteur/engine/boot.rb
CHANGED
|
@@ -48,7 +48,8 @@ module Saboteur
|
|
|
48
48
|
return if ::SimpleCov.singleton_class.include?(SimpleCovOff)
|
|
49
49
|
|
|
50
50
|
::SimpleCov.singleton_class.prepend(SimpleCovOff)
|
|
51
|
-
|
|
51
|
+
notice = "saboteur: SimpleCov.start ignored for this run (saboteur needs Ruby's Coverage itself)"
|
|
52
|
+
warn Saboteur::Reporter::Palette.for($stderr).paint(:warn, notice)
|
|
52
53
|
end
|
|
53
54
|
|
|
54
55
|
# The Rails profile when rails: true, else nil (set by load_target).
|
|
@@ -74,7 +75,8 @@ module Saboteur
|
|
|
74
75
|
# Running outside the bundle or beside an Appraisal gemfile set
|
|
75
76
|
# loads the wrong gems; say so before the spec load fails in
|
|
76
77
|
# confusing ways (#598, #609). Advisory only.
|
|
77
|
-
|
|
78
|
+
hints = Saboteur::Reporter::Palette.for($stderr)
|
|
79
|
+
EnvironmentHints.all(@root).each { |hint| warn hints.paint(:warn, "saboteur: warning: #{hint}") }
|
|
78
80
|
integration.setup
|
|
79
81
|
rails&.after_boot
|
|
80
82
|
@hooks.run(:after_boot)
|
|
@@ -16,7 +16,8 @@ module Saboteur
|
|
|
16
16
|
# same: the method's source, the whole file the method lives in (so a
|
|
17
17
|
# constant or helper changed elsewhere in that file is seen), the mutant,
|
|
18
18
|
# the set of covering tests, the content of those tests' files, the
|
|
19
|
-
# lockfile, the run's inputs (config, saboteur version
|
|
19
|
+
# lockfile, the run's inputs (config, the saboteur version, the Ruby
|
|
20
|
+
# runtime's engine, version, patchlevel and platform), and every source
|
|
20
21
|
# file a covering test touched (a method it calls, a constant defined
|
|
21
22
|
# elsewhere - #197). Only killed, survived and timeout are stored;
|
|
22
23
|
# errors and skips always run again.
|
|
@@ -162,7 +163,8 @@ module Saboteur
|
|
|
162
163
|
|
|
163
164
|
def fingerprint(subject, tests)
|
|
164
165
|
digest = Digest::SHA256.new
|
|
165
|
-
digest << [VERSION,
|
|
166
|
+
digest << [VERSION, RUBY_ENGINE, RUBY_VERSION, RUBY_PATCHLEVEL, RUBY_PLATFORM, @inputs, @suite_digest,
|
|
167
|
+
source_set, subject.source].join("\0") << "\0"
|
|
166
168
|
# A hex digest has a fixed width, so it needs no separator after it.
|
|
167
169
|
digest << Digest::SHA256.hexdigest(subject.file_source)
|
|
168
170
|
digest_tests(digest, tests)
|
|
@@ -77,6 +77,28 @@ module Saboteur
|
|
|
77
77
|
deadline(recorded * slowdown)
|
|
78
78
|
end
|
|
79
79
|
|
|
80
|
+
# Survivor confirmation (#1063) drives the runner and the coverage map
|
|
81
|
+
# from outside after the run.
|
|
82
|
+
attr_reader :runner, :map
|
|
83
|
+
|
|
84
|
+
# The eval prefix a post-run confirmation run needs for this subject
|
|
85
|
+
# (#1063): the same wrapper detection load_problem applies, reduced to
|
|
86
|
+
# the sorbet sig prefix a survivor could have run with.
|
|
87
|
+
def confirmation_prefix(subject)
|
|
88
|
+
return nil unless @verify_loaded
|
|
89
|
+
return nil if @reevaluator.whole_file?(subject)
|
|
90
|
+
|
|
91
|
+
owner = @reevaluator.resolve_owner(subject)
|
|
92
|
+
target = subject.singleton ? owner.singleton_class : owner
|
|
93
|
+
method = own_method(target, subject.name)
|
|
94
|
+
return nil unless method
|
|
95
|
+
|
|
96
|
+
file, = method.source_location
|
|
97
|
+
Execution::Wrapper.around(method, file) == :sorbet ? subject.sig_source : nil
|
|
98
|
+
rescue Error
|
|
99
|
+
nil
|
|
100
|
+
end
|
|
101
|
+
|
|
80
102
|
private
|
|
81
103
|
|
|
82
104
|
# Verdicts that did not hold up are warned about once per subject;
|
|
@@ -232,26 +254,6 @@ module Saboteur
|
|
|
232
254
|
result
|
|
233
255
|
end
|
|
234
256
|
|
|
235
|
-
attr_reader :runner, :map
|
|
236
|
-
|
|
237
|
-
# The eval prefix a post-run confirmation run needs for this subject
|
|
238
|
-
# (#1063): the same wrapper detection load_problem applies, reduced to
|
|
239
|
-
# the sorbet sig prefix a survivor could have run with.
|
|
240
|
-
def confirmation_prefix(subject)
|
|
241
|
-
return nil unless @verify_loaded
|
|
242
|
-
return nil if @reevaluator.whole_file?(subject)
|
|
243
|
-
|
|
244
|
-
owner = @reevaluator.resolve_owner(subject)
|
|
245
|
-
target = subject.singleton ? owner.singleton_class : owner
|
|
246
|
-
method = own_method(target, subject.name)
|
|
247
|
-
return nil unless method
|
|
248
|
-
|
|
249
|
-
file, = method.source_location
|
|
250
|
-
Execution::Wrapper.around(method, file) == :sorbet ? subject.sig_source : nil
|
|
251
|
-
rescue Error
|
|
252
|
-
nil
|
|
253
|
-
end
|
|
254
|
-
|
|
255
257
|
# The tests that ran the edited code (a coarse edit spans the whole
|
|
256
258
|
# body, so it gets every test of the method), prior killers first, then
|
|
257
259
|
# fastest-first.
|
|
@@ -82,7 +82,7 @@ module Saboteur
|
|
|
82
82
|
"the resolved range has no changes"
|
|
83
83
|
elsif !@raw_changed_files.intersect?(@files)
|
|
84
84
|
"#{count} changed file#{'s' unless count == 1}, none in mutation scope " \
|
|
85
|
-
"(docs
|
|
85
|
+
"(specs, docs, excluded, or non-Ruby files)"
|
|
86
86
|
else
|
|
87
87
|
"no subject covers the changed lines (layout-only edits, deletions, " \
|
|
88
88
|
"or intentionally untested code)"
|
|
@@ -44,7 +44,7 @@ module Saboteur
|
|
|
44
44
|
return unconfirmed(result, "the --max-duration budget is spent") if @budget_exceeded.call
|
|
45
45
|
|
|
46
46
|
tests = unrun_covering_tests(result)
|
|
47
|
-
return result if tests.empty?
|
|
47
|
+
return unconfirmed(result, "no covering tests") if tests.empty?
|
|
48
48
|
|
|
49
49
|
outcome = run_confirmation(result, tests)
|
|
50
50
|
@remaining -= outcome.runtime || 0
|
|
@@ -81,7 +81,7 @@ module Saboteur
|
|
|
81
81
|
# a file whose own entry broke heard that in the warning (#640).
|
|
82
82
|
mapped = without_map_entry.any? &&
|
|
83
83
|
" If the layout defeats the naming convention, map #{names(without_map_entry)} by hand in " \
|
|
84
|
-
"config/saboteur_custom_map.rb
|
|
84
|
+
"config/saboteur_custom_map.rb (no entry covers it yet)."
|
|
85
85
|
"no spec found for #{pairs.join(', ')}, so nothing ran.#{changed} Re-run the same command with " \
|
|
86
86
|
"one --test per existing spec path, or append --all to use the whole suite without " \
|
|
87
87
|
"changing the source scope.#{mapped}"
|
|
@@ -17,10 +17,12 @@ module Saboteur
|
|
|
17
17
|
# fork_mode, whose default is :both, i.e. attach (RUBY_DEBUG_FORK_MODE;
|
|
18
18
|
# ruby/debug's config.rb and SESSION.activate on_fork: true in
|
|
19
19
|
# session.rb).
|
|
20
|
+
REAPER = "AR Pool Reaper"
|
|
21
|
+
|
|
20
22
|
KNOWN_FORK_THREADS = {
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
|
|
23
|
+
REAPER => "ActiveRecord's connection-pool reaper: expected harmless when children " \
|
|
24
|
+
"reconnect the database after fork (rails mode does) - it only maintains " \
|
|
25
|
+
"the parent's pool",
|
|
24
26
|
"DEBUGGER__::SESSION@server" => "the debug gem's remote-console server: what it does in " \
|
|
25
27
|
"children depends on RUBY_DEBUG_FORK_MODE - with parent no " \
|
|
26
28
|
"debugger runs in children; with child or both (the gem's " \
|
|
@@ -30,8 +32,11 @@ module Saboteur
|
|
|
30
32
|
"RUBY_DEBUG_OPEN or require \"debug\" without :open"
|
|
31
33
|
}.freeze
|
|
32
34
|
|
|
33
|
-
|
|
35
|
+
# In rails mode the children reconnect the database, so the pool
|
|
36
|
+
# reaper is not worth a warning on every run (#1229).
|
|
37
|
+
def self.warn(diagnostics, rails: false)
|
|
34
38
|
others = Thread.list - [Thread.current]
|
|
39
|
+
others = others.reject { |thread| thread.name == REAPER } if rails
|
|
35
40
|
return if others.empty?
|
|
36
41
|
|
|
37
42
|
unknown = false
|
data/lib/saboteur/engine.rb
CHANGED
|
@@ -30,6 +30,11 @@ module Saboteur
|
|
|
30
30
|
include GatedRun
|
|
31
31
|
include SelectionAnnouncement
|
|
32
32
|
|
|
33
|
+
# Confirmation only guards a partial coverage map, so it is silent unless
|
|
34
|
+
# the user opted into partial_coverage = use (#1228).
|
|
35
|
+
CONFIRMATION_SKIPPED = "survivor confirmation skipped: partial_coverage = use applies only to a cached " \
|
|
36
|
+
"whole-suite run (no --test, one coverage part)"
|
|
37
|
+
|
|
33
38
|
# Everything besides sources and tests that can change a verdict.
|
|
34
39
|
VERDICT_INPUTS = %w[integration require rspec_args rails reeval sources operators exclude_operators arid
|
|
35
40
|
timeout_factor timeout_min
|
|
@@ -51,7 +56,7 @@ module Saboteur
|
|
|
51
56
|
@replay = Replay.new(digest: digest) if digest
|
|
52
57
|
if capture_evidence && (digest || config["since"] || config["incremental"] || config["resume"] || config["shard"])
|
|
53
58
|
raise ConfigError,
|
|
54
|
-
"baseline evidence requires fresh whole-scope execution: no since, replay, shard or incremental"
|
|
59
|
+
"baseline evidence requires fresh whole-scope execution: no since, replay, shard, resume or incremental"
|
|
55
60
|
end
|
|
56
61
|
|
|
57
62
|
@capture_evidence = capture_evidence
|
|
@@ -174,7 +179,7 @@ module Saboteur
|
|
|
174
179
|
hazard_suppressions: @diagnostics.hazard_suppressions, early_stop: @early_stop,
|
|
175
180
|
matrix: config["matrix"], inline_suppressions: @diagnostics.inline_suppressions,
|
|
176
181
|
operator_policy: config.operator_policy, discovery_failures: @diagnostics.discovery_failures,
|
|
177
|
-
test_command: config["test_command"],
|
|
182
|
+
test_command: config["test_command"], reuse_label: reuse_label, since: config["since"],
|
|
178
183
|
shard: @shard, memory: @memory_report, referenced: @guessed_referenced || {})
|
|
179
184
|
end
|
|
180
185
|
|
|
@@ -240,7 +245,11 @@ module Saboteur
|
|
|
240
245
|
end
|
|
241
246
|
|
|
242
247
|
def confirm_survivors(results, scheduler, cache, ignore_list)
|
|
243
|
-
|
|
248
|
+
unless CoveragePartial.use?(config, config["coverage_parts"])
|
|
249
|
+
live = results.any? { |result| result.survived? && !ignore_list.accepts?(result) }
|
|
250
|
+
@diagnostics.inform(CONFIRMATION_SKIPPED) if live && config["partial_coverage"] == "use"
|
|
251
|
+
return results
|
|
252
|
+
end
|
|
244
253
|
|
|
245
254
|
SurvivorConfirmation.new(config: config, map: scheduler.map, scheduler: scheduler, diagnostics: @diagnostics,
|
|
246
255
|
budget_exceeded: method(:budget_exceeded?), cache: cache)
|
|
@@ -305,7 +314,7 @@ module Saboteur
|
|
|
305
314
|
else
|
|
306
315
|
@diagnostics.inform("#{subjects.size} subjects, #{map.tests.size} tests in coverage map")
|
|
307
316
|
end
|
|
308
|
-
ThreadWarnings.warn(@diagnostics)
|
|
317
|
+
ThreadWarnings.warn(@diagnostics, rails: config["rails"])
|
|
309
318
|
end
|
|
310
319
|
|
|
311
320
|
def coverage_store(integration, load_lines, load_methods = {})
|
|
@@ -368,6 +377,15 @@ module Saboteur
|
|
|
368
377
|
coverage: map, fallback_files: source_files(files))
|
|
369
378
|
end
|
|
370
379
|
|
|
380
|
+
# Which reuse mechanism is on, named as the flag the user typed, so the
|
|
381
|
+
# terminal's reuse note never cites a flag that was not passed. A replay
|
|
382
|
+
# runs one mutant fresh by design; it does not reuse, so it gets no label.
|
|
383
|
+
def reuse_label
|
|
384
|
+
return nil if @replay
|
|
385
|
+
|
|
386
|
+
config["incremental"] ? "--incremental" : ("--resume" if config["resume"])
|
|
387
|
+
end
|
|
388
|
+
|
|
371
389
|
def verdict_inputs
|
|
372
390
|
JSON.generate(VERDICT_INPUTS.to_h { |key| [key, config[key]] }.merge("operator_policy" => config.operator_policy))
|
|
373
391
|
end
|
|
@@ -24,11 +24,38 @@ module Saboteur
|
|
|
24
24
|
# free. A LoadError means the sandbox layers cannot work: REFUSE -
|
|
25
25
|
# the error propagates, it is never downgraded to a warning.
|
|
26
26
|
def self.require_fiddle!(with_import: false)
|
|
27
|
-
|
|
28
|
-
|
|
27
|
+
quietly do
|
|
28
|
+
require "fiddle"
|
|
29
|
+
require "fiddle/import" if with_import
|
|
30
|
+
end
|
|
29
31
|
rescue LoadError
|
|
30
32
|
raise Error, MESSAGE
|
|
31
33
|
end
|
|
34
|
+
|
|
35
|
+
# Ruby 3.4 prints "fiddle was loaded from the standard library, but
|
|
36
|
+
# will no longer be part of the default gems" on this require when the
|
|
37
|
+
# Gemfile does not name it. The notice is about the user's Gemfile, not
|
|
38
|
+
# a saboteur fault, and Ruby 4.0 refuses the load outright (MESSAGE
|
|
39
|
+
# says why), so only that one notice is dropped, only during this
|
|
40
|
+
# require (#1226).
|
|
41
|
+
NOTICE = "fiddle was loaded from the standard library"
|
|
42
|
+
|
|
43
|
+
module NoticeFilter
|
|
44
|
+
def warn(message, **)
|
|
45
|
+
return if Thread.current[:saboteur_fiddle_require] && message.include?(NOTICE)
|
|
46
|
+
|
|
47
|
+
super
|
|
48
|
+
end
|
|
49
|
+
end
|
|
50
|
+
Warning.singleton_class.prepend(NoticeFilter)
|
|
51
|
+
|
|
52
|
+
def self.quietly
|
|
53
|
+
previous = Thread.current[:saboteur_fiddle_require]
|
|
54
|
+
Thread.current[:saboteur_fiddle_require] = true
|
|
55
|
+
yield
|
|
56
|
+
ensure
|
|
57
|
+
Thread.current[:saboteur_fiddle_require] = previous
|
|
58
|
+
end
|
|
32
59
|
end
|
|
33
60
|
end
|
|
34
61
|
end
|
data/lib/saboteur/help.rb
CHANGED
|
@@ -121,7 +121,8 @@ module Saboteur
|
|
|
121
121
|
"accept verifies current inputs and publishes it. No scores or execution are suppressed. " \
|
|
122
122
|
"With --epsilon POINTS, check also fails when the score drops more than POINTS percentage " \
|
|
123
123
|
"points below the accepted baseline's score. " \
|
|
124
|
-
"Only whole or subject scope; no diff, replay, incremental reuse or
|
|
124
|
+
"Only whole or subject scope; no diff, replay, shard, resume, incremental reuse or " \
|
|
125
|
+
"score threshold.",
|
|
125
126
|
takes_options: true,
|
|
126
127
|
exit_codes: { 0 => "Clean check, proposal written or accepted.",
|
|
127
128
|
1 => "New, changed or unmatched survivors, or a score drop beyond --epsilon.",
|
data/lib/saboteur/ignore_list.rb
CHANGED
|
@@ -139,7 +139,7 @@ module Saboteur
|
|
|
139
139
|
result.triage_reason = entry.reason
|
|
140
140
|
result.ignored = entry.reason if entry.state != "todo"
|
|
141
141
|
end
|
|
142
|
-
stale_warnings(results, seen, unscoped_digests)
|
|
142
|
+
duplicate_warnings + stale_warnings(results, seen, unscoped_digests)
|
|
143
143
|
end
|
|
144
144
|
|
|
145
145
|
private
|
|
@@ -148,7 +148,20 @@ module Saboteur
|
|
|
148
148
|
examined = results.to_set { |result| result.subject.id }
|
|
149
149
|
entries.select { |entry| examined.include?(entry.subject) && !seen[[entry.subject, entry.mutation]] }
|
|
150
150
|
.reject { |entry| unscoped_digests&.fetch(entry.subject, nil)&.include?(entry.mutation) }
|
|
151
|
-
.map { |entry|
|
|
151
|
+
.map { |entry| stale_message(entry) }
|
|
152
|
+
end
|
|
153
|
+
|
|
154
|
+
# Entries naming the same (subject, mutation) more than once: the index
|
|
155
|
+
# keeps the last one, which is invisible otherwise (#1207).
|
|
156
|
+
def duplicate_warnings
|
|
157
|
+
entries.group_by { |entry| [entry.subject, entry.mutation] }
|
|
158
|
+
.reject { |_, group| group.one? }
|
|
159
|
+
.map { |(subject, mutation), _| "duplicate ignore entry #{subject} #{mutation}: the last entry wins" }
|
|
160
|
+
end
|
|
161
|
+
|
|
162
|
+
def stale_message(entry)
|
|
163
|
+
"stale ignore entry #{entry.subject} #{entry.mutation}: no such mutation " \
|
|
164
|
+
"- remove the entry or refresh its digest (saboteur survivors --show)"
|
|
152
165
|
end
|
|
153
166
|
end
|
|
154
167
|
end
|
data/lib/saboteur/install.rb
CHANGED
|
@@ -32,6 +32,9 @@ module Saboteur
|
|
|
32
32
|
end
|
|
33
33
|
RUBY
|
|
34
34
|
|
|
35
|
+
BASE_CLASSES = %w[app/models/application_record.rb app/controllers/application_controller.rb
|
|
36
|
+
app/jobs/application_job.rb app/mailers/application_mailer.rb].freeze
|
|
37
|
+
|
|
35
38
|
BIN = <<~RUBY
|
|
36
39
|
#!/usr/bin/env ruby
|
|
37
40
|
# frozen_string_literal: true
|
|
@@ -64,19 +67,31 @@ module Saboteur
|
|
|
64
67
|
|
|
65
68
|
# Returns the list of Results (:created, :overwritten, :skipped, :appended, :unchanged).
|
|
66
69
|
def call
|
|
67
|
-
[write("config/saboteur.rb",
|
|
70
|
+
[write("config/saboteur.rb", config_text), write("bin/saboteur", BIN, mode: 0o755), gitignore]
|
|
68
71
|
end
|
|
69
72
|
|
|
70
|
-
|
|
73
|
+
RUN_HINT = "Next: bin/saboteur doctor, then bin/saboteur run --since origin/main " \
|
|
74
|
+
"(keep SimpleCov off for the run)."
|
|
75
|
+
|
|
76
|
+
# Printed after install; the Gemfile is never edited. The run pointer is
|
|
77
|
+
# always due: the Gemfile block only when the gem is missing.
|
|
71
78
|
def gemfile_hint
|
|
72
79
|
gemfile = File.join(@root, "Gemfile")
|
|
73
|
-
return
|
|
80
|
+
return RUN_HINT if File.file?(gemfile) && Text.utf8(File.read(gemfile)).match?(/^\s*gem\s+["']saboteur["']/)
|
|
74
81
|
|
|
75
82
|
GEMFILE_HINT
|
|
76
83
|
end
|
|
77
84
|
|
|
78
85
|
private
|
|
79
86
|
|
|
87
|
+
# CONFIG without the base-class excludes this app does not have: a
|
|
88
|
+
# minimal Rails app has no ApplicationJob or ApplicationMailer, and an
|
|
89
|
+
# exclude that matches no file makes doctor and every run warn.
|
|
90
|
+
def config_text
|
|
91
|
+
BASE_CLASSES.reject { |path| File.exist?(File.join(@root, path)) }
|
|
92
|
+
.reduce(CONFIG) { |text, path| text.sub(%( "#{path}",\n), "") }
|
|
93
|
+
end
|
|
94
|
+
|
|
80
95
|
# Never writes or chmods through a symlink: not the file itself, and not a
|
|
81
96
|
# symlinked directory above it (bin -> elsewhere). Without force the file is
|
|
82
97
|
# skipped; with force the link is removed and replaced by a real file or
|
data/lib/saboteur/preview.rb
CHANGED
|
@@ -59,8 +59,13 @@ module Saboteur
|
|
|
59
59
|
@diagnostics.warnings.each { |warning| @out.puts("warning: #{warning}") }
|
|
60
60
|
end
|
|
61
61
|
|
|
62
|
+
# The default set is what --list-operators prints, so a preview leads
|
|
63
|
+
# with it only when the selection was changed or --verbose asks (#1178).
|
|
62
64
|
def announce_operators
|
|
63
|
-
|
|
65
|
+
return if @json
|
|
66
|
+
return unless @config["verbose"] || @config["operators"].any? || @config["exclude_operators"].any?
|
|
67
|
+
|
|
68
|
+
@out.puts("Operators: #{@config.operator_policy['effective'].join(', ')}")
|
|
64
69
|
end
|
|
65
70
|
|
|
66
71
|
# Candidate filtering precedes diff restriction and the per-method cap.
|
|
@@ -22,6 +22,11 @@ module Saboteur
|
|
|
22
22
|
baseline_passes: "--baseline-passes", max_duration: "--max-duration",
|
|
23
23
|
shard: "--shard" }.freeze
|
|
24
24
|
|
|
25
|
+
# Not part of the selection a replay reproduces: a shard is a slice of
|
|
26
|
+
# it, and a report path would overwrite the report the replay came
|
|
27
|
+
# from (the audit preset names whole-project reports, #1185).
|
|
28
|
+
REPLAY_SKIPPED = %i[shard json_report html_report].freeze
|
|
29
|
+
|
|
25
30
|
# CLI-only boolean switches recorded in settings, mapped by value, so
|
|
26
31
|
# an explicit false (--no-max-duration-ok over a config true) survives
|
|
27
32
|
# into the generated commands instead of reverting to the config.
|
|
@@ -76,7 +81,7 @@ module Saboteur
|
|
|
76
81
|
# (#317). A typed empty list stays: it may clear a config value.
|
|
77
82
|
def setting_words(provenance, replay: false)
|
|
78
83
|
SETTING_FLAGS.flat_map do |key, flag|
|
|
79
|
-
next [] if replay && key
|
|
84
|
+
next [] if replay && REPLAY_SKIPPED.include?(key)
|
|
80
85
|
next [] unless provenance.settings.key?(key)
|
|
81
86
|
|
|
82
87
|
value = provenance.settings[key]
|
|
@@ -24,12 +24,14 @@ module Saboteur
|
|
|
24
24
|
new_count = gate.new_survivor_ids.size
|
|
25
25
|
return pass_line(gate) if new_count.zero?
|
|
26
26
|
|
|
27
|
-
|
|
28
|
-
|
|
27
|
+
matched = @palette.paint(:dim, "#{matched_count(gate)} matched the baseline#{breakdown(gate)}")
|
|
28
|
+
"saboteur: #{matched} #{@palette.glyph(:middot)} " \
|
|
29
|
+
"#{@palette.paint(:fail, counted(new_count, 'new survivor'))}"
|
|
29
30
|
end
|
|
30
31
|
|
|
31
32
|
def pass_line(gate)
|
|
32
|
-
|
|
33
|
+
matched = @palette.paint(:dim, "#{matched_count(gate)} matched the baseline")
|
|
34
|
+
line = "saboteur: #{matched}, #{@palette.paint(:pass, 'no new survivors')}"
|
|
33
35
|
current, baseline = score_pair(gate)
|
|
34
36
|
line << " - score #{current} (baseline #{baseline})" if current
|
|
35
37
|
line
|
|
@@ -80,13 +82,13 @@ module Saboteur
|
|
|
80
82
|
end
|
|
81
83
|
|
|
82
84
|
def pass_hint
|
|
83
|
-
|
|
84
|
-
|
|
85
|
+
hint = "re-baseline deliberately #{@palette.glyph(:middot)} run without --baseline weekly to catch drift"
|
|
86
|
+
@palette.paint(:info, hint)
|
|
85
87
|
end
|
|
86
88
|
|
|
87
89
|
def fail_hint
|
|
88
|
-
"
|
|
89
|
-
|
|
90
|
+
hint = "re-baseline deliberately: saboteur baseline accept #{@palette.glyph(:middot)} run full weekly"
|
|
91
|
+
" #{@palette.paint(:info, hint)}"
|
|
90
92
|
end
|
|
91
93
|
end
|
|
92
94
|
end
|