saboteur 0.11.1 → 0.12.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +64 -0
- data/README.md +1 -1
- data/lib/saboteur/cli/exit_status.rb +6 -2
- data/lib/saboteur/cli/explain_command.rb +51 -0
- data/lib/saboteur/cli/flaky_exit.rb +23 -0
- data/lib/saboteur/cli/output_options.rb +4 -0
- data/lib/saboteur/cli/run_command.rb +19 -0
- data/lib/saboteur/cli/run_output.rb +10 -0
- data/lib/saboteur/cli/run_reporting.rb +14 -3
- data/lib/saboteur/cli/survivors_command.rb +2 -1
- data/lib/saboteur/cli.rb +4 -1
- data/lib/saboteur/config.rb +25 -3
- data/lib/saboteur/doctor.rb +8 -1
- data/lib/saboteur/engine/coverage_store.rb +54 -3
- data/lib/saboteur/engine/diagnostics.rb +4 -3
- data/lib/saboteur/engine/distribution.rb +6 -2
- data/lib/saboteur/engine/pipelines.rb +14 -2
- data/lib/saboteur/engine/scheduler.rb +1 -1
- data/lib/saboteur/engine/selection_announcement.rb +5 -4
- data/lib/saboteur/engine/subject_selection.rb +10 -0
- data/lib/saboteur/engine.rb +70 -12
- data/lib/saboteur/help.rb +11 -0
- data/lib/saboteur/ignore_list.rb +22 -2
- data/lib/saboteur/mutation_source/builtin.rb +1 -0
- data/lib/saboteur/mutation_source/operator_explain.rb +181 -0
- data/lib/saboteur/reporter/agent_commands.rb +23 -6
- data/lib/saboteur/reporter/agent_incomplete.rb +9 -2
- data/lib/saboteur/reporter/agent_summary.rb +2 -1
- data/lib/saboteur/reporter/default_findings.rb +83 -0
- data/lib/saboteur/reporter/gap_kinds.rb +80 -0
- data/lib/saboteur/reporter/html.rb +19 -4
- data/lib/saboteur/reporter/next_steps.rb +1 -0
- data/lib/saboteur/reporter/review_comments.rb +8 -5
- data/lib/saboteur/reporter/score_table.rb +8 -7
- data/lib/saboteur/reporter/speed_footer.rb +74 -0
- data/lib/saboteur/reporter/survivor_classification.rb +31 -8
- data/lib/saboteur/reporter/survivor_hints.rb +5 -3
- data/lib/saboteur/reporter/survivor_list.rb +78 -7
- data/lib/saboteur/reporter/terminal.rb +52 -36
- data/lib/saboteur/reporter/wording.rb +37 -0
- data/lib/saboteur/result.rb +28 -2
- data/lib/saboteur/survivor_guidance.rb +2 -2
- data/lib/saboteur/text.rb +6 -0
- data/lib/saboteur/version.rb +1 -1
- metadata +8 -1
|
@@ -169,6 +169,11 @@ module Saboteur
|
|
|
169
169
|
return missed.map { |pattern| missed_pattern_message(pattern) }.join("; ")
|
|
170
170
|
end
|
|
171
171
|
|
|
172
|
+
if @config["since"]
|
|
173
|
+
return "--subject #{missed.join(', ')} matches no subject in the --since #{@config['since']} " \
|
|
174
|
+
"diff scope (the scope limits the search to changed files; drop --since or pass --all)"
|
|
175
|
+
end
|
|
176
|
+
|
|
172
177
|
"--subject #{missed.join(', ')} matches no subject (patterns are names like " \
|
|
173
178
|
"Klass, Klass#meth or Klass.meth, not globs)"
|
|
174
179
|
end
|
|
@@ -180,6 +185,11 @@ module Saboteur
|
|
|
180
185
|
macro = attribute_macro_for(pattern)
|
|
181
186
|
return "--subject #{pattern} is defined by #{macro} and has no body to mutate" if macro
|
|
182
187
|
|
|
188
|
+
if @config["since"]
|
|
189
|
+
return "--subject #{pattern} matches no subject in the --since #{@config['since']} diff scope " \
|
|
190
|
+
"(the scope limits the search to changed files)"
|
|
191
|
+
end
|
|
192
|
+
|
|
183
193
|
"fix the --subject pattern \"#{pattern}\": it matches no subject (patterns are names like " \
|
|
184
194
|
"Klass, Klass#meth or Klass.meth, not globs)"
|
|
185
195
|
end
|
data/lib/saboteur/engine.rb
CHANGED
|
@@ -4,6 +4,7 @@
|
|
|
4
4
|
require "fileutils"
|
|
5
5
|
require "json"
|
|
6
6
|
require_relative "replay"
|
|
7
|
+
require_relative "text"
|
|
7
8
|
require_relative "mutation_source/hazard_guard"
|
|
8
9
|
require_relative "baseline/run_evidence"
|
|
9
10
|
require_relative "engine/early_stop_recording"
|
|
@@ -116,6 +117,24 @@ module Saboteur
|
|
|
116
117
|
warnings + map.duplicate_keys(path).map { |key| "duplicate custom map key #{key}: Ruby keeps the last" }
|
|
117
118
|
end
|
|
118
119
|
|
|
120
|
+
# The map this run selected tests from: survivor presentation compares
|
|
121
|
+
# it against the cached whole-suite map (review C1).
|
|
122
|
+
def run_coverage_map
|
|
123
|
+
@coverage_store&.fetched_map
|
|
124
|
+
end
|
|
125
|
+
|
|
126
|
+
# The cached whole-suite map on this tree, for survivor presentation
|
|
127
|
+
# in scoped runs; nil when caching is off or no full build ran (review C1).
|
|
128
|
+
def whole_suite_coverage_map
|
|
129
|
+
@coverage_store&.whole_suite_map
|
|
130
|
+
end
|
|
131
|
+
|
|
132
|
+
# False when caching is off, where no coverage build can ever leave a
|
|
133
|
+
# whole-suite map to consult, so advising one would be wrong (review C1).
|
|
134
|
+
def whole_suite_map_lookup?
|
|
135
|
+
config["cache"] ? true : false
|
|
136
|
+
end
|
|
137
|
+
|
|
119
138
|
private
|
|
120
139
|
|
|
121
140
|
# Inferred test selection, the default since 0.7.0: scope the run to the
|
|
@@ -180,7 +199,8 @@ module Saboteur
|
|
|
180
199
|
matrix: config["matrix"], inline_suppressions: @diagnostics.inline_suppressions,
|
|
181
200
|
operator_policy: config.operator_policy, discovery_failures: @diagnostics.discovery_failures,
|
|
182
201
|
test_command: config["test_command"], reuse_label: reuse_label, since: config["since"],
|
|
183
|
-
shard: @shard, memory: @memory_report, referenced: @guessed_referenced || {}
|
|
202
|
+
shard: @shard, memory: @memory_report, referenced: @guessed_referenced || {},
|
|
203
|
+
phase_timings: @phase_timings)
|
|
184
204
|
end
|
|
185
205
|
|
|
186
206
|
def complete_evidence(results, subjects)
|
|
@@ -207,14 +227,13 @@ module Saboteur
|
|
|
207
227
|
|
|
208
228
|
def mutate(subjects, files, integration, scope, ignore_list, load_lines: {}, load_methods: {})
|
|
209
229
|
store = coverage_store(integration, load_lines, load_methods)
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
check_baseline(integration, full_passes)
|
|
230
|
+
map_reusable = store.reusable?(files)
|
|
231
|
+
full_passes = config["baseline_passes"] + (map_reusable ? 0 : 1)
|
|
232
|
+
baseline_preflight(store, integration, files, full_passes)
|
|
214
233
|
refuse_unaffordable_coverage_build(store, files)
|
|
215
234
|
return [] if abandon_for_budget?(subjects)
|
|
216
235
|
|
|
217
|
-
map = store
|
|
236
|
+
map, map_seconds = timed_map_fetch(store, files)
|
|
218
237
|
inform_about_run(subjects, map)
|
|
219
238
|
cache = result_cache(map, files)
|
|
220
239
|
scheduler = scheduler_for(map, integration, cache)
|
|
@@ -227,13 +246,41 @@ module Saboteur
|
|
|
227
246
|
# mutant when serial. Pack the heap now that the map and candidates
|
|
228
247
|
# are retained, to reduce later copying of shared pages.
|
|
229
248
|
Execution::ForkReadiness.prepare_heap
|
|
230
|
-
results =
|
|
249
|
+
results = run_and_confirm_timed(runnable_subjects, runnable_lists, scheduler, cache, ignore_list,
|
|
250
|
+
map_cached: map_reusable, map_seconds:)
|
|
231
251
|
cache&.save
|
|
232
252
|
results
|
|
233
253
|
rescue BudgetPhaseTimeout => e
|
|
234
254
|
budget_phase_stop(e, subjects)
|
|
235
255
|
end
|
|
236
256
|
|
|
257
|
+
# The map phase's monotonic split, for the speed footer.
|
|
258
|
+
def timed_map_fetch(store, files)
|
|
259
|
+
started = monotonic
|
|
260
|
+
[store.fetch(files), monotonic - started]
|
|
261
|
+
end
|
|
262
|
+
|
|
263
|
+
# Baseline phase: announce, then verify the unchanged tree passes.
|
|
264
|
+
def baseline_preflight(store, integration, files, full_passes)
|
|
265
|
+
warn_unscoped(integration, full_passes)
|
|
266
|
+
announce_quiet_start(store, files)
|
|
267
|
+
check_baseline(integration, full_passes)
|
|
268
|
+
end
|
|
269
|
+
|
|
270
|
+
# The mutant phase plus the run's three disjoint timing intervals
|
|
271
|
+
# (baseline inside check_baseline, map build, mutants) and the worker
|
|
272
|
+
# count, recorded for the terminal's speed footer.
|
|
273
|
+
def run_and_confirm_timed(subjects, lists, scheduler, cache, ignore_list, map_cached:, map_seconds:)
|
|
274
|
+
started = monotonic
|
|
275
|
+
results = run_and_confirm(subjects, lists, scheduler, cache, ignore_list)
|
|
276
|
+
# @baseline_elapsed is a Float from initialize on, so no fallback is needed.
|
|
277
|
+
@phase_timings = PhaseTimings.new(baseline_seconds: @baseline_elapsed,
|
|
278
|
+
map_seconds:, map_cached:,
|
|
279
|
+
mutants_seconds: monotonic - started,
|
|
280
|
+
jobs: config["jobs"])
|
|
281
|
+
results
|
|
282
|
+
end
|
|
283
|
+
|
|
237
284
|
# The candidate run plus the opt-in post-run confirmation pass (#1063):
|
|
238
285
|
# confirmation runs after the last result, before the verdict cache is
|
|
239
286
|
# written, so a confirmed kill overwrites the survivor verdict the run
|
|
@@ -312,16 +359,27 @@ module Saboteur
|
|
|
312
359
|
tests: map.tests.size, cached: @map_cached)
|
|
313
360
|
@diagnostics.quiet_inference(@quiet_spec_count)
|
|
314
361
|
else
|
|
315
|
-
@diagnostics.inform("#{subjects.size
|
|
362
|
+
@diagnostics.inform("#{Text.counted(subjects.size, 'subject')}, " \
|
|
363
|
+
"#{Text.counted(map.tests.size, 'test')} in coverage map")
|
|
316
364
|
end
|
|
317
365
|
ThreadWarnings.warn(@diagnostics, rails: config["rails"])
|
|
318
366
|
end
|
|
319
367
|
|
|
368
|
+
# This run's map is the whole-suite map only when nothing scoped it:
|
|
369
|
+
# no --since, no subjects, and tests either unset or inferred over every
|
|
370
|
+
# included file (never an explicit --test list) (review C1).
|
|
371
|
+
def whole_suite_run?
|
|
372
|
+
config["since"].nil? && config["subjects"].empty? &&
|
|
373
|
+
(config["tests"].empty? || !@guessed_tests.nil?)
|
|
374
|
+
end
|
|
375
|
+
|
|
320
376
|
def coverage_store(integration, load_lines, load_methods = {})
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
|
|
377
|
+
@coverage_store ||=
|
|
378
|
+
CoverageStore.new(config: config, integration: integration, hooks: hooks, load_lines:, load_methods:,
|
|
379
|
+
whole_suite_run: whole_suite_run?,
|
|
380
|
+
supervisor: supervisor(memory_factor: 4), diagnostics: @diagnostics,
|
|
381
|
+
timeout_cap: method(:budget_capped_timeout),
|
|
382
|
+
parts: config["coverage_parts"])
|
|
325
383
|
end
|
|
326
384
|
|
|
327
385
|
def check_baseline(integration, full_passes)
|
data/lib/saboteur/help.rb
CHANGED
|
@@ -152,6 +152,17 @@ module Saboteur
|
|
|
152
152
|
["Diff, replay command and ignore template for one mutant",
|
|
153
153
|
"saboteur survivors .saboteur/report.json --show 1a2b3c4d5e6f7a8b"]]
|
|
154
154
|
),
|
|
155
|
+
Command.new(
|
|
156
|
+
name: "explain",
|
|
157
|
+
summary: "Explain one mutation operator: definition, example and kill guidance",
|
|
158
|
+
description: "Prints what one operator does, a before/after example of the mutation, the test " \
|
|
159
|
+
"shapes that let it survive, and the assertion that kills it. Covers every operator " \
|
|
160
|
+
"the registry can emit.",
|
|
161
|
+
takes_options: false,
|
|
162
|
+
exit_codes: { 0 => "Explanation printed.",
|
|
163
|
+
2 => "Unknown or missing operator name; the valid names are listed." },
|
|
164
|
+
examples: [["Why did this mutant survive?", "saboteur explain operand_replacement"]]
|
|
165
|
+
),
|
|
155
166
|
Command.new(
|
|
156
167
|
name: "badge",
|
|
157
168
|
summary: "Print a shields.io endpoint badge over the opt-in score history",
|
data/lib/saboteur/ignore_list.rb
CHANGED
|
@@ -125,7 +125,7 @@ module Saboteur
|
|
|
125
125
|
# unscoped_digests (subject id => every digest generated before a diff
|
|
126
126
|
# scope cut) keeps a scoped run from calling a scoped-out entry stale
|
|
127
127
|
# (#284).
|
|
128
|
-
def apply(results, unscoped_digests: nil)
|
|
128
|
+
def apply(results, subjects: nil, unscoped_digests: nil, full_run: false)
|
|
129
129
|
seen = {}
|
|
130
130
|
results.each do |result|
|
|
131
131
|
key = [result.subject.id, result.digest]
|
|
@@ -139,7 +139,8 @@ module Saboteur
|
|
|
139
139
|
result.triage_reason = entry.reason
|
|
140
140
|
result.ignored = entry.reason if entry.state != "todo"
|
|
141
141
|
end
|
|
142
|
-
duplicate_warnings + stale_warnings(results, seen, unscoped_digests)
|
|
142
|
+
duplicate_warnings + stale_warnings(results, seen, unscoped_digests) +
|
|
143
|
+
missing_subject_warnings(results, subjects, full_run: full_run)
|
|
143
144
|
end
|
|
144
145
|
|
|
145
146
|
private
|
|
@@ -159,6 +160,25 @@ module Saboteur
|
|
|
159
160
|
.map { |(subject, mutation), _| "duplicate ignore entry #{subject} #{mutation}: the last entry wins" }
|
|
160
161
|
end
|
|
161
162
|
|
|
163
|
+
# A full run examines every in-scope subject, so an entry whose subject
|
|
164
|
+
# never appears points at code that was renamed or removed - say so, but
|
|
165
|
+
# only then: a scoped (--since/--subject) run cannot tell "renamed" from
|
|
166
|
+
# "not in this diff" (#1206). The run's subject list is the source of
|
|
167
|
+
# truth when given: an examined subject with no results (uncovered, or
|
|
168
|
+
# nothing mutable left) is not missing.
|
|
169
|
+
def missing_subject_warnings(results, subjects, full_run:)
|
|
170
|
+
return [] unless full_run
|
|
171
|
+
|
|
172
|
+
examined = (subjects || results).to_set { |item| item.respond_to?(:id) ? item.id : item.subject.id }
|
|
173
|
+
entries.reject { |entry| examined.include?(entry.subject) }
|
|
174
|
+
.map { |entry| missing_subject_message(entry) }
|
|
175
|
+
end
|
|
176
|
+
|
|
177
|
+
def missing_subject_message(entry)
|
|
178
|
+
"stale ignore entry #{entry.subject} #{entry.mutation}: subject not found " \
|
|
179
|
+
"(renamed, removed, or its file is excluded?) - remove the entry or fix the subject name"
|
|
180
|
+
end
|
|
181
|
+
|
|
162
182
|
def stale_message(entry)
|
|
163
183
|
"stale ignore entry #{entry.subject} #{entry.mutation}: no such mutation " \
|
|
164
184
|
"- remove the entry or refresh its digest (saboteur survivors --show)"
|
|
@@ -13,6 +13,7 @@ require_relative "pair_removal"
|
|
|
13
13
|
require_relative "element_removal"
|
|
14
14
|
require_relative "endpoint_replacement"
|
|
15
15
|
require_relative "operator_registry"
|
|
16
|
+
require_relative "operator_explain"
|
|
16
17
|
require_relative "range_replacement"
|
|
17
18
|
require_relative "operator_assignment"
|
|
18
19
|
require_relative "condition_negation"
|
|
@@ -0,0 +1,181 @@
|
|
|
1
|
+
# SPDX-License-Identifier: MIT
|
|
2
|
+
# frozen_string_literal: true
|
|
3
|
+
|
|
4
|
+
require_relative "operator_examples"
|
|
5
|
+
require_relative "operator_registry"
|
|
6
|
+
|
|
7
|
+
module Saboteur
|
|
8
|
+
module MutationSource
|
|
9
|
+
# Curated text behind `saboteur explain NAME`. The definition and the
|
|
10
|
+
# example stay shared with --list-operators (one wording source); the
|
|
11
|
+
# survival and kill hints below are the only text owned here. Every
|
|
12
|
+
# registry operator needs an entry: the coverage spec fails the build
|
|
13
|
+
# when one ships without it.
|
|
14
|
+
module OperatorExplain
|
|
15
|
+
Entry = Data.define(:name, :definition, :example, :survives, :kill)
|
|
16
|
+
|
|
17
|
+
GUIDANCE = {
|
|
18
|
+
"body_nil" => {
|
|
19
|
+
survives: "the method is called for side effects and no test asserts the return value.",
|
|
20
|
+
kill: "an assertion on the exact return value, or on a caller behaviour that depends on it."
|
|
21
|
+
},
|
|
22
|
+
"body_true" => {
|
|
23
|
+
survives: "tests never check the predicate outcome, or only take the path true already forces.",
|
|
24
|
+
kill: "an example whose input makes the predicate false, asserting that path is taken."
|
|
25
|
+
},
|
|
26
|
+
"body_false" => {
|
|
27
|
+
survives: "tests never check the predicate outcome, or only take the path false already forces.",
|
|
28
|
+
kill: "an example whose input makes the predicate true, asserting that path is taken."
|
|
29
|
+
},
|
|
30
|
+
"body_raise" => {
|
|
31
|
+
survives: "tests stub or skip the method, or wrap the call in a broad rescue.",
|
|
32
|
+
kill: "any test that calls the method and lets it complete; the raise fails that example."
|
|
33
|
+
},
|
|
34
|
+
"comparison" => {
|
|
35
|
+
survives: "tests never use values at the boundary the change moves, such as equal values for < vs <=.",
|
|
36
|
+
kill: "a boundary example: the equal case, or the two values just either side of the operator."
|
|
37
|
+
},
|
|
38
|
+
"arithmetic" => {
|
|
39
|
+
survives: "tests assert the result loosely (type, presence, sign) instead of its exact value.",
|
|
40
|
+
kill: "an assertion on the exact computed value with operands where the two operators differ."
|
|
41
|
+
},
|
|
42
|
+
"boolean_logic" => {
|
|
43
|
+
survives: "tests never exercise the dropped operand deciding the outcome on its own.",
|
|
44
|
+
kill: "one example per operand where that operand alone flips the result, asserting each outcome."
|
|
45
|
+
},
|
|
46
|
+
"condition_true" => {
|
|
47
|
+
survives: "tests only exercise the branch the forced condition takes.",
|
|
48
|
+
kill: "an example with a falsy input, asserting the else branch still runs."
|
|
49
|
+
},
|
|
50
|
+
"condition_false" => {
|
|
51
|
+
survives: "tests only exercise the branch the forced condition takes.",
|
|
52
|
+
kill: "an example with a truthy input, asserting the then branch still runs."
|
|
53
|
+
},
|
|
54
|
+
"negation_removal" => {
|
|
55
|
+
survives: "tests assert loosely, or never cover the input the negation inverts.",
|
|
56
|
+
kill: "assert the behaviour for both a truthy and a falsy input through the public API."
|
|
57
|
+
},
|
|
58
|
+
"literal_number" => {
|
|
59
|
+
survives: "the literal flows into nothing a test asserts, or every fixture value works for both.",
|
|
60
|
+
kill: "an assertion on the exact value the literal feeds (a total, a limit, an index)."
|
|
61
|
+
},
|
|
62
|
+
"or_assign" => {
|
|
63
|
+
survives: "tests only cover the nil or false case, where both forms assign.",
|
|
64
|
+
kill: "an example where the variable is already truthy and must keep its value."
|
|
65
|
+
},
|
|
66
|
+
"and_assign" => {
|
|
67
|
+
survives: "tests only cover the truthy case, where both forms assign.",
|
|
68
|
+
kill: "an example where the variable is falsy and must stay untouched."
|
|
69
|
+
},
|
|
70
|
+
"safe_navigation" => {
|
|
71
|
+
survives: "the receiver is never nil in any covering test.",
|
|
72
|
+
kill: "an example with a nil receiver asserting nil comes back instead of NoMethodError."
|
|
73
|
+
},
|
|
74
|
+
"statement_removal" => {
|
|
75
|
+
survives: "the removed statement's effect is never observed: an unasserted side effect or return.",
|
|
76
|
+
kill: "assert the side effect (a spy, a state change) or the value the statement produced."
|
|
77
|
+
},
|
|
78
|
+
"copy_replacement" => {
|
|
79
|
+
survives: "tests never mutate the returned object, so the aliasing goes unnoticed.",
|
|
80
|
+
kill: "mutate what the method returns and assert the original is unchanged (or assert a copy)."
|
|
81
|
+
},
|
|
82
|
+
"filter_replacement" => {
|
|
83
|
+
survives: "tests assert loosely, or the fixture collection is empty or all one kind.",
|
|
84
|
+
kill: "assert the exact surviving elements of a mixed collection, in order."
|
|
85
|
+
},
|
|
86
|
+
"literal_string" => {
|
|
87
|
+
survives: "no test asserts the string's content, only that something was returned.",
|
|
88
|
+
kill: "an exact string equality assertion on the value that carries the literal."
|
|
89
|
+
},
|
|
90
|
+
"literal_boolean" => {
|
|
91
|
+
survives: "the boolean is never asserted, or both values pass the covering assertions.",
|
|
92
|
+
kill: "an example whose expected behaviour flips with the boolean."
|
|
93
|
+
},
|
|
94
|
+
"argument_swap" => {
|
|
95
|
+
survives: "tests use equal or interchangeable arguments, or assert the result loosely.",
|
|
96
|
+
kill: "asymmetric arguments with an assertion on the exact result order."
|
|
97
|
+
},
|
|
98
|
+
"argument_removal" => {
|
|
99
|
+
survives: "the removed argument changes nothing the covering tests observe.",
|
|
100
|
+
kill: "a case where that argument changes the result, asserted exactly."
|
|
101
|
+
},
|
|
102
|
+
"pair_removal" => {
|
|
103
|
+
survives: "tests never read the removed key, directly or through a consumer.",
|
|
104
|
+
kill: "assert the value stored under the removed key, or the whole hash."
|
|
105
|
+
},
|
|
106
|
+
"element_removal" => {
|
|
107
|
+
survives: "tests never exercise the removed element of the array.",
|
|
108
|
+
kill: "an example per element whose behaviour changes when that element is missing."
|
|
109
|
+
},
|
|
110
|
+
"endpoint_replacement" => {
|
|
111
|
+
survives: "fixture collections have one element, or the wrong end is never asserted.",
|
|
112
|
+
kill: "a multi-element collection with an assertion on exactly which end comes back."
|
|
113
|
+
},
|
|
114
|
+
"chain_removal" => {
|
|
115
|
+
survives: "in every tested case the skipped call returns an equivalent value.",
|
|
116
|
+
kill: "a case where the removed call changes the value, asserted exactly."
|
|
117
|
+
},
|
|
118
|
+
"range" => {
|
|
119
|
+
survives: "tests never use the endpoint the exclusive range drops.",
|
|
120
|
+
kill: "an example whose input is the boundary value, asserting it is included."
|
|
121
|
+
},
|
|
122
|
+
"operator_assignment" => {
|
|
123
|
+
survives: "tests check the variable changed, not its exact new value.",
|
|
124
|
+
kill: "assert the exact value after the assignment, where += and -= disagree."
|
|
125
|
+
},
|
|
126
|
+
"condition_negation" => {
|
|
127
|
+
survives: "tests only cover the side of the predicate the negation keeps.",
|
|
128
|
+
kill: "assert both branches with inputs on each side of the predicate."
|
|
129
|
+
},
|
|
130
|
+
"method_swap" => {
|
|
131
|
+
survives: "tests assert loosely, or min and max coincide in every fixture.",
|
|
132
|
+
kill: "a fixture where the two methods differ, asserting the exact expected one."
|
|
133
|
+
},
|
|
134
|
+
"operand_replacement" => {
|
|
135
|
+
survives: "tests assert the result loosely (present, positive, non-nil) or never exercise " \
|
|
136
|
+
"the removed operand.",
|
|
137
|
+
kill: "an assertion on the exact value through the public API, with inputs where the " \
|
|
138
|
+
"removed operand changes the result."
|
|
139
|
+
},
|
|
140
|
+
"interpolation_removal" => {
|
|
141
|
+
survives: "no test asserts the full interpolated content of the string.",
|
|
142
|
+
kill: "an exact string assertion with a known value in the interpolation."
|
|
143
|
+
},
|
|
144
|
+
"jump_value" => {
|
|
145
|
+
survives: "callers ignore the returned value, or tests never take that return path.",
|
|
146
|
+
kill: "an example that takes the early return and asserts the exact value it hands back."
|
|
147
|
+
},
|
|
148
|
+
"rescue_removal" => {
|
|
149
|
+
survives: "tests never raise the error the handler rescues.",
|
|
150
|
+
kill: "an example raising that error, asserting the handler's fallback behaviour."
|
|
151
|
+
},
|
|
152
|
+
"ensure_removal" => {
|
|
153
|
+
survives: "the cleanup's effect is never observed on the tested paths.",
|
|
154
|
+
kill: "assert the cleanup ran (resource released, flag reset), including on the error path."
|
|
155
|
+
},
|
|
156
|
+
"when_removal" => {
|
|
157
|
+
survives: "tests never hit the removed branch of the case.",
|
|
158
|
+
kill: "one example per when branch, asserting its exact outcome."
|
|
159
|
+
},
|
|
160
|
+
"unary_removal" => {
|
|
161
|
+
survives: "fixtures use zero or symmetric values, or tests assert loosely.",
|
|
162
|
+
kill: "an assertion on the exact signed or complemented value with a non-trivial input."
|
|
163
|
+
},
|
|
164
|
+
"regex_replacement" => {
|
|
165
|
+
survives: "every test input matches under both the original and the loosened regex.",
|
|
166
|
+
kill: "an input the anchor or quantifier excludes, asserting it is rejected."
|
|
167
|
+
}
|
|
168
|
+
}.freeze
|
|
169
|
+
|
|
170
|
+
# nil for an unknown name; the command turns that into the exit-2 error.
|
|
171
|
+
def self.for(name)
|
|
172
|
+
return unless OperatorRegistry::DESCRIPTIONS.key?(name)
|
|
173
|
+
|
|
174
|
+
guidance = GUIDANCE.fetch(name)
|
|
175
|
+
Entry.new(name:, definition: OperatorRegistry::DESCRIPTIONS.fetch(name),
|
|
176
|
+
example: OperatorExamples::EXAMPLES.fetch(name),
|
|
177
|
+
survives: guidance.fetch(:survives), kill: guidance.fetch(:kill))
|
|
178
|
+
end
|
|
179
|
+
end
|
|
180
|
+
end
|
|
181
|
+
end
|
|
@@ -45,14 +45,30 @@ module Saboteur
|
|
|
45
45
|
|
|
46
46
|
# Replay one survivor by digest in the same selection. capped: false
|
|
47
47
|
# skips the protocol byte cap, for callers that print to a terminal.
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
48
|
+
# report: narrows the replay to the mutant's covering spec files
|
|
49
|
+
# instead of the run's full inferred list (#1148).
|
|
50
|
+
def replay(provenance, digest, capped: true, report: nil)
|
|
51
|
+
words = ["saboteur", "run", "--id", digest,
|
|
52
|
+
*selection(provenance, replay: true, tests: report && covering_files(report, digest)),
|
|
53
|
+
"--seed", provenance.seed.to_s]
|
|
51
54
|
finish(words, provenance, capped: capped)
|
|
52
55
|
end
|
|
53
56
|
|
|
54
57
|
private
|
|
55
58
|
|
|
59
|
+
# A replay needs only the specs that cover the mutant, not every
|
|
60
|
+
# inferred file (#1148): covered_by holds RSpec ids like
|
|
61
|
+
# "./spec/fee_spec.rb[1:2]", so strip the example suffix. nil when
|
|
62
|
+
# the survivor has no covering ids - then the whole selection
|
|
63
|
+
# stays, since dropping it would widen the replay to inference.
|
|
64
|
+
def covering_files(report, digest)
|
|
65
|
+
result = report.results.find { |entry| entry.digest == digest }
|
|
66
|
+
return unless result
|
|
67
|
+
|
|
68
|
+
files = result.covered_by.filter_map { |id| id[%r{\A\.?/?(.+?\.rb)\[}, 1] }.uniq
|
|
69
|
+
files unless files.empty?
|
|
70
|
+
end
|
|
71
|
+
|
|
56
72
|
def finish(words, provenance, capped: true)
|
|
57
73
|
words += ["--", *provenance.sources] unless provenance.sources.empty?
|
|
58
74
|
command = Shellwords.join(words)
|
|
@@ -93,13 +109,14 @@ module Saboteur
|
|
|
93
109
|
|
|
94
110
|
# Whole-suite scope is a bare --all, only when no tests resolved;
|
|
95
111
|
# --all never combines with --test.
|
|
96
|
-
def selection(provenance, replay: false)
|
|
112
|
+
def selection(provenance, replay: false, tests: nil)
|
|
113
|
+
tests ||= provenance.tests
|
|
97
114
|
words = []
|
|
98
115
|
words.push("--config", provenance.config_path) if provenance.config_path
|
|
99
116
|
words.push("--since", provenance.ref) if provenance.ref
|
|
100
117
|
provenance.subjects.each { |subject| words.push("--subject", subject) }
|
|
101
|
-
words << "--all" if provenance.settings[:all] &&
|
|
102
|
-
words.push(*test_words(
|
|
118
|
+
words << "--all" if provenance.settings[:all] && tests.empty?
|
|
119
|
+
words.push(*test_words(tests))
|
|
103
120
|
words.concat(setting_words(provenance, replay: replay))
|
|
104
121
|
words.concat(mode_words(provenance))
|
|
105
122
|
words.push("--id", provenance.digest) if provenance.digest && !replay
|
|
@@ -40,8 +40,7 @@ module Saboteur
|
|
|
40
40
|
add(reasons, "mutant_cap", omitted_by_cap(report), "mutants omitted by max_mutants_per_method")
|
|
41
41
|
add(reasons, "errors", report.counts[:error], "mutants errored; their verdict did not hold up")
|
|
42
42
|
add(reasons, "resource_limit", report.counts[:resource_limit], "mutants hit the memory or process cap")
|
|
43
|
-
|
|
44
|
-
"mutants the --max-duration budget left unverified; their kills are not confirmed")
|
|
43
|
+
unverified_reasons(reasons, report)
|
|
45
44
|
discovery_failures(reasons, report)
|
|
46
45
|
early_stop_reason(reasons, report)
|
|
47
46
|
inline_scope(reasons, report)
|
|
@@ -70,6 +69,14 @@ module Saboteur
|
|
|
70
69
|
"included untracked files could not be read; their subjects were not covered")
|
|
71
70
|
end
|
|
72
71
|
|
|
72
|
+
def unverified_reasons(reasons, report)
|
|
73
|
+
flaky = report.results.count(&:flaky?)
|
|
74
|
+
add(reasons, "unverified", report.counts[:unverified] - flaky,
|
|
75
|
+
"mutants the --max-duration budget left unverified; their kills are not confirmed")
|
|
76
|
+
add(reasons, "flaky", flaky,
|
|
77
|
+
"mutants gave a different verdict on the --flaky-check reordered rerun; they are unverified")
|
|
78
|
+
end
|
|
79
|
+
|
|
73
80
|
def early_stop_reason(reasons, report)
|
|
74
81
|
stop = report.early_stop
|
|
75
82
|
return unless stop
|
|
@@ -123,7 +123,8 @@ module Saboteur
|
|
|
123
123
|
steps = {}
|
|
124
124
|
add_command(steps, :rerun, Commands.rerun(provenance))
|
|
125
125
|
first = Findings.build(report)[:survivors].first
|
|
126
|
-
|
|
126
|
+
# report: narrows the replay to covering specs, as on the run screen (#1148).
|
|
127
|
+
add_command(steps, :replay, Commands.replay(provenance, first[:digest], report: report)) if first
|
|
127
128
|
steps[:fullReport] = provenance.json_report if short_path?(provenance.json_report)
|
|
128
129
|
steps
|
|
129
130
|
end
|
|
@@ -0,0 +1,83 @@
|
|
|
1
|
+
# SPDX-License-Identifier: MIT
|
|
2
|
+
# frozen_string_literal: true
|
|
3
|
+
|
|
4
|
+
require_relative "mutation_text"
|
|
5
|
+
require_relative "survivor_classification"
|
|
6
|
+
require_relative "survivor_list"
|
|
7
|
+
|
|
8
|
+
module Saboteur
|
|
9
|
+
module Reporter
|
|
10
|
+
# Default whole-project findings (D1): the one-line-per-finding list the
|
|
11
|
+
# --quiet report popularised, then full blocks for the top
|
|
12
|
+
# SurvivorRanking picks only, then a dim closer naming the rest. A
|
|
13
|
+
# --since run or --details keeps the full block list.
|
|
14
|
+
class DefaultFindings
|
|
15
|
+
# Blocks a default run expands; the closer counts what is left out.
|
|
16
|
+
TOP_BLOCKS = 3
|
|
17
|
+
|
|
18
|
+
# Two lines: the fixed text alone is 87 columns at rest; keep it inside 80.
|
|
19
|
+
CLOSER = "... %<more>d more %<findings>s - saboteur survivors .saboteur/report.json,\nor re-run with --details"
|
|
20
|
+
|
|
21
|
+
def initialize(out, palette, all_mutants: false, details: false,
|
|
22
|
+
whole_suite_map: SurvivorList::NO_WHOLE_SUITE_LOOKUP, run_map: nil)
|
|
23
|
+
@out = out
|
|
24
|
+
@palette = palette
|
|
25
|
+
@all_mutants = all_mutants
|
|
26
|
+
@details = details
|
|
27
|
+
@whole_suite_map = whole_suite_map
|
|
28
|
+
@run_map = run_map
|
|
29
|
+
end
|
|
30
|
+
|
|
31
|
+
def call(report, ranked)
|
|
32
|
+
live = ranked.select(&:unignored_survivor?)
|
|
33
|
+
return full(report, ranked) if @details || report.diff? || live.empty?
|
|
34
|
+
|
|
35
|
+
@out.puts(list_lines(report, live))
|
|
36
|
+
@out.puts
|
|
37
|
+
SurvivorList.new(@out, @palette, all_mutants: @all_mutants, limit: TOP_BLOCKS,
|
|
38
|
+
whole_suite_map: @whole_suite_map, run_map: @run_map).call(report, ranked)
|
|
39
|
+
more = finding_count(report, live) - TOP_BLOCKS
|
|
40
|
+
findings = "finding#{'s' unless more == 1}"
|
|
41
|
+
@out.puts @palette.paint(:dim, format(CLOSER, more:, findings:)) if more.positive?
|
|
42
|
+
@out.puts
|
|
43
|
+
end
|
|
44
|
+
|
|
45
|
+
private
|
|
46
|
+
|
|
47
|
+
# --details and --since runs: every finding expanded, exactly the
|
|
48
|
+
# pre-D1 default.
|
|
49
|
+
def full(report, ranked)
|
|
50
|
+
SurvivorList.new(@out, @palette, all_mutants: @all_mutants,
|
|
51
|
+
whole_suite_map: @whole_suite_map, run_map: @run_map).call(report, ranked)
|
|
52
|
+
@out.puts unless report.survivors.empty?
|
|
53
|
+
end
|
|
54
|
+
|
|
55
|
+
# The quiet list's line shape: path:line subject, label and digest,
|
|
56
|
+
# sibling mutants on the statement collapsed to a count (#1126/#1133).
|
|
57
|
+
def list_lines(report, live)
|
|
58
|
+
groups(report, live).map do |(path, line, subject_id), results|
|
|
59
|
+
first, *rest = results
|
|
60
|
+
label = SurvivorClassification.label(first)
|
|
61
|
+
text = "#{path}:#{line} #{subject_id}#{" [#{label}]" if label} - #{first.digest}"
|
|
62
|
+
rest.any? ? "#{text} (+#{rest.size} more)" : text
|
|
63
|
+
end
|
|
64
|
+
end
|
|
65
|
+
|
|
66
|
+
# --all-mutants gives every survivor its own block, so the closer
|
|
67
|
+
# counts mutants rather than statements.
|
|
68
|
+
def finding_count(report, live)
|
|
69
|
+
@all_mutants ? live.size : groups(report, live).size
|
|
70
|
+
end
|
|
71
|
+
|
|
72
|
+
def groups(report, live)
|
|
73
|
+
live.group_by do |result|
|
|
74
|
+
[relative(result.subject.file, report.root), MutationText.line(result), result.subject.id]
|
|
75
|
+
end
|
|
76
|
+
end
|
|
77
|
+
|
|
78
|
+
def relative(file, root)
|
|
79
|
+
file.delete_prefix("#{root}/")
|
|
80
|
+
end
|
|
81
|
+
end
|
|
82
|
+
end
|
|
83
|
+
end
|