saboteur 0.12.1 → 0.12.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +86 -0
- data/README.md +18 -3
- data/lib/saboteur/baseline/command.rb +6 -1
- data/lib/saboteur/baseline/comparison.rb +8 -0
- data/lib/saboteur/baseline/completeness.rb +14 -4
- data/lib/saboteur/baseline/evidence.rb +8 -1
- data/lib/saboteur/baseline/run_evidence.rb +10 -1
- data/lib/saboteur/baseline/workflow.rb +17 -5
- data/lib/saboteur/cli/color_option.rb +26 -0
- data/lib/saboteur/cli/doctor_command.rb +1 -0
- data/lib/saboteur/cli/exit_status.rb +4 -3
- data/lib/saboteur/cli/explain_command.rb +2 -1
- data/lib/saboteur/cli/failure_output.rb +13 -2
- data/lib/saboteur/cli/help_command.rb +5 -0
- data/lib/saboteur/cli/history_command.rb +2 -0
- data/lib/saboteur/cli/output_options.rb +1 -5
- data/lib/saboteur/cli/run_command.rb +1 -1
- data/lib/saboteur/cli/run_reporting.rb +1 -9
- data/lib/saboteur/cli/run_validation.rb +0 -1
- data/lib/saboteur/cli/survivors_command.rb +10 -6
- data/lib/saboteur/cli/whole_suite_map_lookup.rb +33 -0
- data/lib/saboteur/cli.rb +6 -2
- data/lib/saboteur/engine/coverage_store.rb +7 -4
- data/lib/saboteur/engine/diagnostics.rb +7 -0
- data/lib/saboteur/engine/pipelines.rb +9 -2
- data/lib/saboteur/engine/result_cache.rb +8 -5
- data/lib/saboteur/engine/scheduler.rb +3 -2
- data/lib/saboteur/engine/survivor_confirmation.rb +75 -11
- data/lib/saboteur/engine/test_guess.rb +2 -1
- data/lib/saboteur/engine.rb +34 -16
- data/lib/saboteur/execution/runner.rb +14 -4
- data/lib/saboteur/execution/site_probe.rb +161 -0
- data/lib/saboteur/gates.rb +1 -1
- data/lib/saboteur/integration/rspec.rb +30 -0
- data/lib/saboteur/reporter/default_findings.rb +1 -1
- data/lib/saboteur/reporter/findings.rb +2 -1
- data/lib/saboteur/reporter/gap_kinds.rb +18 -0
- data/lib/saboteur/reporter/next_steps.rb +3 -1
- data/lib/saboteur/reporter/review_comments.rb +4 -1
- data/lib/saboteur/reporter/score_table.rb +6 -11
- data/lib/saboteur/reporter/stryker.rb +12 -2
- data/lib/saboteur/reporter/survivor_classification.rb +53 -6
- data/lib/saboteur/reporter/survivor_hints.rb +13 -9
- data/lib/saboteur/reporter/survivor_list.rb +6 -6
- data/lib/saboteur/reporter/terminal.rb +4 -2
- data/lib/saboteur/reporter/unconfirmed_section.rb +44 -0
- data/lib/saboteur/result.rb +46 -4
- data/lib/saboteur/selection/coverage_map.rb +11 -1
- data/lib/saboteur/version.rb +1 -1
- data/lib/saboteur.rb +5 -0
- metadata +5 -1
|
@@ -57,7 +57,7 @@ module Saboteur
|
|
|
57
57
|
def list_lines(report, live)
|
|
58
58
|
groups(report, live).map do |(path, line, subject_id), results|
|
|
59
59
|
first, *rest = results
|
|
60
|
-
label = SurvivorClassification.
|
|
60
|
+
label = SurvivorClassification.group_label(results)
|
|
61
61
|
text = "#{path}:#{line} #{subject_id}#{" [#{label}]" if label} - #{first.digest}"
|
|
62
62
|
rest.any? ? "#{text} (+#{rest.size} more)" : text
|
|
63
63
|
end
|
|
@@ -71,7 +71,8 @@ module Saboteur
|
|
|
71
71
|
edits: entries,
|
|
72
72
|
coarse: result.mutation.coarse,
|
|
73
73
|
# Why the survivor is worth a look, as a label only (#192).
|
|
74
|
-
classification: SurvivorClassification.label(result)
|
|
74
|
+
classification: SurvivorClassification.label(result),
|
|
75
|
+
gapKind: SurvivorClassification.kind(result)
|
|
75
76
|
}.compact
|
|
76
77
|
entry = with_changed_lines(report, result, entry)
|
|
77
78
|
result.ignored ? entry.merge(reason: result.ignored) : entry
|
|
@@ -58,6 +58,12 @@ module Saboteur
|
|
|
58
58
|
|
|
59
59
|
FALLBACK = "result unchecked"
|
|
60
60
|
|
|
61
|
+
# Which kind names a statement when its surviving mutants disagree
|
|
62
|
+
# (#P4): the first one listed that any of them carries. The broadest
|
|
63
|
+
# unchecked thing wins, so a branch condition is not called a
|
|
64
|
+
# boundary because one of its mutants happened to flip a comparison.
|
|
65
|
+
PRECEDENCE = ["result unchecked", "error path", "branch", "boundary", "effect", "number"].freeze
|
|
66
|
+
|
|
61
67
|
# The deterministic per-kind reason printed next to a survivor's rank.
|
|
62
68
|
REASONS = {
|
|
63
69
|
"boundary" => "the covering specs pass without pinning this boundary",
|
|
@@ -68,6 +74,18 @@ module Saboteur
|
|
|
68
74
|
"result unchecked" => "the covering specs pass without pinning this result"
|
|
69
75
|
}.freeze
|
|
70
76
|
|
|
77
|
+
# What an observed label (input_gap/assertion_gap, #1290) replaces
|
|
78
|
+
# the kind with, and the advice the kind gave: kept in the reason.
|
|
79
|
+
OBSERVED = %w[input_gap assertion_gap].freeze
|
|
80
|
+
ADVICE = {
|
|
81
|
+
"boundary" => "exercise both sides of this boundary",
|
|
82
|
+
"branch" => "exercise both branches",
|
|
83
|
+
"effect" => "assert the effect of this statement",
|
|
84
|
+
"number" => "no test depends on this exact number",
|
|
85
|
+
"error path" => "exercise the error path",
|
|
86
|
+
"result unchecked" => "assert this result"
|
|
87
|
+
}.freeze
|
|
88
|
+
|
|
71
89
|
module_function
|
|
72
90
|
|
|
73
91
|
def kind(operator, comparison_edit: false)
|
|
@@ -20,7 +20,9 @@ module Saboteur
|
|
|
20
20
|
end
|
|
21
21
|
|
|
22
22
|
def call(report)
|
|
23
|
-
survivor
|
|
23
|
+
# The replay example is the #1 survivor of the ranked list, not
|
|
24
|
+
# whichever one the results hold first.
|
|
25
|
+
survivor = SurvivorRanking.ordered(report, report.unignored_survivors).first
|
|
24
26
|
return unless survivor
|
|
25
27
|
|
|
26
28
|
@out.puts "\n#{@palette.paint(:head, 'Next steps:')}"
|
|
@@ -32,7 +32,10 @@ module Saboteur
|
|
|
32
32
|
TOOL_LIMIT_HINT = "Saboteur could not observe this mutant faithfully. Check it by hand."
|
|
33
33
|
HINTS = GapKinds::KINDS.to_h { |label| [label, GAP_HINT] }
|
|
34
34
|
.merge("tool_limit" => TOOL_LIMIT_HINT,
|
|
35
|
-
"input_gap" => "No test
|
|
35
|
+
"input_gap" => "No test input makes this change differ. " \
|
|
36
|
+
"Exercise it with a varied input.",
|
|
37
|
+
"assertion_gap" => "A test reaches this line, the value changes, and it still " \
|
|
38
|
+
"passes. Add an assertion that pins this behaviour.").freeze
|
|
36
39
|
DEFAULT_HINT = "No test fails with this change. Check whether a test should pin it."
|
|
37
40
|
|
|
38
41
|
Plan = Struct.new(:post, :delete, :summary, keyword_init: true)
|
|
@@ -15,7 +15,6 @@ module Saboteur
|
|
|
15
15
|
# it. Untested subjects are never dropped silently, even at 100 %.
|
|
16
16
|
class ScoreTable
|
|
17
17
|
MAX_ROWS = 15
|
|
18
|
-
MAX_UNTESTED_NAMES = 3
|
|
19
18
|
# The skip reason CandidateSelection gives a subject no test covers.
|
|
20
19
|
NO_COVERAGE = "no coverage"
|
|
21
20
|
|
|
@@ -63,7 +62,7 @@ module Saboteur
|
|
|
63
62
|
detected: breakdown.detected, tested: breakdown.tested, untested: untested }
|
|
64
63
|
end
|
|
65
64
|
|
|
66
|
-
# "cart.rb 100.0% (2/2) · n/a - 2 subjects - no test executes them
|
|
65
|
+
# "cart.rb 100.0% (2/2) · n/a - 2 subjects - no test executes them": the
|
|
67
66
|
# tested score and the untested count stand next to each other; a
|
|
68
67
|
# file with nothing tested gets no percentage at all.
|
|
69
68
|
def line(row, width)
|
|
@@ -75,18 +74,14 @@ module Saboteur
|
|
|
75
74
|
note ? "#{base} #{@palette.glyph(:middot)} #{note}" : base
|
|
76
75
|
end
|
|
77
76
|
|
|
78
|
-
# "n/a - 2 subjects - no test executes them:
|
|
79
|
-
#
|
|
80
|
-
#
|
|
81
|
-
#
|
|
82
|
-
# they just never execute it.
|
|
77
|
+
# "n/a - 2 subjects - no test executes them": only the count. The
|
|
78
|
+
# Not scored block below names every subject once (P6), so the table
|
|
79
|
+
# does not repeat the names. Wording owns the claim (#1145): a
|
|
80
|
+
# stubbed-away subject has tests, they just never execute it.
|
|
83
81
|
def untested_note(untested)
|
|
84
82
|
return if untested.empty?
|
|
85
83
|
|
|
86
|
-
|
|
87
|
-
shown = names.first(MAX_UNTESTED_NAMES)
|
|
88
|
-
shown << "+#{names.size - MAX_UNTESTED_NAMES} more" if names.size > MAX_UNTESTED_NAMES
|
|
89
|
-
@palette.paint(:warn, "n/a - #{Wording.untested_note(untested.size)}: #{shown.join(', ')}")
|
|
84
|
+
@palette.paint(:warn, "n/a - #{Wording.untested_note(untested.size)}")
|
|
90
85
|
end
|
|
91
86
|
end
|
|
92
87
|
end
|
|
@@ -98,7 +98,7 @@ module Saboteur
|
|
|
98
98
|
end
|
|
99
99
|
|
|
100
100
|
def self.test_index(results)
|
|
101
|
-
names = results.flat_map { |result| result.covered_by + result.killed_by + result.executed.to_a }.uniq
|
|
101
|
+
names = results.flat_map { |result| result.covered_by + result.killed_by + result.executed.to_a }.uniq.sort
|
|
102
102
|
names.each_with_index.to_h { |name, index| [name, index.to_s] }
|
|
103
103
|
end
|
|
104
104
|
|
|
@@ -139,6 +139,15 @@ module Saboteur
|
|
|
139
139
|
# Why the survivor is worth a look, as a label only: scores and
|
|
140
140
|
# the exit code never change (#192).
|
|
141
141
|
classification: SurvivorClassification.label(result),
|
|
142
|
+
# Additive (#1290): the gap kind an observed label replaced.
|
|
143
|
+
gapKind: SurvivorClassification.kind(result),
|
|
144
|
+
# Additive (#1286): confirmed, killed_by_confirmation or unconfirmed.
|
|
145
|
+
# An unconfirmed survivor reports Ignored: outside the score.
|
|
146
|
+
confirmation: result.confirmation_state,
|
|
147
|
+
# Additive (#1290): whether the mutated expression ever produced a
|
|
148
|
+
# different value from the original; absent when not probed.
|
|
149
|
+
valueDiffered: result.value_differed,
|
|
150
|
+
creditedTest: result.credited_test&.then { |name| test_ids.fetch(name) },
|
|
142
151
|
killedBy: result.killed_by.map { |name| test_ids.fetch(name) }.then { |ids| ids.empty? ? nil : ids },
|
|
143
152
|
coveredBy: result.covered_by.map { |name| test_ids.fetch(name) }.then { |ids| ids.empty? ? nil : ids },
|
|
144
153
|
# Which of the covering tests actually ran against the mutant; absent
|
|
@@ -151,7 +160,7 @@ module Saboteur
|
|
|
151
160
|
def self.status(result)
|
|
152
161
|
case result.status
|
|
153
162
|
when :killed then "Killed"
|
|
154
|
-
when :survived then "Survived"
|
|
163
|
+
when :survived then result.unconfirmed? ? "Ignored" : "Survived"
|
|
155
164
|
when :timeout then "Timeout"
|
|
156
165
|
when :error, :resource_limit then "RuntimeError"
|
|
157
166
|
when :skipped then result.reason&.start_with?("no coverage") ? "NoCoverage" : "Ignored"
|
|
@@ -169,6 +178,7 @@ module Saboteur
|
|
|
169
178
|
# ignore, and only wont_fix adds a suffix after the reason.
|
|
170
179
|
return "ignored: #{result.ignored} (wont_fix)" if result.triage_state == "wont_fix"
|
|
171
180
|
return "ignored: #{result.ignored}" if result.ignored
|
|
181
|
+
return "unconfirmed: #{result.unconfirmed_reason}" if result.unconfirmed?
|
|
172
182
|
return "resource limit: #{result.reason}" if result.status == :resource_limit
|
|
173
183
|
return result.crash_kill? ? "crash: #{result.kill_cause}" : result.kill_cause if result.kill_cause
|
|
174
184
|
|
|
@@ -29,11 +29,14 @@ module Saboteur
|
|
|
29
29
|
# operator and the edit shape only. The covering list is only the
|
|
30
30
|
# PLANNED coverage: whole-body (coarse) mutants carry method-level
|
|
31
31
|
# guesses, so they never get a gap kind.
|
|
32
|
-
# input_gap -
|
|
33
|
-
#
|
|
34
|
-
#
|
|
35
|
-
#
|
|
36
|
-
#
|
|
32
|
+
# input_gap - the tests never gave the mutated expression an input
|
|
33
|
+
# where its value differs (the runner recorded it, #1290), or no
|
|
34
|
+
# real test covered the mutant at all (D6). Coarse mutants stay
|
|
35
|
+
# unlabeled: their planned coverage is a method-level guess.
|
|
36
|
+
# assertion_gap - the mutated expression produced a different value
|
|
37
|
+
# while the covering tests ran, and they still passed: a test
|
|
38
|
+
# reaches the code and no assertion pins it (#1290). Only side-effect
|
|
39
|
+
# free sites are observed; an unprobed site keeps its gap kind.
|
|
37
40
|
# tool_limit - survivors saboteur could not observe faithfully: an
|
|
38
41
|
# external-runner run records only the pseudo-test id, never real
|
|
39
42
|
# coverage (#1071). A re-eval refusal records no signal on the
|
|
@@ -41,7 +44,10 @@ module Saboteur
|
|
|
41
44
|
#
|
|
42
45
|
# A survivor with no matching signal stays unlabeled.
|
|
43
46
|
module SurvivorClassification
|
|
44
|
-
LABELS = %w[likely_equivalent arid tool_limit input_gap].concat(GapKinds::KINDS).freeze
|
|
47
|
+
LABELS = %w[likely_equivalent arid tool_limit input_gap assertion_gap].concat(GapKinds::KINDS).freeze
|
|
48
|
+
# assertion_gap decides a statement's label ahead of every gap kind: the
|
|
49
|
+
# value changed and the tests passed.
|
|
50
|
+
GROUP_ORDER = (["assertion_gap"] + GapKinds::PRECEDENCE).freeze
|
|
45
51
|
|
|
46
52
|
class << self
|
|
47
53
|
def label(result, arid: MutationSource::Builtin::DEFAULT_ARID)
|
|
@@ -49,12 +55,44 @@ module Saboteur
|
|
|
49
55
|
return "likely_equivalent" if equivalent?(result)
|
|
50
56
|
return "arid" if arid?(result, arid)
|
|
51
57
|
return "tool_limit" if tool_limited?(result)
|
|
58
|
+
|
|
59
|
+
observed = observed_gap(result)
|
|
60
|
+
return observed if observed
|
|
52
61
|
return gap_kind(result) if possible_test_gap?(result)
|
|
53
62
|
return "input_gap" unless result.mutation.coarse
|
|
54
63
|
|
|
55
64
|
nil
|
|
56
65
|
end
|
|
57
66
|
|
|
67
|
+
# The label of one statement's surviving mutants as a whole, the
|
|
68
|
+
# same whichever mutant sorts first. Any gap kind decides, by
|
|
69
|
+
# GapKinds::PRECEDENCE, so one equivalent, logging or input_gap
|
|
70
|
+
# sibling never hides a real survivor. With no gap kind, the first
|
|
71
|
+
# label present stands: likely_equivalent, arid or tool_limit then
|
|
72
|
+
# describe the statement when all its mutants carry it.
|
|
73
|
+
#
|
|
74
|
+
# Among gap kinds, result unchecked wins because it is the strictly
|
|
75
|
+
# larger gap: nothing pins the statement's value at all, which
|
|
76
|
+
# subsumes a missing branch, boundary or effect check on it.
|
|
77
|
+
def group_label(results)
|
|
78
|
+
labels = results.map { |result| label(result) }.compact
|
|
79
|
+
GROUP_ORDER.find { |kind| labels.include?(kind) } || labels.first
|
|
80
|
+
end
|
|
81
|
+
|
|
82
|
+
# The gap kind an observed label (input_gap/assertion_gap) replaced:
|
|
83
|
+
# what the edit shape says is unchecked. nil unless the observation
|
|
84
|
+
# decided the label, so a plain kind label stays its own kind.
|
|
85
|
+
def kind(result)
|
|
86
|
+
return nil unless GapKinds::OBSERVED.include?(observed_gap(result)) && possible_test_gap?(result)
|
|
87
|
+
|
|
88
|
+
gap_kind(result)
|
|
89
|
+
end
|
|
90
|
+
|
|
91
|
+
def group_kind(results)
|
|
92
|
+
kinds = results.map { |result| kind(result) }.compact
|
|
93
|
+
GapKinds::PRECEDENCE.find { |kind| kinds.include?(kind) }
|
|
94
|
+
end
|
|
95
|
+
|
|
58
96
|
private
|
|
59
97
|
|
|
60
98
|
# The mutant's own def text, compiled side by side with the
|
|
@@ -76,6 +114,15 @@ module Saboteur
|
|
|
76
114
|
# An external runner reports one pseudo id for every mutant and no
|
|
77
115
|
# coverage map, so a survivor there was never observed against real
|
|
78
116
|
# tests. Pseudo id next to a real id keeps the gap kind (#305).
|
|
117
|
+
# The runner's observation (#1290): the mutated expression never gave
|
|
118
|
+
# a different value (input_gap), or gave one and the tests passed
|
|
119
|
+
# (assertion_gap). nil when the site was not probed.
|
|
120
|
+
def observed_gap(result)
|
|
121
|
+
return nil if result.mutation.coarse || result.value_differed.nil?
|
|
122
|
+
|
|
123
|
+
result.value_differed ? "assertion_gap" : "input_gap"
|
|
124
|
+
end
|
|
125
|
+
|
|
79
126
|
def tool_limited?(result)
|
|
80
127
|
result.covered_by.any? && result.covered_by.all? { |id| id == Execution::ExternalRunner::PSEUDO_TEST }
|
|
81
128
|
end
|
|
@@ -48,9 +48,8 @@ module Saboteur
|
|
|
48
48
|
UNEQUAL_CASE = [%w[< >], %w[<= >=]].map(&:to_set).freeze
|
|
49
49
|
|
|
50
50
|
# Labels that never get a hint: a generated assertion on an equivalent
|
|
51
|
-
# mutant is the worst kind of hint noise.
|
|
52
|
-
#
|
|
53
|
-
# label keeps the operator hint unchanged.
|
|
51
|
+
# mutant is the worst kind of hint noise. Every other label, including
|
|
52
|
+
# input_gap and assertion_gap, keeps the operator hint.
|
|
54
53
|
NO_HINT_LABELS = %w[likely_equivalent].freeze
|
|
55
54
|
|
|
56
55
|
# One deterministic template sentence per operator: what the change
|
|
@@ -91,13 +90,14 @@ module Saboteur
|
|
|
91
90
|
|
|
92
91
|
# The deterministic per-label reason printed next to a survivor's rank.
|
|
93
92
|
# Labels come from SurvivorClassification; they are heuristics, not
|
|
94
|
-
# diagnoses.
|
|
95
|
-
#
|
|
93
|
+
# diagnoses. input_gap and assertion_gap keep the operator hint: it is
|
|
94
|
+
# the right place to look in both cases (#1290).
|
|
96
95
|
LABEL_REASONS = {
|
|
97
96
|
"likely_equivalent" => "compiles to the same instructions as the original",
|
|
98
97
|
"arid" => "every edited line matches an arid pattern",
|
|
99
98
|
"tool_limit" => "the tool could not observe this mutant faithfully",
|
|
100
|
-
"input_gap" => "no test
|
|
99
|
+
"input_gap" => "no test input made this mutation change the value, so its input never varied",
|
|
100
|
+
"assertion_gap" => "a test ran this mutant, the value changed, and no assertion noticed"
|
|
101
101
|
}.merge(GapKinds::REASONS).freeze
|
|
102
102
|
|
|
103
103
|
module_function
|
|
@@ -114,9 +114,13 @@ module Saboteur
|
|
|
114
114
|
WHY_IT_MATTERS.fetch(operator, WHY_FALLBACK)
|
|
115
115
|
end
|
|
116
116
|
|
|
117
|
-
# The per-label reason sentence; nil when the label has none.
|
|
118
|
-
|
|
119
|
-
|
|
117
|
+
# The per-label reason sentence; nil when the label has none. An
|
|
118
|
+
# observed label (input_gap/assertion_gap) keeps the advice of the gap
|
|
119
|
+
# kind it replaced, so the kind is not lost behind the observation.
|
|
120
|
+
def label_reason(label, kind: nil)
|
|
121
|
+
reason = LABEL_REASONS[label]
|
|
122
|
+
advice = GapKinds::ADVICE[kind]
|
|
123
|
+
advice && GapKinds::OBSERVED.include?(label) ? "#{reason}; #{advice}" : reason
|
|
120
124
|
end
|
|
121
125
|
|
|
122
126
|
def for(operator, coarse: false)
|
|
@@ -77,8 +77,8 @@ module Saboteur
|
|
|
77
77
|
line = MutationText.line(first)
|
|
78
78
|
path = relative(first.subject.file, report.root)
|
|
79
79
|
@out.puts "\n#{@palette.paint(:head, "#{rank}. #{path}:#{line}")} #{first.subject.id} " \
|
|
80
|
-
"#{@palette.paint(:dim, label_suffix(
|
|
81
|
-
reason = SurvivorHints.label_reason(@classifier.
|
|
80
|
+
"#{@palette.paint(:dim, label_suffix(group, rank, total))}".rstrip
|
|
81
|
+
reason = SurvivorHints.label_reason(@classifier.group_label(group), kind: @classifier.group_kind(group))
|
|
82
82
|
@out.puts " #{@palette.paint(:dim, reason)}" if reason
|
|
83
83
|
fence(first, line)
|
|
84
84
|
@out.puts " #{@palette.paint(:dim, "why it matters: #{why(first.mutation.operator)}")}"
|
|
@@ -105,10 +105,10 @@ module Saboteur
|
|
|
105
105
|
# siblings read "#3 of 14 blocks", never blocks against mutants (#1147).
|
|
106
106
|
# Label and rank stay on the header line; the reason moves to its own
|
|
107
107
|
# dim line so the fixed text cannot push the header past 80 columns.
|
|
108
|
-
def label_suffix(
|
|
109
|
-
label = @classifier.
|
|
108
|
+
def label_suffix(group, rank, total)
|
|
109
|
+
label = @classifier.group_label(group)
|
|
110
110
|
unit = total == 1 ? "block" : "blocks"
|
|
111
|
-
parts = [("coarse" if
|
|
111
|
+
parts = [("coarse" if group.first.mutation.coarse), label, "##{rank} of #{total} #{unit}"].compact
|
|
112
112
|
"[#{parts.join(' · ')}]"
|
|
113
113
|
end
|
|
114
114
|
|
|
@@ -226,7 +226,7 @@ module Saboteur
|
|
|
226
226
|
|
|
227
227
|
whole = @whole_suite_map&.tests_for(group.first.subject)
|
|
228
228
|
if whole.nil?
|
|
229
|
-
@out.puts @palette.paint(:dim, " covering
|
|
229
|
+
@out.puts @palette.paint(:dim, " covering list is this run's selection only; run `saboteur run --all`")
|
|
230
230
|
return
|
|
231
231
|
end
|
|
232
232
|
|
|
@@ -6,6 +6,7 @@ require_relative "wording"
|
|
|
6
6
|
require_relative "baseline_tail"
|
|
7
7
|
require_relative "next_steps"
|
|
8
8
|
require_relative "kill_causes"
|
|
9
|
+
require_relative "unconfirmed_section"
|
|
9
10
|
require_relative "survivor_ranking"
|
|
10
11
|
require_relative "palette"
|
|
11
12
|
require_relative "notice"
|
|
@@ -62,6 +63,7 @@ module Saboteur
|
|
|
62
63
|
return quiet_call(report) if @quiet
|
|
63
64
|
|
|
64
65
|
findings(report)
|
|
66
|
+
UnconfirmedSection.new(@out, @palette).call(report)
|
|
65
67
|
errors(report)
|
|
66
68
|
skipped_subjects(report) if @verbose
|
|
67
69
|
test_command_gaps(report)
|
|
@@ -94,7 +96,7 @@ module Saboteur
|
|
|
94
96
|
|
|
95
97
|
def quiet_survivor_line(path, line, subject_id, results)
|
|
96
98
|
first, *rest = results
|
|
97
|
-
label = SurvivorClassification.
|
|
99
|
+
label = SurvivorClassification.group_label(results)
|
|
98
100
|
text = "#{path}:#{line} #{subject_id}#{" [#{label}]" if label} - #{first.digest}"
|
|
99
101
|
rest.any? ? "#{text} (+#{rest.size} more)" : text
|
|
100
102
|
end
|
|
@@ -346,7 +348,7 @@ module Saboteur
|
|
|
346
348
|
untested = untested_subjects(report).size
|
|
347
349
|
[counted(counts[:killed], "killed", plural: false, color: :pass),
|
|
348
350
|
counted(live, "survived", plural: false, color: :fail),
|
|
349
|
-
counted(report.survivors.size - live, "accepted", plural: false),
|
|
351
|
+
counted(report.survivors.size - live, "accepted", plural: false), UnconfirmedSection.part(report),
|
|
350
352
|
counted(counts[:timeout], "timeout", color: :warn), counted(counts[:resource_limit], "resource limit"),
|
|
351
353
|
error_part(report, counts[:error]), counted(counts[:unverified], "unverified", plural: false),
|
|
352
354
|
("#{untested} subject#{'s' unless untested == 1} untested" if untested.positive?)].compact
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
# SPDX-License-Identifier: MIT
|
|
2
|
+
# frozen_string_literal: true
|
|
3
|
+
|
|
4
|
+
require_relative "mutation_text"
|
|
5
|
+
require_relative "survivor_ranking"
|
|
6
|
+
|
|
7
|
+
module Saboteur
|
|
8
|
+
module Reporter
|
|
9
|
+
# Survivors whose covering tests were not all run (#1286): outside the
|
|
10
|
+
# score and the exit code, so they get their own short list instead of
|
|
11
|
+
# a place among the findings.
|
|
12
|
+
class UnconfirmedSection
|
|
13
|
+
LIMIT = 10
|
|
14
|
+
|
|
15
|
+
# The count for the one-line (--quiet) tally, nil when there is none.
|
|
16
|
+
def self.part(report)
|
|
17
|
+
count = report.unconfirmed_survivors.size
|
|
18
|
+
"#{count} unconfirmed" if count.positive?
|
|
19
|
+
end
|
|
20
|
+
|
|
21
|
+
def initialize(out, palette)
|
|
22
|
+
@out = out
|
|
23
|
+
@palette = palette
|
|
24
|
+
end
|
|
25
|
+
|
|
26
|
+
def call(report)
|
|
27
|
+
pending = report.unconfirmed_survivors
|
|
28
|
+
return if pending.empty?
|
|
29
|
+
|
|
30
|
+
@out.puts @palette.paint(:head, "unconfirmed (covering tests not run) - #{pending.size}, not in the score")
|
|
31
|
+
SurvivorRanking.ordered(report, pending).first(LIMIT).each { |result| @out.puts line(report, result) }
|
|
32
|
+
@out.puts " ... #{pending.size - LIMIT} more" if pending.size > LIMIT
|
|
33
|
+
@out.puts
|
|
34
|
+
end
|
|
35
|
+
|
|
36
|
+
private
|
|
37
|
+
|
|
38
|
+
def line(report, result)
|
|
39
|
+
path = result.subject.file.delete_prefix("#{report.root}/")
|
|
40
|
+
" #{path}:#{MutationText.line(result)} #{result.subject.id} - #{result.digest} (#{result.unconfirmed_reason})"
|
|
41
|
+
end
|
|
42
|
+
end
|
|
43
|
+
end
|
|
44
|
+
end
|
data/lib/saboteur/result.rb
CHANGED
|
@@ -35,9 +35,13 @@ module Saboteur
|
|
|
35
35
|
# status keeps its meaning either way.
|
|
36
36
|
attr_reader :confirmation
|
|
37
37
|
|
|
38
|
+
# Whether the mutated expression ever produced a value different from the
|
|
39
|
+
# original one while the covering tests ran; nil when not probed (#1290).
|
|
40
|
+
attr_reader :value_differed
|
|
41
|
+
|
|
38
42
|
def initialize(subject:, mutation:, status:, mutated_source:, reason: nil, runtime: nil, killed_by: [],
|
|
39
43
|
covered_by: [], kill_cause: nil, reused: false, rejected_kill: false, executed: nil,
|
|
40
|
-
confirmation: nil)
|
|
44
|
+
confirmation: nil, value_differed: nil)
|
|
41
45
|
raise ArgumentError, "unknown status #{status.inspect}" unless STATUSES.include?(status)
|
|
42
46
|
|
|
43
47
|
@subject = subject
|
|
@@ -53,6 +57,7 @@ module Saboteur
|
|
|
53
57
|
@rejected_kill = rejected_kill
|
|
54
58
|
@reused = reused
|
|
55
59
|
@confirmation = confirmation
|
|
60
|
+
@value_differed = value_differed
|
|
56
61
|
@ignored = nil
|
|
57
62
|
# triage_state/triage_reason stay unset until the ignore list writes
|
|
58
63
|
# them: the accessors read as nil before that, and there is no
|
|
@@ -69,7 +74,7 @@ module Saboteur
|
|
|
69
74
|
MutantResult.new(subject: subject, mutation: mutation, status: status, mutated_source: mutated_source,
|
|
70
75
|
reason: reason, runtime: runtime, killed_by: killed_by, covered_by: covered_by,
|
|
71
76
|
kill_cause: kill_cause, reused: reused?, rejected_kill: rejected_kill?,
|
|
72
|
-
executed: executed, confirmation: confirmation)
|
|
77
|
+
executed: executed, confirmation: confirmation, value_differed: value_differed)
|
|
73
78
|
end
|
|
74
79
|
|
|
75
80
|
# The reason a --flaky-check disagreement starts with: unverified, but
|
|
@@ -112,8 +117,37 @@ module Saboteur
|
|
|
112
117
|
@digest ||= mutation.digest(subject_start: subject.start_byte, original_source: subject.file_source)
|
|
113
118
|
end
|
|
114
119
|
|
|
120
|
+
# A survivor whose confirmation run could not happen (budget spent, or the
|
|
121
|
+
# run failed) is outside the score and the exit code until it is
|
|
122
|
+
# confirmed (#1286). A survivor the whole-suite map cannot speak for
|
|
123
|
+
# ("unconfirmed_map(...)") stays counted: a stale map never lowers the gate.
|
|
115
124
|
def survived?
|
|
116
|
-
status == :survived
|
|
125
|
+
status == :survived && !unconfirmed?
|
|
126
|
+
end
|
|
127
|
+
|
|
128
|
+
def unconfirmed?
|
|
129
|
+
status == :survived && @confirmation.to_s.start_with?("unconfirmed(")
|
|
130
|
+
end
|
|
131
|
+
|
|
132
|
+
# The additive report field: confirmed, killed_by_confirmation or
|
|
133
|
+
# unconfirmed; nil when confirmation never applied (#1286).
|
|
134
|
+
def confirmation_state
|
|
135
|
+
return nil if @confirmation.nil?
|
|
136
|
+
|
|
137
|
+
label = @confirmation.to_s
|
|
138
|
+
label.start_with?("unconfirmed") ? "unconfirmed" : label
|
|
139
|
+
end
|
|
140
|
+
|
|
141
|
+
# The test a confirmation kill is credited to.
|
|
142
|
+
def credited_test
|
|
143
|
+
killed_by.first if confirmation_state == "killed_by_confirmation"
|
|
144
|
+
end
|
|
145
|
+
|
|
146
|
+
# Why the survivor stayed unconfirmed: the text inside unconfirmed(...).
|
|
147
|
+
def unconfirmed_reason
|
|
148
|
+
return unless unconfirmed?
|
|
149
|
+
|
|
150
|
+
@confirmation.delete_prefix("unconfirmed(").delete_suffix(")")
|
|
117
151
|
end
|
|
118
152
|
|
|
119
153
|
# Killed because a test's error, not its check, fired: NoMethodError and
|
|
@@ -280,7 +314,9 @@ module Saboteur
|
|
|
280
314
|
end
|
|
281
315
|
|
|
282
316
|
def counts
|
|
283
|
-
STATUSES.to_h
|
|
317
|
+
STATUSES.to_h do |status|
|
|
318
|
+
[status, results.count { |result| status == :survived ? result.survived? : result.status == status }]
|
|
319
|
+
end
|
|
284
320
|
end
|
|
285
321
|
|
|
286
322
|
# (killed + timeout) / (killed + survived + timeout); nil with nothing tested.
|
|
@@ -308,6 +344,12 @@ module Saboteur
|
|
|
308
344
|
results.select(&:survived?)
|
|
309
345
|
end
|
|
310
346
|
|
|
347
|
+
# Survivors whose covering tests were not all run: outside the score and
|
|
348
|
+
# the exit code, listed on their own (#1286).
|
|
349
|
+
def unconfirmed_survivors
|
|
350
|
+
results.select(&:unconfirmed?)
|
|
351
|
+
end
|
|
352
|
+
|
|
311
353
|
def unignored_survivors
|
|
312
354
|
results.select(&:unignored_survivor?)
|
|
313
355
|
end
|
|
@@ -443,9 +443,19 @@ module Saboteur
|
|
|
443
443
|
[prefix.count("\n") + 1, offset - ((prefix.rindex("\n") || -1) + 1)]
|
|
444
444
|
end
|
|
445
445
|
|
|
446
|
+
# Durations are compared to the millisecond. Two tests within a
|
|
447
|
+
# millisecond of each other swap places from one map build to the next,
|
|
448
|
+
# and with fail-fast the first failing test names a mutant's killer and
|
|
449
|
+
# kill cause, so sub-millisecond noise must not decide the order (#1274).
|
|
450
|
+
# Suite order breaks the tie instead.
|
|
446
451
|
def sort_test_ids(ids)
|
|
447
452
|
order = test_order
|
|
448
|
-
ids.sort_by { |id| [
|
|
453
|
+
ids.sort_by { |id| [duration_bucket(id), order.fetch(id, Float::INFINITY)] }
|
|
454
|
+
end
|
|
455
|
+
|
|
456
|
+
def duration_bucket(id)
|
|
457
|
+
seconds = durations[id]
|
|
458
|
+
seconds ? (seconds * 1000).ceil : Float::INFINITY
|
|
449
459
|
end
|
|
450
460
|
|
|
451
461
|
def test_order
|
data/lib/saboteur/version.rb
CHANGED
data/lib/saboteur.rb
CHANGED
|
@@ -6,6 +6,10 @@ require_relative "saboteur/version"
|
|
|
6
6
|
module Saboteur
|
|
7
7
|
class Error < StandardError; end
|
|
8
8
|
|
|
9
|
+
# A spec path from the cached coverage map that is not a spec file inside
|
|
10
|
+
# the project: the map is untrusted input (#427, #1286).
|
|
11
|
+
class RefusedSpecPath < Error; end
|
|
12
|
+
|
|
9
13
|
# A phase timeout that --max-duration shortened: budget exhaustion, not a
|
|
10
14
|
# broken suite. The engine turns it into the budget partial-report path
|
|
11
15
|
# instead of a phase error (#296).
|
|
@@ -66,6 +70,7 @@ require_relative "saboteur/execution/marker_sweep"
|
|
|
66
70
|
require_relative "saboteur/execution/supervisor"
|
|
67
71
|
require_relative "saboteur/execution/reevaluator"
|
|
68
72
|
require_relative "saboteur/execution/wrapper"
|
|
73
|
+
require_relative "saboteur/execution/site_probe"
|
|
69
74
|
require_relative "saboteur/execution/runner"
|
|
70
75
|
require_relative "saboteur/execution/bundle_gemfile_pin"
|
|
71
76
|
require_relative "saboteur/execution/gemfile_path_sources"
|
metadata
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
|
2
2
|
name: saboteur
|
|
3
3
|
version: !ruby/object:Gem::Version
|
|
4
|
-
version: 0.12.
|
|
4
|
+
version: 0.12.3
|
|
5
5
|
platform: ruby
|
|
6
6
|
authors:
|
|
7
7
|
- Vamsi Pavan Mahesh Gunturu
|
|
@@ -63,6 +63,7 @@ files:
|
|
|
63
63
|
- lib/saboteur/cli.rb
|
|
64
64
|
- lib/saboteur/cli/agent_provenance.rb
|
|
65
65
|
- lib/saboteur/cli/baseline_gate.rb
|
|
66
|
+
- lib/saboteur/cli/color_option.rb
|
|
66
67
|
- lib/saboteur/cli/dirty_tree_notice.rb
|
|
67
68
|
- lib/saboteur/cli/doctor_command.rb
|
|
68
69
|
- lib/saboteur/cli/error_cause.rb
|
|
@@ -89,6 +90,7 @@ files:
|
|
|
89
90
|
- lib/saboteur/cli/shard_options.rb
|
|
90
91
|
- lib/saboteur/cli/source_set.rb
|
|
91
92
|
- lib/saboteur/cli/survivors_command.rb
|
|
93
|
+
- lib/saboteur/cli/whole_suite_map_lookup.rb
|
|
92
94
|
- lib/saboteur/config.rb
|
|
93
95
|
- lib/saboteur/config_file.rb
|
|
94
96
|
- lib/saboteur/constant_reads.rb
|
|
@@ -162,6 +164,7 @@ files:
|
|
|
162
164
|
- lib/saboteur/execution/runner.rb
|
|
163
165
|
- lib/saboteur/execution/sandbox.rb
|
|
164
166
|
- lib/saboteur/execution/seatbelt.rb
|
|
167
|
+
- lib/saboteur/execution/site_probe.rb
|
|
165
168
|
- lib/saboteur/execution/stale_captures.rb
|
|
166
169
|
- lib/saboteur/execution/supervisor.rb
|
|
167
170
|
- lib/saboteur/execution/supervisor_batch.rb
|
|
@@ -253,6 +256,7 @@ files:
|
|
|
253
256
|
- lib/saboteur/reporter/survivor_ranking.rb
|
|
254
257
|
- lib/saboteur/reporter/terminal.rb
|
|
255
258
|
- lib/saboteur/reporter/test_matrix.rb
|
|
259
|
+
- lib/saboteur/reporter/unconfirmed_section.rb
|
|
256
260
|
- lib/saboteur/reporter/wording.rb
|
|
257
261
|
- lib/saboteur/result.rb
|
|
258
262
|
- lib/saboteur/score.rb
|