saboteur 0.11.0 → 0.11.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +86 -0
- data/README.md +2 -2
- data/lib/saboteur/baseline/command.rb +46 -2
- data/lib/saboteur/baseline/comparison.rb +6 -1
- data/lib/saboteur/baseline/completeness.rb +21 -0
- data/lib/saboteur/baseline/evidence.rb +11 -3
- data/lib/saboteur/baseline/loader.rb +18 -5
- data/lib/saboteur/baseline/manifest.rb +13 -4
- data/lib/saboteur/baseline/run_evidence.rb +3 -1
- data/lib/saboteur/baseline/workflow.rb +57 -9
- data/lib/saboteur/baseline_health.rb +4 -1
- data/lib/saboteur/cli/dirty_tree_notice.rb +7 -1
- data/lib/saboteur/cli/doctor_command.rb +10 -1
- data/lib/saboteur/cli/exit_status.rb +10 -5
- data/lib/saboteur/cli/explain_command.rb +51 -0
- data/lib/saboteur/cli/explicit_sources.rb +1 -1
- data/lib/saboteur/cli/failure_output.rb +2 -2
- data/lib/saboteur/cli/flaky_exit.rb +23 -0
- data/lib/saboteur/cli/presets.rb +11 -1
- data/lib/saboteur/cli/run_command.rb +17 -1
- data/lib/saboteur/cli/run_output.rb +10 -0
- data/lib/saboteur/cli/run_reporting.rb +13 -2
- data/lib/saboteur/cli/survivors_command.rb +82 -18
- data/lib/saboteur/cli.rb +3 -0
- data/lib/saboteur/config.rb +7 -2
- data/lib/saboteur/crash_report.rb +3 -2
- data/lib/saboteur/doctor.rb +11 -2
- data/lib/saboteur/engine/boot.rb +4 -2
- data/lib/saboteur/engine/coverage_store.rb +54 -3
- data/lib/saboteur/engine/distribution.rb +6 -2
- data/lib/saboteur/engine/pipelines.rb +14 -2
- data/lib/saboteur/engine/result_cache.rb +4 -2
- data/lib/saboteur/engine/scheduler.rb +23 -21
- data/lib/saboteur/engine/selection_announcement.rb +5 -4
- data/lib/saboteur/engine/subject_selection.rb +11 -1
- data/lib/saboteur/engine/survivor_confirmation.rb +1 -1
- data/lib/saboteur/engine/test_guess.rb +1 -1
- data/lib/saboteur/engine/thread_warnings.rb +9 -4
- data/lib/saboteur/engine.rb +57 -9
- data/lib/saboteur/execution/fiddle_loader.rb +29 -2
- data/lib/saboteur/help.rb +13 -1
- data/lib/saboteur/ignore_list.rb +36 -3
- data/lib/saboteur/install.rb +18 -3
- data/lib/saboteur/mutation_source/builtin.rb +1 -0
- data/lib/saboteur/mutation_source/operator_explain.rb +181 -0
- data/lib/saboteur/preview.rb +6 -1
- data/lib/saboteur/reporter/agent_commands.rb +29 -7
- data/lib/saboteur/reporter/agent_incomplete.rb +9 -2
- data/lib/saboteur/reporter/agent_summary.rb +2 -1
- data/lib/saboteur/reporter/baseline_tail.rb +9 -7
- data/lib/saboteur/reporter/gap_kinds.rb +80 -0
- data/lib/saboteur/reporter/html.rb +160 -10
- data/lib/saboteur/reporter/next_steps.rb +9 -7
- data/lib/saboteur/reporter/notice.rb +24 -0
- data/lib/saboteur/reporter/reuse_note.rb +37 -0
- data/lib/saboteur/reporter/review_comments.rb +17 -12
- data/lib/saboteur/reporter/score_table.rb +8 -7
- data/lib/saboteur/reporter/stryker.rb +11 -1
- data/lib/saboteur/reporter/survivor_classification.rb +26 -8
- data/lib/saboteur/reporter/survivor_hints.rb +3 -2
- data/lib/saboteur/reporter/survivor_list.rb +58 -4
- data/lib/saboteur/reporter/survivor_ranking.rb +13 -5
- data/lib/saboteur/reporter/terminal.rb +32 -33
- data/lib/saboteur/reporter/wording.rb +37 -0
- data/lib/saboteur/result.rb +39 -8
- data/lib/saboteur/selection/git_diff.rb +19 -1
- data/lib/saboteur/survivor_guidance.rb +33 -0
- data/lib/saboteur/text.rb +6 -0
- data/lib/saboteur/version.rb +1 -1
- metadata +10 -2
|
@@ -1,6 +1,8 @@
|
|
|
1
1
|
# SPDX-License-Identifier: MIT
|
|
2
2
|
# frozen_string_literal: true
|
|
3
3
|
|
|
4
|
+
require_relative "gap_kinds"
|
|
5
|
+
|
|
4
6
|
module Saboteur
|
|
5
7
|
module Reporter
|
|
6
8
|
# Pull-request review comments for surviving mutants (#246): decides what
|
|
@@ -25,11 +27,10 @@ module Saboteur
|
|
|
25
27
|
# Labels whose survivors are probably noise: never worth a comment.
|
|
26
28
|
QUIET = %w[likely_equivalent arid].freeze
|
|
27
29
|
|
|
28
|
-
|
|
29
|
-
|
|
30
|
-
|
|
31
|
-
|
|
32
|
-
}.freeze
|
|
30
|
+
GAP_HINT = "A test reaches this line and still passes with the change. " \
|
|
31
|
+
"Add an assertion that pins this behaviour."
|
|
32
|
+
TOOL_LIMIT_HINT = "Saboteur could not observe this mutant faithfully. Check it by hand."
|
|
33
|
+
HINTS = GapKinds::KINDS.to_h { |label| [label, GAP_HINT] }.merge("tool_limit" => TOOL_LIMIT_HINT).freeze
|
|
33
34
|
DEFAULT_HINT = "No test fails with this change. Check whether a test should pin it."
|
|
34
35
|
|
|
35
36
|
Plan = Struct.new(:post, :delete, :summary, keyword_init: true)
|
|
@@ -57,12 +58,8 @@ module Saboteur
|
|
|
57
58
|
end
|
|
58
59
|
|
|
59
60
|
def body(finding)
|
|
60
|
-
[marker(finding["digest"]),
|
|
61
|
-
"
|
|
62
|
-
"",
|
|
63
|
-
fenced(finding["original"], "Original"),
|
|
64
|
-
fenced(finding["replacement"], "Mutant"),
|
|
65
|
-
"",
|
|
61
|
+
[marker(finding["digest"]), opening(finding), "",
|
|
62
|
+
fenced(finding["original"], "Original"), fenced(finding["replacement"], "Mutant"), "",
|
|
66
63
|
hint(finding["classification"])].join("\n")
|
|
67
64
|
end
|
|
68
65
|
|
|
@@ -128,7 +125,8 @@ module Saboteur
|
|
|
128
125
|
return nil if left_out.zero?
|
|
129
126
|
|
|
130
127
|
text = "#{SUMMARY_MARKER}\n#{left_out} more surviving mutant#{'s' unless left_out == 1} " \
|
|
131
|
-
"not shown as comments."
|
|
128
|
+
"not shown as comments. Add a test that fails for each one, or accept it in " \
|
|
129
|
+
"config/saboteur_ignore.rb with a reason."
|
|
132
130
|
HTTPS_URL.match?(run_url.to_s) ? "#{text} Full report: #{run_url}" : text
|
|
133
131
|
end
|
|
134
132
|
|
|
@@ -136,6 +134,13 @@ module Saboteur
|
|
|
136
134
|
HINTS.fetch(label, DEFAULT_HINT)
|
|
137
135
|
end
|
|
138
136
|
|
|
137
|
+
# The subject rides in the opening line: notification mail shows the
|
|
138
|
+
# body without its diff (#1212). Empty when there is none.
|
|
139
|
+
def opening(finding)
|
|
140
|
+
subject = finding["subject"].to_s.gsub(/[`\r\n]/, "")[0, 60].then { |n| n.empty? ? "" : " in `#{n}`" }
|
|
141
|
+
"**Surviving mutant** (`#{word(finding['operator'])}`)#{subject}: this change did not make any test fail."
|
|
142
|
+
end
|
|
143
|
+
|
|
139
144
|
def word(text)
|
|
140
145
|
text.to_s.gsub(/[^\w:.-]/, "")[0, 40]
|
|
141
146
|
end
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
# SPDX-License-Identifier: MIT
|
|
2
2
|
# frozen_string_literal: true
|
|
3
3
|
|
|
4
|
+
require_relative "wording"
|
|
4
5
|
require_relative "../score"
|
|
5
6
|
|
|
6
7
|
module Saboteur
|
|
@@ -62,7 +63,7 @@ module Saboteur
|
|
|
62
63
|
detected: breakdown.detected, tested: breakdown.tested, untested: untested }
|
|
63
64
|
end
|
|
64
65
|
|
|
65
|
-
# "cart.rb 100.0% (2/2) · n/a - 2 subjects
|
|
66
|
+
# "cart.rb 100.0% (2/2) · n/a - 2 subjects - no test executes them: ...": the
|
|
66
67
|
# tested score and the untested count stand next to each other; a
|
|
67
68
|
# file with nothing tested gets no percentage at all.
|
|
68
69
|
def line(row, width)
|
|
@@ -74,18 +75,18 @@ module Saboteur
|
|
|
74
75
|
note ? "#{base} #{@palette.glyph(:middot)} #{note}" : base
|
|
75
76
|
end
|
|
76
77
|
|
|
77
|
-
# "n/a - 2 subjects
|
|
78
|
-
# the subjects the score cannot speak for, capped so a long
|
|
79
|
-
# not swallow the row; the Not scored block lists them all.
|
|
78
|
+
# "n/a - 2 subjects - no test executes them: Cart#total, Cart#biggest":
|
|
79
|
+
# names the subjects the score cannot speak for, capped so a long
|
|
80
|
+
# list does not swallow the row; the Not scored block lists them all.
|
|
81
|
+
# Wording owns the claim (#1145): a stubbed-away subject has tests,
|
|
82
|
+
# they just never execute it.
|
|
80
83
|
def untested_note(untested)
|
|
81
84
|
return if untested.empty?
|
|
82
85
|
|
|
83
86
|
names = untested.map(&:id)
|
|
84
87
|
shown = names.first(MAX_UNTESTED_NAMES)
|
|
85
88
|
shown << "+#{names.size - MAX_UNTESTED_NAMES} more" if names.size > MAX_UNTESTED_NAMES
|
|
86
|
-
|
|
87
|
-
@palette.paint(:warn, "n/a - #{count} subject#{'s' unless count == 1} " \
|
|
88
|
-
"#{count == 1 ? 'has' : 'have'} no tests: #{shown.join(', ')}")
|
|
89
|
+
@palette.paint(:warn, "n/a - #{Wording.untested_note(untested.size)}: #{shown.join(', ')}")
|
|
89
90
|
end
|
|
90
91
|
end
|
|
91
92
|
end
|
|
@@ -41,7 +41,16 @@ module Saboteur
|
|
|
41
41
|
warnings: report.warnings,
|
|
42
42
|
framework: { name: "saboteur", version: VERSION },
|
|
43
43
|
saboteur: saboteur_enrichment(report, exit_cause, score_gates)
|
|
44
|
-
}.merge(report
|
|
44
|
+
}.merge(report_config(report))
|
|
45
|
+
end
|
|
46
|
+
|
|
47
|
+
# The run's recorded settings: the seed, and the --since ref of a diff
|
|
48
|
+
# run, so a consumer can tell a diff-scoped score from a project one.
|
|
49
|
+
def self.report_config(report)
|
|
50
|
+
config = {}
|
|
51
|
+
config[:seed] = report.seed if report.seed
|
|
52
|
+
config[:since] = report.since if report.diff? && report.since
|
|
53
|
+
config.empty? ? {} : { config: config }
|
|
45
54
|
end
|
|
46
55
|
|
|
47
56
|
# saboteur-namespaced keys beyond Stryker's schema: findings and
|
|
@@ -110,6 +119,7 @@ module Saboteur
|
|
|
110
119
|
def self.mutant(result, source, test_ids)
|
|
111
120
|
entry = {
|
|
112
121
|
id: result.digest,
|
|
122
|
+
subject: result.subject.id,
|
|
113
123
|
mutatorName: result.mutation.operator,
|
|
114
124
|
replacement: result.mutation.edits.map(&:replacement).join(" | "),
|
|
115
125
|
location: location(result.mutation.edits.first, source),
|
|
@@ -2,6 +2,7 @@
|
|
|
2
2
|
# frozen_string_literal: true
|
|
3
3
|
|
|
4
4
|
require_relative "../engine/bytecode_equivalence"
|
|
5
|
+
require_relative "gap_kinds"
|
|
5
6
|
require_relative "../execution/external_runner"
|
|
6
7
|
require_relative "../mutation_source/builtin"
|
|
7
8
|
|
|
@@ -21,11 +22,14 @@ module Saboteur
|
|
|
21
22
|
# arid - every line an edit touches matches an arid pattern
|
|
22
23
|
# (logging and friends, #111/#187). The builtin hint list decides;
|
|
23
24
|
# a configured arid policy does not change the label.
|
|
24
|
-
#
|
|
25
|
-
#
|
|
26
|
-
#
|
|
27
|
-
#
|
|
28
|
-
#
|
|
25
|
+
# A gap kind (boundary, branch, effect, number, error path, or the
|
|
26
|
+
# result unchecked fallback) - a test covered the line and passed,
|
|
27
|
+
# which MAY mean the suite reaches the code without pinning this
|
|
28
|
+
# behaviour; the kind names WHAT sort of check is missing, from the
|
|
29
|
+
# operator and the edit shape only. The covering list is only the
|
|
30
|
+
# PLANNED coverage: whole-body (coarse) mutants carry method-level
|
|
31
|
+
# guesses, so they never get a gap kind, and neither does anything
|
|
32
|
+
# without real coverage.
|
|
29
33
|
# tool_limit - survivors saboteur could not observe faithfully: an
|
|
30
34
|
# external-runner run records only the pseudo-test id, never real
|
|
31
35
|
# coverage (#1071). A re-eval refusal records no signal on the
|
|
@@ -33,7 +37,7 @@ module Saboteur
|
|
|
33
37
|
#
|
|
34
38
|
# A survivor with no matching signal stays unlabeled.
|
|
35
39
|
module SurvivorClassification
|
|
36
|
-
LABELS = %w[likely_equivalent arid
|
|
40
|
+
LABELS = %w[likely_equivalent arid tool_limit].concat(GapKinds::KINDS).freeze
|
|
37
41
|
|
|
38
42
|
class << self
|
|
39
43
|
def label(result, arid: MutationSource::Builtin::DEFAULT_ARID)
|
|
@@ -41,7 +45,7 @@ module Saboteur
|
|
|
41
45
|
return "likely_equivalent" if equivalent?(result)
|
|
42
46
|
return "arid" if arid?(result, arid)
|
|
43
47
|
return "tool_limit" if tool_limited?(result)
|
|
44
|
-
return
|
|
48
|
+
return gap_kind(result) if possible_test_gap?(result)
|
|
45
49
|
|
|
46
50
|
nil
|
|
47
51
|
end
|
|
@@ -66,11 +70,25 @@ module Saboteur
|
|
|
66
70
|
|
|
67
71
|
# An external runner reports one pseudo id for every mutant and no
|
|
68
72
|
# coverage map, so a survivor there was never observed against real
|
|
69
|
-
# tests. Pseudo id next to a real id keeps
|
|
73
|
+
# tests. Pseudo id next to a real id keeps the gap kind (#305).
|
|
70
74
|
def tool_limited?(result)
|
|
71
75
|
result.covered_by.any? && result.covered_by.all? { |id| id == Execution::ExternalRunner::PSEUDO_TEST }
|
|
72
76
|
end
|
|
73
77
|
|
|
78
|
+
# The gap kind says which sort of check the survivor suggests:
|
|
79
|
+
# the operator decides for almost everything; operand_replacement
|
|
80
|
+
# looks at the replaced expression (comparison -> boundary).
|
|
81
|
+
def gap_kind(result)
|
|
82
|
+
GapKinds.kind(result.mutation.operator, comparison_edit: comparison_edit?(result))
|
|
83
|
+
end
|
|
84
|
+
|
|
85
|
+
def comparison_edit?(result)
|
|
86
|
+
result.mutation.edits.any? do |edit|
|
|
87
|
+
result.subject.file_source.byteslice(edit.start_byte...edit.end_byte)
|
|
88
|
+
&.match?(/<=>|<=|>=|==|!=|===|<|>/)
|
|
89
|
+
end
|
|
90
|
+
end
|
|
91
|
+
|
|
74
92
|
# covered_by is a PLANNED covering list, not a verdict about what
|
|
75
93
|
# the suite asserts: coarse (whole-body) mutants carry method-level
|
|
76
94
|
# guesses, and a --test-command run records only the pseudo-test id
|
|
@@ -1,6 +1,8 @@
|
|
|
1
1
|
# SPDX-License-Identifier: MIT
|
|
2
2
|
# frozen_string_literal: true
|
|
3
3
|
|
|
4
|
+
require_relative "gap_kinds"
|
|
5
|
+
|
|
4
6
|
module Saboteur
|
|
5
7
|
module Reporter
|
|
6
8
|
# Suggests a check that might kill a surviving mutant, by operator. It does
|
|
@@ -94,9 +96,8 @@ module Saboteur
|
|
|
94
96
|
LABEL_REASONS = {
|
|
95
97
|
"likely_equivalent" => "compiles to the same instructions as the original",
|
|
96
98
|
"arid" => "every edited line matches an arid pattern",
|
|
97
|
-
"missing_test" => "the covering specs pass without pinning this change",
|
|
98
99
|
"tool_limit" => "the tool could not observe this mutant faithfully"
|
|
99
|
-
}.freeze
|
|
100
|
+
}.merge(GapKinds::REASONS).freeze
|
|
100
101
|
|
|
101
102
|
module_function
|
|
102
103
|
|
|
@@ -1,6 +1,8 @@
|
|
|
1
1
|
# SPDX-License-Identifier: MIT
|
|
2
2
|
# frozen_string_literal: true
|
|
3
3
|
|
|
4
|
+
require "shellwords"
|
|
5
|
+
|
|
4
6
|
require_relative "diff"
|
|
5
7
|
require_relative "mutation_text"
|
|
6
8
|
require_relative "survivor_classification"
|
|
@@ -17,6 +19,11 @@ module Saboteur
|
|
|
17
19
|
# --all-mutants gives every survivor its own block. Accepted
|
|
18
20
|
# (ignored) survivors get one quoted dim line each, never a block.
|
|
19
21
|
class SurvivorList
|
|
22
|
+
# Default when a caller never wires the whole-suite lookup (unit pins,
|
|
23
|
+
# other renderers): keep the historical bare covered-by line. Terminal
|
|
24
|
+
# always passes the kwarg - nil there means the lookup ran and found
|
|
25
|
+
# no cached whole-suite map (review C1).
|
|
26
|
+
NO_WHOLE_SUITE_LOOKUP = Object.new
|
|
20
27
|
# Rendered rows in the method fence, elision markers included.
|
|
21
28
|
FENCE_CAP = 10
|
|
22
29
|
|
|
@@ -26,11 +33,14 @@ module Saboteur
|
|
|
26
33
|
# Sibling digests named on the collapse line; the rest are a count.
|
|
27
34
|
SIBLING_DIGEST_CAP = 3
|
|
28
35
|
|
|
29
|
-
def initialize(out, palette, all_mutants: false, classifier: SurvivorClassification
|
|
36
|
+
def initialize(out, palette, all_mutants: false, classifier: SurvivorClassification,
|
|
37
|
+
whole_suite_map: NO_WHOLE_SUITE_LOOKUP, run_map: nil)
|
|
30
38
|
@out = out
|
|
31
39
|
@palette = palette
|
|
32
40
|
@all_mutants = all_mutants
|
|
33
41
|
@classifier = classifier
|
|
42
|
+
@whole_suite_map = whole_suite_map
|
|
43
|
+
@run_map = run_map
|
|
34
44
|
end
|
|
35
45
|
|
|
36
46
|
# Survivors in report (ranked) order; accepted ones trail as dim lines.
|
|
@@ -39,7 +49,8 @@ module Saboteur
|
|
|
39
49
|
|
|
40
50
|
live, accepted = survivors.partition(&:unignored_survivor?)
|
|
41
51
|
@out.puts @palette.paint(:head, "Survivors (#{live.size}) - changes your tests did not catch")
|
|
42
|
-
|
|
52
|
+
block_list = groups(live)
|
|
53
|
+
block_list.each_with_index { |group, index| block(report, group, index + 1, block_list.size) }
|
|
43
54
|
accepted_section(report, accepted)
|
|
44
55
|
hinted = live.select { |result| hint_for(result) }
|
|
45
56
|
@out.puts "\n#{@palette.paint(:dim, SurvivorHints::CAVEAT)}" if hinted.any?
|
|
@@ -70,9 +81,12 @@ module Saboteur
|
|
|
70
81
|
siblings(report, group)
|
|
71
82
|
end
|
|
72
83
|
|
|
84
|
+
# The count names blocks, the unit the leading ordinal uses: grouped
|
|
85
|
+
# siblings read "#3 of 14 blocks", never blocks against mutants (#1147).
|
|
73
86
|
def label_suffix(result, rank, total)
|
|
74
87
|
label = @classifier.label(result)
|
|
75
|
-
|
|
88
|
+
unit = total == 1 ? "block" : "blocks"
|
|
89
|
+
parts = [("coarse" if result.mutation.coarse), label, "##{rank} of #{total} #{unit}"].compact
|
|
76
90
|
reason = SurvivorHints.label_reason(label)
|
|
77
91
|
"[#{parts.join(" \u00B7 ")}#{": #{reason}" if reason}]"
|
|
78
92
|
end
|
|
@@ -176,8 +190,38 @@ module Saboteur
|
|
|
176
190
|
|
|
177
191
|
# The covering tests are a group union (#502): the first names where an
|
|
178
192
|
# assertion would pin the change, the count says the rest also passed.
|
|
193
|
+
#
|
|
194
|
+
# The list comes from the run's own map, which a scoped run narrows to
|
|
195
|
+
# the specs it loaded. The cached whole-suite map (when one exists)
|
|
196
|
+
# tells the rest of the story (review C1): a covering spec the run never
|
|
197
|
+
# loaded is named as not run here, and a spec pinning the mutant's exact
|
|
198
|
+
# line marks the block as likely covered elsewhere, instead of reading
|
|
199
|
+
# as a bare finding. With no whole-suite map, say so rather than
|
|
200
|
+
# implying the list is complete. A whole-project run's own map IS the
|
|
201
|
+
# whole-suite map, so its line is unchanged.
|
|
179
202
|
def covered(group)
|
|
180
203
|
tests = group.flat_map(&:covered_by).uniq
|
|
204
|
+
return legacy_covered(group, tests) if @whole_suite_map.equal?(NO_WHOLE_SUITE_LOOKUP)
|
|
205
|
+
|
|
206
|
+
whole = @whole_suite_map&.tests_for(group.first.subject)
|
|
207
|
+
if whole.nil?
|
|
208
|
+
@out.puts @palette.paint(:dim, " covering tests unknown - run a full coverage build")
|
|
209
|
+
return
|
|
210
|
+
end
|
|
211
|
+
|
|
212
|
+
run_known = @run_map ? @run_map.tests_for(group.first.subject) : tests
|
|
213
|
+
not_run = whole - run_known
|
|
214
|
+
return legacy_covered(group, tests) if not_run.empty?
|
|
215
|
+
|
|
216
|
+
@out.puts " covered by #{whole.size} #{whole.one? ? 'spec' : 'specs'}, " \
|
|
217
|
+
"incl. #{not_run.first} (not run here)"
|
|
218
|
+
exact = exact_line_tests(group.first) - run_known
|
|
219
|
+
return if exact.empty?
|
|
220
|
+
|
|
221
|
+
@out.puts @palette.paint(:dim, " likely covered elsewhere: #{exact.first} pins this line (not run here)")
|
|
222
|
+
end
|
|
223
|
+
|
|
224
|
+
def legacy_covered(group, tests)
|
|
181
225
|
return if tests.empty?
|
|
182
226
|
|
|
183
227
|
first = group.find { |result| result.covered_by.include?(tests.first) }
|
|
@@ -186,6 +230,14 @@ module Saboteur
|
|
|
186
230
|
@out.puts @palette.paint(:dim, " #{line}") if line
|
|
187
231
|
end
|
|
188
232
|
|
|
233
|
+
# Tests pinning the mutant's exact edit lines in the whole-suite map,
|
|
234
|
+
# on the same line basis the scheduler used to select the run's tests:
|
|
235
|
+
# the strongest "already pinned elsewhere" evidence a map can give.
|
|
236
|
+
def exact_line_tests(result)
|
|
237
|
+
lines = result.mutation.edits.flat_map { |edit| result.subject.coverage_lines_for(edit) }.uniq
|
|
238
|
+
@whole_suite_map.tests_for_edit(result.subject, lines)
|
|
239
|
+
end
|
|
240
|
+
|
|
189
241
|
# The verdict names what this run did: executed is nil when the run
|
|
190
242
|
# did not record which tests ran, so only the bare verdict is safe.
|
|
191
243
|
def more_covered_line(group, tests)
|
|
@@ -235,7 +287,9 @@ module Saboteur
|
|
|
235
287
|
overflow = rest.size - SIBLING_DIGEST_CAP
|
|
236
288
|
list = "#{list} (+#{overflow} more)" if overflow.positive?
|
|
237
289
|
@out.puts @palette.paint(:dim, " #{rest.size} more #{rest.one? ? 'mutant' : 'mutants'} here: #{list}")
|
|
238
|
-
|
|
290
|
+
words = ["saboteur", "run", "--all-mutants"]
|
|
291
|
+
words.push("--since", report.since) if report.diff? && report.since
|
|
292
|
+
@out.puts @palette.paint(:dim, " expand: #{Shellwords.join(words)}")
|
|
239
293
|
end
|
|
240
294
|
|
|
241
295
|
# Accepted (ignored) survivors stay visible but never earn a block:
|
|
@@ -5,19 +5,27 @@ module Saboteur
|
|
|
5
5
|
module Reporter
|
|
6
6
|
# Review priority, not a claim that a survivor is a real defect.
|
|
7
7
|
module SurvivorRanking
|
|
8
|
-
ORDERING = "
|
|
9
|
-
"
|
|
8
|
+
ORDERING = "surviving mutants per statement descending, then fewer covering tests, " \
|
|
9
|
+
"coarse first, larger edited-line span, nearer changed lines, " \
|
|
10
|
+
"then file, line and digest"
|
|
10
11
|
|
|
11
12
|
def self.ordered(report, results)
|
|
12
|
-
results.
|
|
13
|
+
sizes = results.group_by { |result| statement(report, result) }.transform_values(&:size)
|
|
14
|
+
results.sort_by { |result| key(report, result, sizes) }
|
|
13
15
|
end
|
|
14
16
|
|
|
15
|
-
def self.key(report, result)
|
|
16
|
-
[
|
|
17
|
+
def self.key(report, result, statement_sizes = {})
|
|
18
|
+
[-statement_sizes.fetch(statement(report, result), 0), result.covered_by.uniq.size,
|
|
19
|
+
result.mutation.coarse ? 0 : 1, -edited_lines(result).size,
|
|
17
20
|
report.changed_line_distance(result) || Float::INFINITY,
|
|
18
21
|
result.subject.file.delete_prefix("#{report.root}/"), first_line(result), result.digest]
|
|
19
22
|
end
|
|
20
23
|
|
|
24
|
+
# Statement identity: same subject at the same first mutated line.
|
|
25
|
+
def self.statement(report, result)
|
|
26
|
+
[result.subject.file.delete_prefix("#{report.root}/"), result.subject.id, first_line(result)]
|
|
27
|
+
end
|
|
28
|
+
|
|
21
29
|
def self.first_line(result)
|
|
22
30
|
result.subject.line_of(result.mutation.edits.map(&:start_byte).min)
|
|
23
31
|
end
|
|
@@ -2,15 +2,18 @@
|
|
|
2
2
|
# frozen_string_literal: true
|
|
3
3
|
|
|
4
4
|
require_relative "safe_out"
|
|
5
|
+
require_relative "wording"
|
|
5
6
|
require_relative "baseline_tail"
|
|
6
7
|
require_relative "next_steps"
|
|
7
8
|
require_relative "kill_causes"
|
|
8
9
|
require_relative "survivor_ranking"
|
|
9
10
|
require_relative "palette"
|
|
11
|
+
require_relative "notice"
|
|
10
12
|
require_relative "score_table"
|
|
11
13
|
require_relative "survivor_list"
|
|
12
14
|
require_relative "survivor_classification"
|
|
13
15
|
require_relative "test_matrix"
|
|
16
|
+
require_relative "reuse_note"
|
|
14
17
|
|
|
15
18
|
module Saboteur
|
|
16
19
|
module Reporter
|
|
@@ -25,17 +28,19 @@ module Saboteur
|
|
|
25
28
|
def initialize(out, verbose: false, quiet: false, color: :auto, overrides: nil,
|
|
26
29
|
replay_command: nil, all_mutants: false, exit_code: nil,
|
|
27
30
|
threshold: nil, previous_score: nil, baseline_gate: nil,
|
|
28
|
-
scope: "whole project",
|
|
31
|
+
scope: "whole project", baseline: nil, coverage: nil, mutant_phase: nil,
|
|
32
|
+
whole_suite_map: SurvivorList::NO_WHOLE_SUITE_LOOKUP, run_map: nil)
|
|
29
33
|
@out = SafeOut.new(out)
|
|
30
34
|
@all_mutants = all_mutants
|
|
31
35
|
@verbose = verbose
|
|
32
36
|
@quiet = quiet
|
|
33
|
-
# Clean-screen context: scope
|
|
37
|
+
# Clean-screen context: scope from the CLI; PR4a plumbs the phase stats.
|
|
34
38
|
@scope = scope
|
|
35
|
-
@html_report = html_report
|
|
36
39
|
@baseline = baseline
|
|
37
40
|
@coverage = coverage
|
|
38
41
|
@mutant_phase = mutant_phase
|
|
42
|
+
@whole_suite_map = whole_suite_map
|
|
43
|
+
@run_map = run_map
|
|
39
44
|
@palette = Palette.for(out, mode: color, overrides: overrides)
|
|
40
45
|
@score_table = ScoreTable.new(@out, @palette)
|
|
41
46
|
@next_steps = NextSteps.new(@out, @palette, replay_command: replay_command)
|
|
@@ -141,9 +146,9 @@ module Saboteur
|
|
|
141
146
|
# numbers so "4 mutants" is not read as the whole plan (#582).
|
|
142
147
|
def mutant_counts(report)
|
|
143
148
|
planned = report.early_stop&.planned_mutants
|
|
144
|
-
return
|
|
149
|
+
return Text.counted(report.results.size, "mutant") unless planned && planned > report.results.size
|
|
145
150
|
|
|
146
|
-
"#{planned}
|
|
151
|
+
"#{Text.counted(planned, 'mutant')} selected, #{report.results.size} attempted"
|
|
147
152
|
end
|
|
148
153
|
|
|
149
154
|
def summary(report)
|
|
@@ -151,7 +156,7 @@ module Saboteur
|
|
|
151
156
|
return clean_screen(report) if !@baseline_gate && clean_run?(report)
|
|
152
157
|
|
|
153
158
|
seed = report.seed && ", seed #{report.seed} (repeat with --seed #{report.seed})"
|
|
154
|
-
tally = "saboteur #{VERSION}: #{report.subjects.size}
|
|
159
|
+
tally = "saboteur #{VERSION}: #{Text.counted(report.subjects.size, 'subject')}, #{mutant_counts(report)}, " \
|
|
155
160
|
"wall #{format('%.2f', report.wall_time)} s#{seed}"
|
|
156
161
|
@out.puts @palette.paint(:dim, tally)
|
|
157
162
|
@out.puts report.shard.line if report.shard
|
|
@@ -183,8 +188,11 @@ module Saboteur
|
|
|
183
188
|
total = @palette.paint(:dim, "#{middot} #{format('%.1f', report.wall_time)}s total")
|
|
184
189
|
@out.puts "#{@palette.paint(:pass, win)} #{total}"
|
|
185
190
|
untested = untested_subjects(report)
|
|
186
|
-
@out.puts " #{@palette.paint(:warn, untested_win_note(untested))}" if untested.any?
|
|
187
|
-
|
|
191
|
+
@out.puts " #{@palette.paint(:warn, Wording.untested_win_note(untested))}" if untested.any?
|
|
192
|
+
ReuseNote.new(@out).print(report)
|
|
193
|
+
# The footer owns the report paths (#1187); the clean screen's own
|
|
194
|
+
# hint is the follow-up run a whole-project win suggests.
|
|
195
|
+
@out.puts " #{@palette.paint(:dim, 'next: saboteur run --since origin/main')}" if @scope == "whole project"
|
|
188
196
|
end
|
|
189
197
|
|
|
190
198
|
# The phase lines and the progress-dots line print only when their data
|
|
@@ -206,19 +214,6 @@ module Saboteur
|
|
|
206
214
|
"#{report.counts[:killed]}/#{report.executed.size} mutants killed"
|
|
207
215
|
end
|
|
208
216
|
|
|
209
|
-
def untested_win_note(untested)
|
|
210
|
-
ids = untested.map(&:id).join(", ")
|
|
211
|
-
"#{untested.size} subject#{'s' unless untested.one?} not scored - no tests exist: #{ids}"
|
|
212
|
-
end
|
|
213
|
-
|
|
214
|
-
def clean_reports_line
|
|
215
|
-
return unless @html_report
|
|
216
|
-
|
|
217
|
-
line = "reports: #{@html_report}"
|
|
218
|
-
line += " #{middot} next: saboteur run --since origin/main" if @scope == "whole project"
|
|
219
|
-
@out.puts " #{@palette.paint(:dim, line)}"
|
|
220
|
-
end
|
|
221
|
-
|
|
222
217
|
def hazard_suppressions(report)
|
|
223
218
|
return unless report.hazard_suppressions.positive?
|
|
224
219
|
|
|
@@ -247,22 +242,26 @@ module Saboteur
|
|
|
247
242
|
end
|
|
248
243
|
|
|
249
244
|
line = "Score: #{painted_score(breakdown.value)}#{headline_context(breakdown.value)}"
|
|
250
|
-
line <<
|
|
251
|
-
" - nothing tested"
|
|
252
|
-
else
|
|
253
|
-
" (#{breakdown.detected}/#{breakdown.tested} killed, tested subjects)"
|
|
254
|
-
end
|
|
245
|
+
line << score_parenthetical(breakdown)
|
|
255
246
|
if @verbose && breakdown.tested.positive?
|
|
256
247
|
line << "#{score_split(breakdown)} (killed+timeout)/(killed+survived+timeout); " \
|
|
257
248
|
"errors, resource limits and skips excluded"
|
|
258
249
|
end
|
|
259
250
|
if untested.any?
|
|
260
251
|
line << " #{@palette.paint(:warn, "#{untested.size} subject#{'s' unless untested.one?} " \
|
|
261
|
-
|
|
252
|
+
"not scored - #{Wording.no_execution(untested.size)}")}"
|
|
262
253
|
end
|
|
263
254
|
@out.puts line
|
|
264
255
|
end
|
|
265
256
|
|
|
257
|
+
# The share of tested mutants the run detected: "killed" alone would
|
|
258
|
+
# count timeouts too, so Wording names both when one is in the number.
|
|
259
|
+
def score_parenthetical(breakdown)
|
|
260
|
+
return " - nothing tested" if breakdown.tested.zero?
|
|
261
|
+
|
|
262
|
+
" (#{breakdown.detected}/#{breakdown.tested} #{Wording.detected_word(breakdown)}, tested subjects)"
|
|
263
|
+
end
|
|
264
|
+
|
|
266
265
|
# What the score is judged against and where it came from: the
|
|
267
266
|
# threshold it must meet, and the last score the history recorded.
|
|
268
267
|
def headline_context(value)
|
|
@@ -304,7 +303,7 @@ module Saboteur
|
|
|
304
303
|
return if untested.empty?
|
|
305
304
|
|
|
306
305
|
@out.puts
|
|
307
|
-
@out.puts @palette.paint(:warn, "Not scored - no
|
|
306
|
+
@out.puts @palette.paint(:warn, "Not scored - no test executes these (treat these as your biggest gaps):")
|
|
308
307
|
entries = untested.map { |subject| "#{subject.id} (#{subject.location(report.root)})" }
|
|
309
308
|
line = " #{@palette.paint(:warn, '!')} #{entries.first(MAX_NOT_SCORED).join(" #{middot} ")}"
|
|
310
309
|
line << " #{middot} ... #{entries.size - MAX_NOT_SCORED} more" if entries.size > MAX_NOT_SCORED
|
|
@@ -410,9 +409,7 @@ module Saboteur
|
|
|
410
409
|
|
|
411
410
|
def details(report)
|
|
412
411
|
report.skip_reasons.each { |reason, count| @out.puts skip_reason_line(reason, count) } if @verbose
|
|
413
|
-
|
|
414
|
-
@out.puts "reused #{reused} of #{report.results.size} verdicts from earlier runs (--incremental)" if
|
|
415
|
-
reused.positive?
|
|
412
|
+
ReuseNote.new(@out).print(report)
|
|
416
413
|
@out.puts "invalid mutations dropped (did not parse): #{report.invalid_mutations}" if
|
|
417
414
|
report.invalid_mutations.positive?
|
|
418
415
|
hazard_suppressions(report)
|
|
@@ -568,7 +565,7 @@ module Saboteur
|
|
|
568
565
|
# but are not printed a second time here.
|
|
569
566
|
def warnings(report)
|
|
570
567
|
report.warnings.each do |warning|
|
|
571
|
-
@out.puts "warning: #{warning}" unless report.live_announced.include?(warning)
|
|
568
|
+
@out.puts Notice.paint(@palette, "warning: #{warning}") unless report.live_announced.include?(warning)
|
|
572
569
|
end
|
|
573
570
|
end
|
|
574
571
|
|
|
@@ -596,7 +593,9 @@ module Saboteur
|
|
|
596
593
|
end
|
|
597
594
|
|
|
598
595
|
def survivors(report)
|
|
599
|
-
SurvivorList.new(@out, @palette, all_mutants: @all_mutants
|
|
596
|
+
SurvivorList.new(@out, @palette, all_mutants: @all_mutants,
|
|
597
|
+
whole_suite_map: @whole_suite_map, run_map: @run_map)
|
|
598
|
+
.call(report, ranked_survivors(report))
|
|
600
599
|
@out.puts unless report.survivors.empty?
|
|
601
600
|
end
|
|
602
601
|
|
|
@@ -0,0 +1,37 @@
|
|
|
1
|
+
# SPDX-License-Identifier: MIT
|
|
2
|
+
# frozen_string_literal: true
|
|
3
|
+
|
|
4
|
+
module Saboteur
|
|
5
|
+
module Reporter
|
|
6
|
+
# One wording for the score parenthetical and the not-scored note, so
|
|
7
|
+
# the terminal screen and the HTML page never drift (#1145, #1146).
|
|
8
|
+
module Wording
|
|
9
|
+
module_function
|
|
10
|
+
|
|
11
|
+
# detected counts timeouts, the tally does not: name both when a
|
|
12
|
+
# timeout is in the number, or "killed" means two things in adjacent
|
|
13
|
+
# lines (#1146).
|
|
14
|
+
def detected_word(breakdown)
|
|
15
|
+
breakdown.timeouts.positive? ? "killed or timed out" : "killed"
|
|
16
|
+
end
|
|
17
|
+
|
|
18
|
+
# A subject with no coverage may have tests that stub it away: the
|
|
19
|
+
# honest claim is about execution, not existence (#1145).
|
|
20
|
+
def no_execution(untested_count)
|
|
21
|
+
untested_count == 1 ? "no test executes it" : "no test executes them"
|
|
22
|
+
end
|
|
23
|
+
|
|
24
|
+
# The score table's per-file note fragment: count, subject noun and
|
|
25
|
+
# the execution claim in one shape (#1145).
|
|
26
|
+
def untested_note(count)
|
|
27
|
+
"#{count} subject#{'s' unless count == 1} - #{no_execution(count)}"
|
|
28
|
+
end
|
|
29
|
+
|
|
30
|
+
# The clean screen's one-line note for subjects no run scored.
|
|
31
|
+
def untested_win_note(untested)
|
|
32
|
+
ids = untested.map(&:id).join(", ")
|
|
33
|
+
"#{untested.size} subject#{'s' unless untested.one?} not scored - #{no_execution(untested.size)}: #{ids}"
|
|
34
|
+
end
|
|
35
|
+
end
|
|
36
|
+
end
|
|
37
|
+
end
|