saboteur 0.4.0 → 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +167 -0
- data/README.md +393 -20
- data/exe/saboteur +6 -2
- data/lib/saboteur/agent_skill.rb +6 -0
- data/lib/saboteur/baseline/acceptance.rb +218 -0
- data/lib/saboteur/baseline/command.rb +80 -0
- data/lib/saboteur/baseline/comparison.rb +105 -0
- data/lib/saboteur/baseline/completeness.rb +53 -0
- data/lib/saboteur/baseline/evidence.rb +114 -0
- data/lib/saboteur/baseline/identity.rb +78 -0
- data/lib/saboteur/baseline/manifest.rb +164 -0
- data/lib/saboteur/baseline/run_evidence.rb +117 -0
- data/lib/saboteur/baseline/workflow.rb +121 -0
- data/lib/saboteur/baseline_health.rb +1 -1
- data/lib/saboteur/cli/exit_status.rb +93 -0
- data/lib/saboteur/cli/run_reporting.rb +75 -0
- data/lib/saboteur/cli.rb +101 -121
- data/lib/saboteur/config.rb +76 -6
- data/lib/saboteur/doctor.rb +40 -8
- data/lib/saboteur/engine/baseline_guard.rb +33 -11
- data/lib/saboteur/engine/bytecode_equivalence.rb +95 -0
- data/lib/saboteur/engine/canary.rb +74 -0
- data/lib/saboteur/engine/candidate_selection.rb +18 -5
- data/lib/saboteur/engine/candidates.rb +59 -7
- data/lib/saboteur/engine/coverage_store.rb +34 -14
- data/lib/saboteur/engine/diagnostics.rb +150 -4
- data/lib/saboteur/engine/diff_scope.rb +10 -0
- data/lib/saboteur/engine/distribution.rb +54 -13
- data/lib/saboteur/engine/result_cache.rb +10 -5
- data/lib/saboteur/engine/run_fingerprint.rb +6 -4
- data/lib/saboteur/engine/sandbox_policy.rb +108 -0
- data/lib/saboteur/engine/scheduler.rb +92 -20
- data/lib/saboteur/engine/subject_selection.rb +24 -1
- data/lib/saboteur/engine/test_guess.rb +52 -0
- data/lib/saboteur/engine/unchanged_syntax.rb +87 -0
- data/lib/saboteur/engine/verdict_check.rb +7 -6
- data/lib/saboteur/engine.rb +192 -18
- data/lib/saboteur/execution/protocol.rb +2 -1
- data/lib/saboteur/execution/reevaluator.rb +72 -10
- data/lib/saboteur/execution/runner.rb +120 -17
- data/lib/saboteur/execution/sandbox.rb +271 -0
- data/lib/saboteur/execution/stale_captures.rb +43 -0
- data/lib/saboteur/execution/supervisor.rb +101 -9
- data/lib/saboteur/execution/worker_pool.rb +20 -4
- data/lib/saboteur/execution/wrapper.rb +45 -0
- data/lib/saboteur/gates.rb +83 -0
- data/lib/saboteur/help.rb +31 -7
- data/lib/saboteur/ignore_list.rb +5 -0
- data/lib/saboteur/inline_suppression.rb +65 -0
- data/lib/saboteur/integration/base.rb +1 -0
- data/lib/saboteur/integration/focus_guard.rb +83 -0
- data/lib/saboteur/integration/rspec.rb +21 -1
- data/lib/saboteur/mutation_source/builtin.rb +22 -3
- data/lib/saboteur/mutation_source/chain_removal.rb +80 -0
- data/lib/saboteur/mutation_source/condition_negation.rb +50 -0
- data/lib/saboteur/mutation_source/endpoint_replacement.rb +61 -0
- data/lib/saboteur/mutation_source/hazard_guard.rb +136 -0
- data/lib/saboteur/mutation_source/operator_assignment.rb +52 -0
- data/lib/saboteur/mutation_source/operator_registry.rb +57 -0
- data/lib/saboteur/mutation_source/range_replacement.rb +43 -0
- data/lib/saboteur/preview.rb +106 -0
- data/lib/saboteur/reporter/agent_commands.rb +7 -1
- data/lib/saboteur/reporter/agent_incomplete.rb +24 -1
- data/lib/saboteur/reporter/agent_summary.rb +11 -5
- data/lib/saboteur/reporter/findings.rb +4 -19
- data/lib/saboteur/reporter/history_paths.rb +38 -0
- data/lib/saboteur/reporter/history_record.rb +132 -0
- data/lib/saboteur/reporter/html.rb +12 -1
- data/lib/saboteur/reporter/mutation_text.rb +36 -0
- data/lib/saboteur/reporter/next_steps.rb +30 -0
- data/lib/saboteur/reporter/palette.rb +37 -0
- data/lib/saboteur/reporter/score_history.rb +123 -0
- data/lib/saboteur/reporter/score_table.rb +57 -0
- data/lib/saboteur/reporter/stryker.rb +15 -2
- data/lib/saboteur/reporter/survivor_hints.rb +68 -0
- data/lib/saboteur/reporter/survivor_list.rb +126 -0
- data/lib/saboteur/reporter/survivor_ranking.rb +34 -0
- data/lib/saboteur/reporter/terminal.rb +131 -79
- data/lib/saboteur/reporter/test_matrix.rb +57 -0
- data/lib/saboteur/result.rb +41 -6
- data/lib/saboteur/selection/coverage_map.rb +119 -13
- data/lib/saboteur/selection/git_diff.rb +28 -1
- data/lib/saboteur/selection/recorder.rb +55 -13
- data/lib/saboteur/subject.rb +40 -4
- data/lib/saboteur/subject_finder.rb +62 -12
- data/lib/saboteur/version.rb +1 -1
- data/lib/saboteur/working_tree.rb +8 -0
- data/lib/saboteur.rb +6 -0
- metadata +43 -2
|
@@ -0,0 +1,34 @@
|
|
|
1
|
+
# SPDX-License-Identifier: MIT
|
|
2
|
+
# frozen_string_literal: true
|
|
3
|
+
|
|
4
|
+
module Saboteur
|
|
5
|
+
module Reporter
|
|
6
|
+
# Review priority, not a claim that a survivor is a real defect.
|
|
7
|
+
module SurvivorRanking
|
|
8
|
+
ORDERING = "coarse first, then more covering tests, larger edited-line span, " \
|
|
9
|
+
"nearer changed lines, then file, line and digest"
|
|
10
|
+
|
|
11
|
+
def self.ordered(report, results)
|
|
12
|
+
results.sort_by { |result| key(report, result) }
|
|
13
|
+
end
|
|
14
|
+
|
|
15
|
+
def self.key(report, result)
|
|
16
|
+
[result.mutation.coarse ? 0 : 1, -result.covered_by.uniq.size, -edited_lines(result).size,
|
|
17
|
+
report.changed_line_distance(result) || Float::INFINITY,
|
|
18
|
+
result.subject.file.delete_prefix("#{report.root}/"), first_line(result), result.digest]
|
|
19
|
+
end
|
|
20
|
+
|
|
21
|
+
def self.first_line(result)
|
|
22
|
+
result.subject.line_of(result.mutation.edits.map(&:start_byte).min)
|
|
23
|
+
end
|
|
24
|
+
|
|
25
|
+
def self.edited_lines(result)
|
|
26
|
+
result.mutation.edits.flat_map do |edit|
|
|
27
|
+
first = result.subject.line_of(edit.start_byte)
|
|
28
|
+
last = result.subject.line_of([edit.end_byte - 1, edit.start_byte].max)
|
|
29
|
+
(first..last).to_a
|
|
30
|
+
end.uniq
|
|
31
|
+
end
|
|
32
|
+
end
|
|
33
|
+
end
|
|
34
|
+
end
|
|
@@ -1,35 +1,92 @@
|
|
|
1
1
|
# SPDX-License-Identifier: MIT
|
|
2
2
|
# frozen_string_literal: true
|
|
3
3
|
|
|
4
|
+
require_relative "next_steps"
|
|
5
|
+
require_relative "survivor_ranking"
|
|
6
|
+
require_relative "palette"
|
|
7
|
+
require_relative "score_table"
|
|
8
|
+
require_relative "survivor_list"
|
|
9
|
+
require_relative "test_matrix"
|
|
10
|
+
|
|
4
11
|
module Saboteur
|
|
5
12
|
module Reporter
|
|
6
|
-
# Plain-text
|
|
7
|
-
# errors, then a
|
|
13
|
+
# Plain-text report: survivors first (grouped by file, method and line),
|
|
14
|
+
# then errors and notes, then a short summary last so it stays on screen.
|
|
15
|
+
# Debug statistics (score split, timings, kill causes) need verbose.
|
|
8
16
|
class Terminal
|
|
9
17
|
MAX_ERRORS = 10
|
|
10
|
-
MAX_PASSING_TESTS =
|
|
11
|
-
MAX_CHANGED_LINES = 3
|
|
18
|
+
MAX_PASSING_TESTS = SurvivorList::MAX_PASSING_TESTS
|
|
12
19
|
|
|
13
|
-
def initialize(out)
|
|
20
|
+
def initialize(out, verbose: false, color: :auto)
|
|
14
21
|
@out = out
|
|
22
|
+
@verbose = verbose
|
|
23
|
+
@palette = Palette.for(out, mode: color)
|
|
24
|
+
@score_table = ScoreTable.new(out, @palette)
|
|
25
|
+
@next_steps = NextSteps.new(out, @palette)
|
|
15
26
|
end
|
|
16
27
|
|
|
17
28
|
def call(report)
|
|
18
|
-
|
|
29
|
+
survivors(report)
|
|
30
|
+
errors(report)
|
|
19
31
|
skipped_subjects(report)
|
|
32
|
+
outside_the_score(report)
|
|
33
|
+
inline_suppressions(report)
|
|
20
34
|
warnings(report)
|
|
21
|
-
|
|
22
|
-
|
|
35
|
+
early_stop(report)
|
|
36
|
+
test_matrix(report)
|
|
37
|
+
summary(report)
|
|
23
38
|
end
|
|
24
39
|
|
|
25
40
|
private
|
|
26
41
|
|
|
42
|
+
def test_matrix(report)
|
|
43
|
+
return unless report.matrix?
|
|
44
|
+
|
|
45
|
+
matrix = TestMatrix.new(report.results)
|
|
46
|
+
@out.puts "test matrix: #{matrix.tests.size} tests ran against mutants"
|
|
47
|
+
incomplete_matrix_note(matrix)
|
|
48
|
+
matrix_list("blind (covered mutants, killed none)", matrix.blind) do |test|
|
|
49
|
+
"#{matrix.mutants_covered(test)} covered"
|
|
50
|
+
end
|
|
51
|
+
matrix_list("redundant (every kill had another killer)", matrix.redundant) do |test|
|
|
52
|
+
"#{matrix.mutants_killed(test)} kills"
|
|
53
|
+
end
|
|
54
|
+
end
|
|
55
|
+
|
|
56
|
+
def incomplete_matrix_note(matrix)
|
|
57
|
+
return if matrix.incomplete.zero?
|
|
58
|
+
|
|
59
|
+
@out.puts " #{matrix.incomplete} mutants stopped before every covering test ran (incomplete): " \
|
|
60
|
+
"their tests get no blind or redundant verdict"
|
|
61
|
+
end
|
|
62
|
+
|
|
63
|
+
def matrix_list(title, tests)
|
|
64
|
+
@out.puts " #{title}: #{tests.size}"
|
|
65
|
+
tests.first(TestMatrix::MAX_LISTED).each { |test| @out.puts " #{test} #{yield(test)}" }
|
|
66
|
+
more = tests.size - TestMatrix::MAX_LISTED
|
|
67
|
+
@out.puts " ... #{more} more" if more.positive?
|
|
68
|
+
end
|
|
69
|
+
|
|
70
|
+
def inline_suppressions(report)
|
|
71
|
+
return if report.inline_suppressions.empty?
|
|
72
|
+
|
|
73
|
+
@out.puts("Inline suppressions: #{report.inline_suppressions.size} (outside the score)")
|
|
74
|
+
report.inline_suppressions.each do |entry|
|
|
75
|
+
path = entry[:file].delete_prefix("#{report.root}/")
|
|
76
|
+
@out.puts(" #{path}:#{entry[:line]} #{entry[:subject]} #{entry[:operator]} " \
|
|
77
|
+
"#{entry[:digest]}: #{entry[:reason]}")
|
|
78
|
+
end
|
|
79
|
+
end
|
|
80
|
+
|
|
27
81
|
def summary(report)
|
|
28
82
|
seed = report.seed && ", seed #{report.seed} (repeat with --seed #{report.seed})"
|
|
29
83
|
@out.puts "saboteur #{VERSION}: #{report.subjects.size} subjects, #{report.results.size} mutants, " \
|
|
30
84
|
"wall #{format('%.2f', report.wall_time)} s#{seed}"
|
|
31
85
|
@out.puts status_line(report)
|
|
32
86
|
details(report)
|
|
87
|
+
@score_table.call(report)
|
|
88
|
+
@out.puts @score_table.legend
|
|
89
|
+
@next_steps.call(report)
|
|
33
90
|
end
|
|
34
91
|
|
|
35
92
|
def status_line(report)
|
|
@@ -37,23 +94,43 @@ module Saboteur
|
|
|
37
94
|
survived = "survived #{c[:survived]}"
|
|
38
95
|
ignored = report.survivors.size - report.unignored_survivors.size
|
|
39
96
|
survived += " (#{ignored} ignored)" if ignored.positive?
|
|
40
|
-
|
|
41
|
-
|
|
97
|
+
line = "#{@palette.paint(:green,
|
|
98
|
+
"killed #{c[:killed]}")} #{@palette.paint(:red,
|
|
99
|
+
survived)} timeout #{c[:timeout]} " \
|
|
100
|
+
"resource limit #{c[:resource_limit]} error #{c[:error]} skipped #{c[:skipped]}"
|
|
101
|
+
line += " ran at load #{c[:ran_at_load]}" if c[:ran_at_load].positive?
|
|
102
|
+
line += " unreachable #{c[:unreachable]}" if c[:unreachable].positive?
|
|
103
|
+
line
|
|
104
|
+
end
|
|
105
|
+
|
|
106
|
+
def hazard_suppressions(report)
|
|
107
|
+
return unless report.hazard_suppressions.positive?
|
|
108
|
+
|
|
109
|
+
@out.puts "hazardous mutations suppressed: #{report.hazard_suppressions} " \
|
|
110
|
+
"(--allow-hazardous runs them anyway; the canary still watches the tree)"
|
|
42
111
|
end
|
|
43
112
|
|
|
44
113
|
def details(report)
|
|
45
114
|
breakdown = report.score_breakdown
|
|
46
|
-
@out.puts "score #{percent(breakdown.value)}
|
|
47
|
-
"
|
|
48
|
-
|
|
49
|
-
|
|
50
|
-
|
|
51
|
-
@out.puts "invalid mutations dropped (did not parse): #{report.invalid_mutations}" if
|
|
52
|
-
report.invalid_mutations.positive?
|
|
115
|
+
@out.puts "score #{percent(breakdown.value)}" \
|
|
116
|
+
"#{if @verbose
|
|
117
|
+
"#{score_split(breakdown)} (killed+timeout)/(killed+survived+timeout); " \
|
|
118
|
+
'errors, resource limits and skips excluded'
|
|
119
|
+
end}"
|
|
53
120
|
report.skip_reasons.each { |reason, count| @out.puts "skipped: #{reason}: #{count}" }
|
|
54
121
|
reused = report.results.count(&:reused?)
|
|
55
122
|
@out.puts "reused #{reused} of #{report.results.size} verdicts from earlier runs (--incremental)" if
|
|
56
123
|
reused.positive?
|
|
124
|
+
@out.puts "invalid mutations dropped (did not parse): #{report.invalid_mutations}" if
|
|
125
|
+
report.invalid_mutations.positive?
|
|
126
|
+
hazard_suppressions(report)
|
|
127
|
+
debug_details(report) if @verbose
|
|
128
|
+
end
|
|
129
|
+
|
|
130
|
+
def debug_details(report)
|
|
131
|
+
whole_body_score(report)
|
|
132
|
+
mean = report.mean_runtime
|
|
133
|
+
@out.puts "mean time per executed mutant #{format('%.3f', mean)} s" if mean
|
|
57
134
|
@out.puts "kills rejected by unchanged-method check: #{report.rejected_kills}"
|
|
58
135
|
kill_causes(report)
|
|
59
136
|
end
|
|
@@ -119,6 +196,37 @@ module Saboteur
|
|
|
119
196
|
end
|
|
120
197
|
end
|
|
121
198
|
|
|
199
|
+
# Mutants no test can judge, listed apart so they are not read as
|
|
200
|
+
# misses: code that only ran while the app loaded, and methods a
|
|
201
|
+
# captured Method object still calls in their original form.
|
|
202
|
+
def outside_the_score(report)
|
|
203
|
+
{ ran_at_load: "Ran at load (out of the score):",
|
|
204
|
+
unreachable: "Unreachable (out of the score):" }.each do |status, heading|
|
|
205
|
+
results = report.results.select { |result| result.status == status }
|
|
206
|
+
next if results.empty?
|
|
207
|
+
|
|
208
|
+
@out.puts heading
|
|
209
|
+
results.each do |result|
|
|
210
|
+
location = "#{relative(result.subject.file, report.root)}:#{result.subject.start_line}"
|
|
211
|
+
@out.puts " #{result.subject.id} #{result.mutation.operator} #{result.digest} #{location}"
|
|
212
|
+
end
|
|
213
|
+
@out.puts " hint: #{results.first.reason}" if results.first.reason
|
|
214
|
+
end
|
|
215
|
+
end
|
|
216
|
+
|
|
217
|
+
# The verdict covers only the subjects that ran, so say it where the
|
|
218
|
+
# score is read.
|
|
219
|
+
def early_stop(report)
|
|
220
|
+
stop = report.early_stop
|
|
221
|
+
return unless stop
|
|
222
|
+
|
|
223
|
+
@out.puts @palette.paint(:yellow, "stopped early (--fail-fast): #{stop.unreached.size} selected " \
|
|
224
|
+
"subject#{'s' unless stop.unreached.one?} did not run; the score below " \
|
|
225
|
+
"covers only the subjects that ran")
|
|
226
|
+
stop.unreached.first(MAX_ERRORS).each { |id| @out.puts " not run: #{id}" }
|
|
227
|
+
@out.puts " ... #{stop.unreached.size - MAX_ERRORS} more" if stop.unreached.size > MAX_ERRORS
|
|
228
|
+
end
|
|
229
|
+
|
|
122
230
|
def warnings(report)
|
|
123
231
|
report.warnings.each { |warning| @out.puts "warning: #{warning}" }
|
|
124
232
|
end
|
|
@@ -136,70 +244,14 @@ module Saboteur
|
|
|
136
244
|
end
|
|
137
245
|
|
|
138
246
|
def survivors(report)
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
@out.puts "\nSurvivors:"
|
|
142
|
-
shown(report).each do |result, hidden|
|
|
143
|
-
location = "#{relative(result.subject.file, report.root)}:#{result.subject.start_line}"
|
|
144
|
-
@out.puts "\n#{result.subject.id} #{result.mutation.operator} #{result.digest} #{location}"
|
|
145
|
-
changed_lines(report, result)
|
|
146
|
-
@out.puts "ignored: #{result.ignored}" if result.ignored
|
|
147
|
-
@out.puts result.diff(root: report.root)
|
|
148
|
-
passing_tests(result)
|
|
149
|
-
@out.puts "#{hidden} more survivor#{'s' if hidden > 1} on this line in the JSON report" if hidden.positive?
|
|
150
|
-
end
|
|
151
|
-
end
|
|
152
|
-
|
|
153
|
-
# The tests that ran with the mutant applied and still passed: the
|
|
154
|
-
# missing assertion belongs in one of them. They are listed likeliest
|
|
155
|
-
# killers first, the order the run used. Line coverage can overstate which
|
|
156
|
-
# tests exercise the line, so the list is a lead, not proof.
|
|
157
|
-
def passing_tests(result)
|
|
158
|
-
tests = result.covered_by
|
|
159
|
-
return if tests.empty?
|
|
160
|
-
|
|
161
|
-
@out.puts "#{tests.size} #{tests.one? ? 'test' : 'tests'} covered this line and #{passed_phrase(result)} " \
|
|
162
|
-
"(the missing check may belong in one of them):"
|
|
163
|
-
tests.first(MAX_PASSING_TESTS).each { |name| @out.puts " #{name}" }
|
|
164
|
-
more = tests.size - MAX_PASSING_TESTS
|
|
165
|
-
@out.puts " ... #{more} more in the JSON report (coveredBy)" if more.positive?
|
|
166
|
-
end
|
|
167
|
-
|
|
168
|
-
# A reused verdict was not run again: its tests passed in the earlier run.
|
|
169
|
-
def passed_phrase(result)
|
|
170
|
-
result.reused? ? "passed in an earlier run (--incremental)" : "still passed"
|
|
171
|
-
end
|
|
172
|
-
|
|
173
|
-
# In diff mode: the changed lines this mutant touches. The location above
|
|
174
|
-
# is where the mutated statement starts, which can be a line or two above
|
|
175
|
-
# the line that changed.
|
|
176
|
-
def changed_lines(report, result)
|
|
177
|
-
lines = report.changed_lines_for(result)
|
|
178
|
-
return if lines.empty?
|
|
179
|
-
|
|
180
|
-
path = relative(result.subject.file, report.root)
|
|
181
|
-
shown = lines.first(MAX_CHANGED_LINES).map { |line| "#{path}:#{line}" }.join(", ")
|
|
182
|
-
more = lines.size - MAX_CHANGED_LINES
|
|
183
|
-
@out.puts "changed #{lines.one? ? 'line' : 'lines'}: #{shown}#{" (+#{more} more)" if more.positive?}"
|
|
184
|
-
end
|
|
185
|
-
|
|
186
|
-
# [[result, how many more on the same line]]. In review mode survivors
|
|
187
|
-
# on one line usually share a fate, so one is shown: live before
|
|
188
|
-
# ignored, then coarse before fine, then the earliest edit.
|
|
189
|
-
def shown(report)
|
|
190
|
-
return report.survivors.map { |result| [result, 0] } unless report.review?
|
|
191
|
-
|
|
192
|
-
lines = report.survivors.group_by { |result| [result.subject.file, edit_line(result)] }
|
|
193
|
-
lines.values.map do |same_line|
|
|
194
|
-
best = same_line.min_by do |result|
|
|
195
|
-
[result.ignored ? 1 : 0, result.mutation.coarse ? 0 : 1, result.mutation.edits.first.start_byte]
|
|
196
|
-
end
|
|
197
|
-
[best, same_line.size - 1]
|
|
198
|
-
end
|
|
247
|
+
SurvivorList.new(@out, @palette).call(report, ranked_survivors(report))
|
|
248
|
+
@out.puts unless report.survivors.empty?
|
|
199
249
|
end
|
|
200
250
|
|
|
201
|
-
|
|
202
|
-
|
|
251
|
+
# Live survivors precede accepted ones; each group follows SurvivorRanking.
|
|
252
|
+
def ranked_survivors(report)
|
|
253
|
+
live, accepted = report.survivors.partition(&:unignored_survivor?)
|
|
254
|
+
[live, accepted].flat_map { |group| SurvivorRanking.ordered(report, group) }
|
|
203
255
|
end
|
|
204
256
|
|
|
205
257
|
def relative(file, root)
|
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
# SPDX-License-Identifier: MIT
|
|
2
|
+
# frozen_string_literal: true
|
|
3
|
+
|
|
4
|
+
module Saboteur
|
|
5
|
+
module Reporter
|
|
6
|
+
# What a --matrix run says about the tests themselves. Every covering test
|
|
7
|
+
# ran against every mutant, so each killed mutant lists all of its
|
|
8
|
+
# killers. A blind test covered mutants and killed none of them. A
|
|
9
|
+
# redundant test killed some, and every one of those had another killer.
|
|
10
|
+
#
|
|
11
|
+
# Only mutants whose every planned test is known to have run (see
|
|
12
|
+
# MutantResult#complete?) count. A run that stopped early or could not
|
|
13
|
+
# report what it ran is an incomplete mutant: the tests planned for it
|
|
14
|
+
# are unverified, and no blind or redundant verdict is given for them.
|
|
15
|
+
class TestMatrix
|
|
16
|
+
MAX_LISTED = 20
|
|
17
|
+
|
|
18
|
+
attr_reader :blind, :redundant, :tests, :incomplete
|
|
19
|
+
|
|
20
|
+
def initialize(results)
|
|
21
|
+
executed = results.select { |result| %i[killed survived].include?(result.status) }
|
|
22
|
+
complete, partial = executed.partition(&:complete?)
|
|
23
|
+
@incomplete = partial.size
|
|
24
|
+
@covered = Hash.new(0)
|
|
25
|
+
@killed = Hash.new { |hash, test| hash[test] = [] }
|
|
26
|
+
complete.each { |result| tally(result) }
|
|
27
|
+
@unverified = partial.flat_map(&:covered_by).uniq
|
|
28
|
+
@tests = @covered.keys.sort
|
|
29
|
+
judged = @tests - @unverified
|
|
30
|
+
@blind = judged.select { |test| @killed[test].empty? }
|
|
31
|
+
@redundant = judged.select { |test| redundant?(test) }
|
|
32
|
+
end
|
|
33
|
+
|
|
34
|
+
def mutants_covered(test)
|
|
35
|
+
@covered.fetch(test)
|
|
36
|
+
end
|
|
37
|
+
|
|
38
|
+
def mutants_killed(test)
|
|
39
|
+
@killed[test].size
|
|
40
|
+
end
|
|
41
|
+
|
|
42
|
+
private
|
|
43
|
+
|
|
44
|
+
def tally(result)
|
|
45
|
+
key = [result.subject.id, result.digest]
|
|
46
|
+
result.covered_by.each { |test| @covered[test] += 1 }
|
|
47
|
+
killers = result.killed_by & result.covered_by
|
|
48
|
+
killers.each { |test| @killed[test] << [key, killers.size - 1] }
|
|
49
|
+
end
|
|
50
|
+
|
|
51
|
+
def redundant?(test)
|
|
52
|
+
kills = @killed[test]
|
|
53
|
+
kills.any? && kills.all? { |_key, others| others.positive? }
|
|
54
|
+
end
|
|
55
|
+
end
|
|
56
|
+
end
|
|
57
|
+
end
|
data/lib/saboteur/result.rb
CHANGED
|
@@ -8,17 +8,21 @@ module Saboteur
|
|
|
8
8
|
# resource_limit: the mutant's process tree hit the memory or process cap.
|
|
9
9
|
# Like error it is outside the score: a mutant that eats all memory was
|
|
10
10
|
# stopped by saboteur, not detected by a test.
|
|
11
|
-
|
|
11
|
+
# ran_at_load: the mutated code only ran while the app loaded; no forked
|
|
12
|
+
# test can observe a change there, so it is outside the score.
|
|
13
|
+
# unreachable: a Method object captured before re-evaluation still calls
|
|
14
|
+
# the original body, so no test can observe the mutant. Outside the score.
|
|
15
|
+
STATUSES = %i[killed survived timeout resource_limit error skipped ran_at_load unreachable].freeze
|
|
12
16
|
|
|
13
17
|
# Outcome of one mutation. reason carries the skip reason, the error
|
|
14
18
|
# message or the timeout limit. ignored is the ignore-list reason, if any.
|
|
15
19
|
class MutantResult
|
|
16
20
|
attr_reader :subject, :mutation, :status, :reason, :runtime, :killed_by, :covered_by, :mutated_source,
|
|
17
|
-
:kill_cause
|
|
21
|
+
:kill_cause, :executed
|
|
18
22
|
attr_accessor :ignored
|
|
19
23
|
|
|
20
24
|
def initialize(subject:, mutation:, status:, mutated_source:, reason: nil, runtime: nil, killed_by: [],
|
|
21
|
-
covered_by: [], kill_cause: nil, reused: false, rejected_kill: false)
|
|
25
|
+
covered_by: [], kill_cause: nil, reused: false, rejected_kill: false, executed: nil)
|
|
22
26
|
raise ArgumentError, "unknown status #{status.inspect}" unless STATUSES.include?(status)
|
|
23
27
|
|
|
24
28
|
@subject = subject
|
|
@@ -30,6 +34,7 @@ module Saboteur
|
|
|
30
34
|
@covered_by = covered_by
|
|
31
35
|
@mutated_source = mutated_source
|
|
32
36
|
@kill_cause = kill_cause
|
|
37
|
+
@executed = executed
|
|
33
38
|
@rejected_kill = rejected_kill
|
|
34
39
|
@reused = reused
|
|
35
40
|
@ignored = nil
|
|
@@ -39,7 +44,14 @@ module Saboteur
|
|
|
39
44
|
def as_error(reason, rejected_kill: false)
|
|
40
45
|
MutantResult.new(subject: subject, mutation: mutation, status: :error, mutated_source: mutated_source,
|
|
41
46
|
reason: reason, runtime: runtime, killed_by: killed_by, covered_by: covered_by,
|
|
42
|
-
rejected_kill: rejected_kill)
|
|
47
|
+
executed: executed, rejected_kill: rejected_kill)
|
|
48
|
+
end
|
|
49
|
+
|
|
50
|
+
# True when every test planned for this mutant is known to have run.
|
|
51
|
+
# A killed run that stopped early, a test that called exit, or a verdict
|
|
52
|
+
# whose executed tests were never recorded is not complete.
|
|
53
|
+
def complete?
|
|
54
|
+
!executed.nil? && (covered_by - executed).empty?
|
|
43
55
|
end
|
|
44
56
|
|
|
45
57
|
def rejected_kill?
|
|
@@ -88,18 +100,25 @@ module Saboteur
|
|
|
88
100
|
end
|
|
89
101
|
end
|
|
90
102
|
|
|
103
|
+
# A run that stopped before every selected subject was mutated (--fail-fast).
|
|
104
|
+
# unreached lists the subject ids that never ran; the verdict covers only
|
|
105
|
+
# the subjects that did.
|
|
106
|
+
EarlyStop = Data.define(:reason, :unreached)
|
|
107
|
+
|
|
91
108
|
# Everything one run produced.
|
|
92
109
|
class Report
|
|
93
110
|
# More than half of the executed mutants errored: the run tested
|
|
94
111
|
# almost nothing and must not pass a gate.
|
|
95
112
|
MAX_ERROR_FRACTION = 0.5
|
|
96
113
|
|
|
97
|
-
attr_reader :results, :subjects, :wall_time, :invalid_mutations, :warnings, :root, :seed, :skipped_subjects
|
|
114
|
+
attr_reader :results, :subjects, :wall_time, :invalid_mutations, :warnings, :root, :seed, :skipped_subjects,
|
|
115
|
+
:generation_events, :omissions, :run_evidence, :hazard_suppressions, :early_stop, :inline_suppressions
|
|
98
116
|
|
|
99
117
|
# REVIEW: diff mode; the terminal then shows one survivor per line.
|
|
100
118
|
# seed: the random seed every test run started from (nil when unknown).
|
|
101
119
|
def initialize(results:, subjects:, wall_time:, root:, invalid_mutations: 0, warnings: [], review: false,
|
|
102
|
-
seed: nil, skipped_subjects: {}, diff_scope: nil
|
|
120
|
+
seed: nil, skipped_subjects: {}, diff_scope: nil, generation_events: [], omissions: [],
|
|
121
|
+
run_evidence: nil, hazard_suppressions: 0, early_stop: nil, matrix: false, inline_suppressions: [])
|
|
103
122
|
@results = results
|
|
104
123
|
@subjects = subjects
|
|
105
124
|
@wall_time = wall_time
|
|
@@ -110,6 +129,18 @@ module Saboteur
|
|
|
110
129
|
@seed = seed
|
|
111
130
|
@skipped_subjects = skipped_subjects
|
|
112
131
|
@diff_scope = diff_scope
|
|
132
|
+
@generation_events = generation_events.dup.freeze
|
|
133
|
+
@omissions = omissions.dup.freeze
|
|
134
|
+
@run_evidence = run_evidence
|
|
135
|
+
@hazard_suppressions = hazard_suppressions
|
|
136
|
+
@early_stop = early_stop
|
|
137
|
+
@matrix = matrix
|
|
138
|
+
@inline_suppressions = inline_suppressions.dup.freeze
|
|
139
|
+
end
|
|
140
|
+
|
|
141
|
+
# A --matrix run: every covering test ran per mutant.
|
|
142
|
+
def matrix?
|
|
143
|
+
@matrix
|
|
113
144
|
end
|
|
114
145
|
|
|
115
146
|
# In diff mode, the changed lines (sorted) that a mutant touches; empty
|
|
@@ -119,6 +150,10 @@ module Saboteur
|
|
|
119
150
|
@diff_scope ? @diff_scope.changed_lines_for(result.subject, result.mutation) : []
|
|
120
151
|
end
|
|
121
152
|
|
|
153
|
+
def changed_line_distance(result)
|
|
154
|
+
@diff_scope&.distance_for(result.subject, result.mutation)
|
|
155
|
+
end
|
|
156
|
+
|
|
122
157
|
def rejected_kills
|
|
123
158
|
results.count(&:rejected_kill?)
|
|
124
159
|
end
|
|
@@ -16,21 +16,87 @@ module Saboteur
|
|
|
16
16
|
# to every test in the file of the next test that starts. Each test's
|
|
17
17
|
# duration is recorded so selected tests can run fastest first.
|
|
18
18
|
class CoverageMap
|
|
19
|
-
FORMAT =
|
|
19
|
+
FORMAT = 8
|
|
20
20
|
|
|
21
|
-
|
|
21
|
+
RAN_AT_LOAD = "ran at load: the code ran while the app loaded; a forked test cannot observe a change here"
|
|
22
|
+
|
|
23
|
+
attr_reader :lines, :tests, :build_seconds, :durations, :methods, :load_lines, :branches
|
|
22
24
|
|
|
23
25
|
# lines: { "/abs/file.rb" => { 12 => ["id1", "id2"] } }
|
|
26
|
+
# methods: { "/abs/file.rb" => { 12 => ["id1", "id2"] } } keyed by the
|
|
27
|
+
# def's first line; line coverage never counts an endless def's body,
|
|
28
|
+
# so methods carry those subjects (issue #107).
|
|
24
29
|
# durations: { "id1" => seconds }
|
|
25
|
-
def initialize(lines:, tests:, build_seconds: nil, durations: {})
|
|
30
|
+
def initialize(lines:, tests:, build_seconds: nil, durations: {}, methods: {}, load_lines: {}, branches: {})
|
|
26
31
|
@lines = lines
|
|
27
32
|
@tests = tests
|
|
28
33
|
@build_seconds = build_seconds
|
|
29
34
|
@durations = durations
|
|
35
|
+
@methods = methods
|
|
36
|
+
@load_lines = load_lines
|
|
37
|
+
@branches = branches
|
|
30
38
|
end
|
|
31
39
|
|
|
32
40
|
def tests_for(subject)
|
|
33
|
-
tests_for_lines(subject.file, subject.coverage_lines)
|
|
41
|
+
tests = tests_for_lines(subject.file, subject.coverage_lines)
|
|
42
|
+
return tests unless tests.empty?
|
|
43
|
+
|
|
44
|
+
by_method = methods[CoverageMap.canonical(subject.file)]
|
|
45
|
+
return [] unless by_method
|
|
46
|
+
|
|
47
|
+
span = subject.start_line..subject.end_line
|
|
48
|
+
ids = by_method.flat_map { |line, method_ids| span.cover?(line) ? method_ids : [] }.uniq
|
|
49
|
+
sort_test_ids(ids)
|
|
50
|
+
end
|
|
51
|
+
|
|
52
|
+
# Attach the parent's boot counters without losing the child's method coverage.
|
|
53
|
+
def with_load_lines(load_lines)
|
|
54
|
+
self.class.new(lines: lines, tests: tests, build_seconds: build_seconds, durations: durations,
|
|
55
|
+
methods: methods, load_lines: load_lines, branches: branches)
|
|
56
|
+
end
|
|
57
|
+
|
|
58
|
+
# A requested line ran at load but no test later hit it.
|
|
59
|
+
def ran_at_load?(file, line_numbers)
|
|
60
|
+
at_load = load_lines[CoverageMap.canonical(file)]
|
|
61
|
+
return false unless at_load
|
|
62
|
+
|
|
63
|
+
covered = lines[CoverageMap.canonical(file)] || {}
|
|
64
|
+
line_numbers.any? { |line| at_load.include?(line) && !covered.key?(line) }
|
|
65
|
+
end
|
|
66
|
+
|
|
67
|
+
# Tests for the lines of one edit. A branch no test executes keeps its
|
|
68
|
+
# "no coverage" skip, but a subject whose lines never record at all
|
|
69
|
+
# (an endless def) falls back to its method coverage (issue #107).
|
|
70
|
+
def tests_for_edit(subject, line_numbers)
|
|
71
|
+
tests = tests_for_lines(subject.file, line_numbers)
|
|
72
|
+
return tests unless tests.empty?
|
|
73
|
+
return [] unless tests_for_lines(subject.file, subject.coverage_lines).empty?
|
|
74
|
+
|
|
75
|
+
tests_for(subject)
|
|
76
|
+
end
|
|
77
|
+
|
|
78
|
+
# nil means Ruby supplied no arm evidence for this edit (including &&/||).
|
|
79
|
+
# An empty list means a known arm was never taken. Nested arms intersect.
|
|
80
|
+
def tests_for_branches(subject, edits)
|
|
81
|
+
arms = branches.fetch(CoverageMap.canonical(subject.file), {})
|
|
82
|
+
matching = edits.map do |edit|
|
|
83
|
+
first = position(subject.file_source, edit.start_byte)
|
|
84
|
+
last = position(subject.file_source, edit.end_byte)
|
|
85
|
+
selected = arms.select do |span, ids|
|
|
86
|
+
(span.first(2) <=> first) <= 0 && (last <=> span.last(2)) <= 0 &&
|
|
87
|
+
!ran_before_tests?(subject.file, span, ids)
|
|
88
|
+
end
|
|
89
|
+
selected.values.reduce { |ids, other| ids & other }
|
|
90
|
+
end
|
|
91
|
+
return nil if matching.include?(nil)
|
|
92
|
+
|
|
93
|
+
sort_test_ids(matching.flatten.uniq)
|
|
94
|
+
end
|
|
95
|
+
|
|
96
|
+
# True when neither boot nor tests recorded included code. Load-only
|
|
97
|
+
# evidence is valid, even though it supplies no judged mutation score.
|
|
98
|
+
def empty?
|
|
99
|
+
[lines, methods, load_lines].all? { |hits| hits.none? { |_file, values| values.any? } }
|
|
34
100
|
end
|
|
35
101
|
|
|
36
102
|
# Tests that executed any of the lines, fastest first (suite order breaks
|
|
@@ -46,19 +112,19 @@ module Saboteur
|
|
|
46
112
|
by_line = lines[CoverageMap.canonical(file)]
|
|
47
113
|
return [] unless by_line
|
|
48
114
|
|
|
49
|
-
|
|
50
|
-
order = test_order
|
|
51
|
-
ids.sort_by { |id| [durations.fetch(id, Float::INFINITY), order.fetch(id, Float::INFINITY)] }
|
|
115
|
+
sort_test_ids(line_numbers.flat_map { |line| by_line.fetch(line, []) }.uniq)
|
|
52
116
|
end
|
|
53
117
|
|
|
54
118
|
# Lines refer to tests by their index in tests, which keeps the cache
|
|
55
119
|
# small for suites with long test ids.
|
|
56
120
|
def to_json(*)
|
|
57
121
|
index = test_order
|
|
58
|
-
|
|
59
|
-
by_line.to_h { |line, ids| [line.to_s, ids.map { |id| index.fetch(id) }] }
|
|
122
|
+
encode = lambda do |hits|
|
|
123
|
+
hits.transform_values { |by_line| by_line.to_h { |line, ids| [line.to_s, ids.map { |id| index.fetch(id) }] } }
|
|
60
124
|
end
|
|
61
|
-
JSON.generate(format: FORMAT, tests: tests, build_seconds: build_seconds, lines:
|
|
125
|
+
JSON.generate(format: FORMAT, tests: tests, build_seconds: build_seconds, lines: encode.call(lines),
|
|
126
|
+
methods: encode.call(methods), load_lines: load_lines,
|
|
127
|
+
branches: encode_branches(index),
|
|
62
128
|
durations: tests.map { |id| durations[id]&.round(6) })
|
|
63
129
|
end
|
|
64
130
|
|
|
@@ -67,11 +133,22 @@ module Saboteur
|
|
|
67
133
|
raise Error, "coverage map format #{data['format']} is not #{FORMAT}" unless data["format"] == FORMAT
|
|
68
134
|
|
|
69
135
|
tests = data["tests"]
|
|
70
|
-
|
|
71
|
-
|
|
136
|
+
decode = lambda do |hits|
|
|
137
|
+
hits.transform_values do |by_line|
|
|
138
|
+
by_line.to_h { |line, indexes| [line.to_i, indexes.map { |i| tests.fetch(i) }] }
|
|
139
|
+
end
|
|
72
140
|
end
|
|
73
141
|
durations = tests.zip(data["durations"]).to_h.compact
|
|
74
|
-
|
|
142
|
+
load_lines = (data["load_lines"] || {}).transform_values { |list| list.map(&:to_i) }
|
|
143
|
+
new(lines: decode.call(data["lines"]), tests: tests, build_seconds: data["build_seconds"],
|
|
144
|
+
durations: durations, methods: decode.call(data["methods"] || {}), load_lines: load_lines,
|
|
145
|
+
branches: decode_branches(data.fetch("branches", {}), tests))
|
|
146
|
+
end
|
|
147
|
+
|
|
148
|
+
def self.decode_branches(branches, tests)
|
|
149
|
+
branches.transform_values do |arms|
|
|
150
|
+
arms.to_h { |span, ids| [span, ids.map { |id| tests.fetch(id) }] }
|
|
151
|
+
end
|
|
75
152
|
end
|
|
76
153
|
|
|
77
154
|
# Cache key over every input that can change which test runs which line.
|
|
@@ -105,6 +182,35 @@ module Saboteur
|
|
|
105
182
|
|
|
106
183
|
private
|
|
107
184
|
|
|
185
|
+
def encode_branches(index)
|
|
186
|
+
branches.transform_values do |arms|
|
|
187
|
+
arms.map { |span, ids| [span, ids.map { |id| index.fetch(id) }] }
|
|
188
|
+
end
|
|
189
|
+
end
|
|
190
|
+
|
|
191
|
+
# An arm whose body the app ran while loading (a def inside an if) has no
|
|
192
|
+
# per-test hits, yet tests ran the lines inside it. Only lines strictly
|
|
193
|
+
# between the arm's first and last line count: the boundary lines can
|
|
194
|
+
# also hold code outside the arm. An arm with no interior line, such as
|
|
195
|
+
# a one-line arm, cannot be told apart, so its empty list still means
|
|
196
|
+
# "not taken".
|
|
197
|
+
def ran_before_tests?(file, span, ids)
|
|
198
|
+
return false unless ids.empty?
|
|
199
|
+
|
|
200
|
+
covered = lines[CoverageMap.canonical(file)] || {}
|
|
201
|
+
((span[0] + 1)...span[2]).any? { |line| covered.key?(line) }
|
|
202
|
+
end
|
|
203
|
+
|
|
204
|
+
def position(source, offset)
|
|
205
|
+
prefix = source.byteslice(0, offset).b
|
|
206
|
+
[prefix.count("\n") + 1, offset - ((prefix.rindex("\n") || -1) + 1)]
|
|
207
|
+
end
|
|
208
|
+
|
|
209
|
+
def sort_test_ids(ids)
|
|
210
|
+
order = test_order
|
|
211
|
+
ids.sort_by { |id| [durations.fetch(id, Float::INFINITY), order.fetch(id, Float::INFINITY)] }
|
|
212
|
+
end
|
|
213
|
+
|
|
108
214
|
def test_order
|
|
109
215
|
@test_order ||= tests.each_with_index.to_h
|
|
110
216
|
end
|