saboteur 0.4.0 → 0.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (90) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +167 -0
  3. data/README.md +393 -20
  4. data/exe/saboteur +6 -2
  5. data/lib/saboteur/agent_skill.rb +6 -0
  6. data/lib/saboteur/baseline/acceptance.rb +218 -0
  7. data/lib/saboteur/baseline/command.rb +80 -0
  8. data/lib/saboteur/baseline/comparison.rb +105 -0
  9. data/lib/saboteur/baseline/completeness.rb +53 -0
  10. data/lib/saboteur/baseline/evidence.rb +114 -0
  11. data/lib/saboteur/baseline/identity.rb +78 -0
  12. data/lib/saboteur/baseline/manifest.rb +164 -0
  13. data/lib/saboteur/baseline/run_evidence.rb +117 -0
  14. data/lib/saboteur/baseline/workflow.rb +121 -0
  15. data/lib/saboteur/baseline_health.rb +1 -1
  16. data/lib/saboteur/cli/exit_status.rb +93 -0
  17. data/lib/saboteur/cli/run_reporting.rb +75 -0
  18. data/lib/saboteur/cli.rb +101 -121
  19. data/lib/saboteur/config.rb +76 -6
  20. data/lib/saboteur/doctor.rb +40 -8
  21. data/lib/saboteur/engine/baseline_guard.rb +33 -11
  22. data/lib/saboteur/engine/bytecode_equivalence.rb +95 -0
  23. data/lib/saboteur/engine/canary.rb +74 -0
  24. data/lib/saboteur/engine/candidate_selection.rb +18 -5
  25. data/lib/saboteur/engine/candidates.rb +59 -7
  26. data/lib/saboteur/engine/coverage_store.rb +34 -14
  27. data/lib/saboteur/engine/diagnostics.rb +150 -4
  28. data/lib/saboteur/engine/diff_scope.rb +10 -0
  29. data/lib/saboteur/engine/distribution.rb +54 -13
  30. data/lib/saboteur/engine/result_cache.rb +10 -5
  31. data/lib/saboteur/engine/run_fingerprint.rb +6 -4
  32. data/lib/saboteur/engine/sandbox_policy.rb +108 -0
  33. data/lib/saboteur/engine/scheduler.rb +92 -20
  34. data/lib/saboteur/engine/subject_selection.rb +24 -1
  35. data/lib/saboteur/engine/test_guess.rb +52 -0
  36. data/lib/saboteur/engine/unchanged_syntax.rb +87 -0
  37. data/lib/saboteur/engine/verdict_check.rb +7 -6
  38. data/lib/saboteur/engine.rb +192 -18
  39. data/lib/saboteur/execution/protocol.rb +2 -1
  40. data/lib/saboteur/execution/reevaluator.rb +72 -10
  41. data/lib/saboteur/execution/runner.rb +120 -17
  42. data/lib/saboteur/execution/sandbox.rb +271 -0
  43. data/lib/saboteur/execution/stale_captures.rb +43 -0
  44. data/lib/saboteur/execution/supervisor.rb +101 -9
  45. data/lib/saboteur/execution/worker_pool.rb +20 -4
  46. data/lib/saboteur/execution/wrapper.rb +45 -0
  47. data/lib/saboteur/gates.rb +83 -0
  48. data/lib/saboteur/help.rb +31 -7
  49. data/lib/saboteur/ignore_list.rb +5 -0
  50. data/lib/saboteur/inline_suppression.rb +65 -0
  51. data/lib/saboteur/integration/base.rb +1 -0
  52. data/lib/saboteur/integration/focus_guard.rb +83 -0
  53. data/lib/saboteur/integration/rspec.rb +21 -1
  54. data/lib/saboteur/mutation_source/builtin.rb +22 -3
  55. data/lib/saboteur/mutation_source/chain_removal.rb +80 -0
  56. data/lib/saboteur/mutation_source/condition_negation.rb +50 -0
  57. data/lib/saboteur/mutation_source/endpoint_replacement.rb +61 -0
  58. data/lib/saboteur/mutation_source/hazard_guard.rb +136 -0
  59. data/lib/saboteur/mutation_source/operator_assignment.rb +52 -0
  60. data/lib/saboteur/mutation_source/operator_registry.rb +57 -0
  61. data/lib/saboteur/mutation_source/range_replacement.rb +43 -0
  62. data/lib/saboteur/preview.rb +106 -0
  63. data/lib/saboteur/reporter/agent_commands.rb +7 -1
  64. data/lib/saboteur/reporter/agent_incomplete.rb +24 -1
  65. data/lib/saboteur/reporter/agent_summary.rb +11 -5
  66. data/lib/saboteur/reporter/findings.rb +4 -19
  67. data/lib/saboteur/reporter/history_paths.rb +38 -0
  68. data/lib/saboteur/reporter/history_record.rb +132 -0
  69. data/lib/saboteur/reporter/html.rb +12 -1
  70. data/lib/saboteur/reporter/mutation_text.rb +36 -0
  71. data/lib/saboteur/reporter/next_steps.rb +30 -0
  72. data/lib/saboteur/reporter/palette.rb +37 -0
  73. data/lib/saboteur/reporter/score_history.rb +123 -0
  74. data/lib/saboteur/reporter/score_table.rb +57 -0
  75. data/lib/saboteur/reporter/stryker.rb +15 -2
  76. data/lib/saboteur/reporter/survivor_hints.rb +68 -0
  77. data/lib/saboteur/reporter/survivor_list.rb +126 -0
  78. data/lib/saboteur/reporter/survivor_ranking.rb +34 -0
  79. data/lib/saboteur/reporter/terminal.rb +131 -79
  80. data/lib/saboteur/reporter/test_matrix.rb +57 -0
  81. data/lib/saboteur/result.rb +41 -6
  82. data/lib/saboteur/selection/coverage_map.rb +119 -13
  83. data/lib/saboteur/selection/git_diff.rb +28 -1
  84. data/lib/saboteur/selection/recorder.rb +55 -13
  85. data/lib/saboteur/subject.rb +40 -4
  86. data/lib/saboteur/subject_finder.rb +62 -12
  87. data/lib/saboteur/version.rb +1 -1
  88. data/lib/saboteur/working_tree.rb +8 -0
  89. data/lib/saboteur.rb +6 -0
  90. metadata +43 -2
@@ -0,0 +1,34 @@
1
+ # SPDX-License-Identifier: MIT
2
+ # frozen_string_literal: true
3
+
4
+ module Saboteur
5
+ module Reporter
6
+ # Review priority, not a claim that a survivor is a real defect.
7
+ module SurvivorRanking
8
+ ORDERING = "coarse first, then more covering tests, larger edited-line span, " \
9
+ "nearer changed lines, then file, line and digest"
10
+
11
+ def self.ordered(report, results)
12
+ results.sort_by { |result| key(report, result) }
13
+ end
14
+
15
+ def self.key(report, result)
16
+ [result.mutation.coarse ? 0 : 1, -result.covered_by.uniq.size, -edited_lines(result).size,
17
+ report.changed_line_distance(result) || Float::INFINITY,
18
+ result.subject.file.delete_prefix("#{report.root}/"), first_line(result), result.digest]
19
+ end
20
+
21
+ def self.first_line(result)
22
+ result.subject.line_of(result.mutation.edits.map(&:start_byte).min)
23
+ end
24
+
25
+ def self.edited_lines(result)
26
+ result.mutation.edits.flat_map do |edit|
27
+ first = result.subject.line_of(edit.start_byte)
28
+ last = result.subject.line_of([edit.end_byte - 1, edit.start_byte].max)
29
+ (first..last).to_a
30
+ end.uniq
31
+ end
32
+ end
33
+ end
34
+ end
@@ -1,35 +1,92 @@
1
1
  # SPDX-License-Identifier: MIT
2
2
  # frozen_string_literal: true
3
3
 
4
+ require_relative "next_steps"
5
+ require_relative "survivor_ranking"
6
+ require_relative "palette"
7
+ require_relative "score_table"
8
+ require_relative "survivor_list"
9
+ require_relative "test_matrix"
10
+
4
11
  module Saboteur
5
12
  module Reporter
6
- # Plain-text summary: counts per status, score, timings, skip reasons,
7
- # errors, then a unified diff for each survivor.
13
+ # Plain-text report: survivors first (grouped by file, method and line),
14
+ # then errors and notes, then a short summary last so it stays on screen.
15
+ # Debug statistics (score split, timings, kill causes) need verbose.
8
16
  class Terminal
9
17
  MAX_ERRORS = 10
10
- MAX_PASSING_TESTS = 5
11
- MAX_CHANGED_LINES = 3
18
+ MAX_PASSING_TESTS = SurvivorList::MAX_PASSING_TESTS
12
19
 
13
- def initialize(out)
20
+ def initialize(out, verbose: false, color: :auto)
14
21
  @out = out
22
+ @verbose = verbose
23
+ @palette = Palette.for(out, mode: color)
24
+ @score_table = ScoreTable.new(out, @palette)
25
+ @next_steps = NextSteps.new(out, @palette)
15
26
  end
16
27
 
17
28
  def call(report)
18
- summary(report)
29
+ survivors(report)
30
+ errors(report)
19
31
  skipped_subjects(report)
32
+ outside_the_score(report)
33
+ inline_suppressions(report)
20
34
  warnings(report)
21
- errors(report)
22
- survivors(report)
35
+ early_stop(report)
36
+ test_matrix(report)
37
+ summary(report)
23
38
  end
24
39
 
25
40
  private
26
41
 
42
+ def test_matrix(report)
43
+ return unless report.matrix?
44
+
45
+ matrix = TestMatrix.new(report.results)
46
+ @out.puts "test matrix: #{matrix.tests.size} tests ran against mutants"
47
+ incomplete_matrix_note(matrix)
48
+ matrix_list("blind (covered mutants, killed none)", matrix.blind) do |test|
49
+ "#{matrix.mutants_covered(test)} covered"
50
+ end
51
+ matrix_list("redundant (every kill had another killer)", matrix.redundant) do |test|
52
+ "#{matrix.mutants_killed(test)} kills"
53
+ end
54
+ end
55
+
56
+ def incomplete_matrix_note(matrix)
57
+ return if matrix.incomplete.zero?
58
+
59
+ @out.puts " #{matrix.incomplete} mutants stopped before every covering test ran (incomplete): " \
60
+ "their tests get no blind or redundant verdict"
61
+ end
62
+
63
+ def matrix_list(title, tests)
64
+ @out.puts " #{title}: #{tests.size}"
65
+ tests.first(TestMatrix::MAX_LISTED).each { |test| @out.puts " #{test} #{yield(test)}" }
66
+ more = tests.size - TestMatrix::MAX_LISTED
67
+ @out.puts " ... #{more} more" if more.positive?
68
+ end
69
+
70
+ def inline_suppressions(report)
71
+ return if report.inline_suppressions.empty?
72
+
73
+ @out.puts("Inline suppressions: #{report.inline_suppressions.size} (outside the score)")
74
+ report.inline_suppressions.each do |entry|
75
+ path = entry[:file].delete_prefix("#{report.root}/")
76
+ @out.puts(" #{path}:#{entry[:line]} #{entry[:subject]} #{entry[:operator]} " \
77
+ "#{entry[:digest]}: #{entry[:reason]}")
78
+ end
79
+ end
80
+
27
81
  def summary(report)
28
82
  seed = report.seed && ", seed #{report.seed} (repeat with --seed #{report.seed})"
29
83
  @out.puts "saboteur #{VERSION}: #{report.subjects.size} subjects, #{report.results.size} mutants, " \
30
84
  "wall #{format('%.2f', report.wall_time)} s#{seed}"
31
85
  @out.puts status_line(report)
32
86
  details(report)
87
+ @score_table.call(report)
88
+ @out.puts @score_table.legend
89
+ @next_steps.call(report)
33
90
  end
34
91
 
35
92
  def status_line(report)
@@ -37,23 +94,43 @@ module Saboteur
37
94
  survived = "survived #{c[:survived]}"
38
95
  ignored = report.survivors.size - report.unignored_survivors.size
39
96
  survived += " (#{ignored} ignored)" if ignored.positive?
40
- "killed #{c[:killed]} #{survived} timeout #{c[:timeout]} resource limit #{c[:resource_limit]} " \
41
- "error #{c[:error]} skipped #{c[:skipped]}"
97
+ line = "#{@palette.paint(:green,
98
+ "killed #{c[:killed]}")} #{@palette.paint(:red,
99
+ survived)} timeout #{c[:timeout]} " \
100
+ "resource limit #{c[:resource_limit]} error #{c[:error]} skipped #{c[:skipped]}"
101
+ line += " ran at load #{c[:ran_at_load]}" if c[:ran_at_load].positive?
102
+ line += " unreachable #{c[:unreachable]}" if c[:unreachable].positive?
103
+ line
104
+ end
105
+
106
+ def hazard_suppressions(report)
107
+ return unless report.hazard_suppressions.positive?
108
+
109
+ @out.puts "hazardous mutations suppressed: #{report.hazard_suppressions} " \
110
+ "(--allow-hazardous runs them anyway; the canary still watches the tree)"
42
111
  end
43
112
 
44
113
  def details(report)
45
114
  breakdown = report.score_breakdown
46
- @out.puts "score #{percent(breakdown.value)}#{score_split(breakdown)} " \
47
- "(killed+timeout)/(killed+survived+timeout); errors, resource limits and skips excluded"
48
- whole_body_score(report)
49
- mean = report.mean_runtime
50
- @out.puts "mean time per executed mutant #{format('%.3f', mean)} s" if mean
51
- @out.puts "invalid mutations dropped (did not parse): #{report.invalid_mutations}" if
52
- report.invalid_mutations.positive?
115
+ @out.puts "score #{percent(breakdown.value)}" \
116
+ "#{if @verbose
117
+ "#{score_split(breakdown)} (killed+timeout)/(killed+survived+timeout); " \
118
+ 'errors, resource limits and skips excluded'
119
+ end}"
53
120
  report.skip_reasons.each { |reason, count| @out.puts "skipped: #{reason}: #{count}" }
54
121
  reused = report.results.count(&:reused?)
55
122
  @out.puts "reused #{reused} of #{report.results.size} verdicts from earlier runs (--incremental)" if
56
123
  reused.positive?
124
+ @out.puts "invalid mutations dropped (did not parse): #{report.invalid_mutations}" if
125
+ report.invalid_mutations.positive?
126
+ hazard_suppressions(report)
127
+ debug_details(report) if @verbose
128
+ end
129
+
130
+ def debug_details(report)
131
+ whole_body_score(report)
132
+ mean = report.mean_runtime
133
+ @out.puts "mean time per executed mutant #{format('%.3f', mean)} s" if mean
57
134
  @out.puts "kills rejected by unchanged-method check: #{report.rejected_kills}"
58
135
  kill_causes(report)
59
136
  end
@@ -119,6 +196,37 @@ module Saboteur
119
196
  end
120
197
  end
121
198
 
199
+ # Mutants no test can judge, listed apart so they are not read as
200
+ # misses: code that only ran while the app loaded, and methods a
201
+ # captured Method object still calls in their original form.
202
+ def outside_the_score(report)
203
+ { ran_at_load: "Ran at load (out of the score):",
204
+ unreachable: "Unreachable (out of the score):" }.each do |status, heading|
205
+ results = report.results.select { |result| result.status == status }
206
+ next if results.empty?
207
+
208
+ @out.puts heading
209
+ results.each do |result|
210
+ location = "#{relative(result.subject.file, report.root)}:#{result.subject.start_line}"
211
+ @out.puts " #{result.subject.id} #{result.mutation.operator} #{result.digest} #{location}"
212
+ end
213
+ @out.puts " hint: #{results.first.reason}" if results.first.reason
214
+ end
215
+ end
216
+
217
+ # The verdict covers only the subjects that ran, so say it where the
218
+ # score is read.
219
+ def early_stop(report)
220
+ stop = report.early_stop
221
+ return unless stop
222
+
223
+ @out.puts @palette.paint(:yellow, "stopped early (--fail-fast): #{stop.unreached.size} selected " \
224
+ "subject#{'s' unless stop.unreached.one?} did not run; the score below " \
225
+ "covers only the subjects that ran")
226
+ stop.unreached.first(MAX_ERRORS).each { |id| @out.puts " not run: #{id}" }
227
+ @out.puts " ... #{stop.unreached.size - MAX_ERRORS} more" if stop.unreached.size > MAX_ERRORS
228
+ end
229
+
122
230
  def warnings(report)
123
231
  report.warnings.each { |warning| @out.puts "warning: #{warning}" }
124
232
  end
@@ -136,70 +244,14 @@ module Saboteur
136
244
  end
137
245
 
138
246
  def survivors(report)
139
- return if report.survivors.empty?
140
-
141
- @out.puts "\nSurvivors:"
142
- shown(report).each do |result, hidden|
143
- location = "#{relative(result.subject.file, report.root)}:#{result.subject.start_line}"
144
- @out.puts "\n#{result.subject.id} #{result.mutation.operator} #{result.digest} #{location}"
145
- changed_lines(report, result)
146
- @out.puts "ignored: #{result.ignored}" if result.ignored
147
- @out.puts result.diff(root: report.root)
148
- passing_tests(result)
149
- @out.puts "#{hidden} more survivor#{'s' if hidden > 1} on this line in the JSON report" if hidden.positive?
150
- end
151
- end
152
-
153
- # The tests that ran with the mutant applied and still passed: the
154
- # missing assertion belongs in one of them. They are listed likeliest
155
- # killers first, the order the run used. Line coverage can overstate which
156
- # tests exercise the line, so the list is a lead, not proof.
157
- def passing_tests(result)
158
- tests = result.covered_by
159
- return if tests.empty?
160
-
161
- @out.puts "#{tests.size} #{tests.one? ? 'test' : 'tests'} covered this line and #{passed_phrase(result)} " \
162
- "(the missing check may belong in one of them):"
163
- tests.first(MAX_PASSING_TESTS).each { |name| @out.puts " #{name}" }
164
- more = tests.size - MAX_PASSING_TESTS
165
- @out.puts " ... #{more} more in the JSON report (coveredBy)" if more.positive?
166
- end
167
-
168
- # A reused verdict was not run again: its tests passed in the earlier run.
169
- def passed_phrase(result)
170
- result.reused? ? "passed in an earlier run (--incremental)" : "still passed"
171
- end
172
-
173
- # In diff mode: the changed lines this mutant touches. The location above
174
- # is where the mutated statement starts, which can be a line or two above
175
- # the line that changed.
176
- def changed_lines(report, result)
177
- lines = report.changed_lines_for(result)
178
- return if lines.empty?
179
-
180
- path = relative(result.subject.file, report.root)
181
- shown = lines.first(MAX_CHANGED_LINES).map { |line| "#{path}:#{line}" }.join(", ")
182
- more = lines.size - MAX_CHANGED_LINES
183
- @out.puts "changed #{lines.one? ? 'line' : 'lines'}: #{shown}#{" (+#{more} more)" if more.positive?}"
184
- end
185
-
186
- # [[result, how many more on the same line]]. In review mode survivors
187
- # on one line usually share a fate, so one is shown: live before
188
- # ignored, then coarse before fine, then the earliest edit.
189
- def shown(report)
190
- return report.survivors.map { |result| [result, 0] } unless report.review?
191
-
192
- lines = report.survivors.group_by { |result| [result.subject.file, edit_line(result)] }
193
- lines.values.map do |same_line|
194
- best = same_line.min_by do |result|
195
- [result.ignored ? 1 : 0, result.mutation.coarse ? 0 : 1, result.mutation.edits.first.start_byte]
196
- end
197
- [best, same_line.size - 1]
198
- end
247
+ SurvivorList.new(@out, @palette).call(report, ranked_survivors(report))
248
+ @out.puts unless report.survivors.empty?
199
249
  end
200
250
 
201
- def edit_line(result)
202
- result.subject.line_of(result.mutation.edits.first.start_byte)
251
+ # Live survivors precede accepted ones; each group follows SurvivorRanking.
252
+ def ranked_survivors(report)
253
+ live, accepted = report.survivors.partition(&:unignored_survivor?)
254
+ [live, accepted].flat_map { |group| SurvivorRanking.ordered(report, group) }
203
255
  end
204
256
 
205
257
  def relative(file, root)
@@ -0,0 +1,57 @@
1
+ # SPDX-License-Identifier: MIT
2
+ # frozen_string_literal: true
3
+
4
+ module Saboteur
5
+ module Reporter
6
+ # What a --matrix run says about the tests themselves. Every covering test
7
+ # ran against every mutant, so each killed mutant lists all of its
8
+ # killers. A blind test covered mutants and killed none of them. A
9
+ # redundant test killed some, and every one of those had another killer.
10
+ #
11
+ # Only mutants whose every planned test is known to have run (see
12
+ # MutantResult#complete?) count. A run that stopped early or could not
13
+ # report what it ran is an incomplete mutant: the tests planned for it
14
+ # are unverified, and no blind or redundant verdict is given for them.
15
+ class TestMatrix
16
+ MAX_LISTED = 20
17
+
18
+ attr_reader :blind, :redundant, :tests, :incomplete
19
+
20
+ def initialize(results)
21
+ executed = results.select { |result| %i[killed survived].include?(result.status) }
22
+ complete, partial = executed.partition(&:complete?)
23
+ @incomplete = partial.size
24
+ @covered = Hash.new(0)
25
+ @killed = Hash.new { |hash, test| hash[test] = [] }
26
+ complete.each { |result| tally(result) }
27
+ @unverified = partial.flat_map(&:covered_by).uniq
28
+ @tests = @covered.keys.sort
29
+ judged = @tests - @unverified
30
+ @blind = judged.select { |test| @killed[test].empty? }
31
+ @redundant = judged.select { |test| redundant?(test) }
32
+ end
33
+
34
+ def mutants_covered(test)
35
+ @covered.fetch(test)
36
+ end
37
+
38
+ def mutants_killed(test)
39
+ @killed[test].size
40
+ end
41
+
42
+ private
43
+
44
+ def tally(result)
45
+ key = [result.subject.id, result.digest]
46
+ result.covered_by.each { |test| @covered[test] += 1 }
47
+ killers = result.killed_by & result.covered_by
48
+ killers.each { |test| @killed[test] << [key, killers.size - 1] }
49
+ end
50
+
51
+ def redundant?(test)
52
+ kills = @killed[test]
53
+ kills.any? && kills.all? { |_key, others| others.positive? }
54
+ end
55
+ end
56
+ end
57
+ end
@@ -8,17 +8,21 @@ module Saboteur
8
8
  # resource_limit: the mutant's process tree hit the memory or process cap.
9
9
  # Like error it is outside the score: a mutant that eats all memory was
10
10
  # stopped by saboteur, not detected by a test.
11
- STATUSES = %i[killed survived timeout resource_limit error skipped].freeze
11
+ # ran_at_load: the mutated code only ran while the app loaded; no forked
12
+ # test can observe a change there, so it is outside the score.
13
+ # unreachable: a Method object captured before re-evaluation still calls
14
+ # the original body, so no test can observe the mutant. Outside the score.
15
+ STATUSES = %i[killed survived timeout resource_limit error skipped ran_at_load unreachable].freeze
12
16
 
13
17
  # Outcome of one mutation. reason carries the skip reason, the error
14
18
  # message or the timeout limit. ignored is the ignore-list reason, if any.
15
19
  class MutantResult
16
20
  attr_reader :subject, :mutation, :status, :reason, :runtime, :killed_by, :covered_by, :mutated_source,
17
- :kill_cause
21
+ :kill_cause, :executed
18
22
  attr_accessor :ignored
19
23
 
20
24
  def initialize(subject:, mutation:, status:, mutated_source:, reason: nil, runtime: nil, killed_by: [],
21
- covered_by: [], kill_cause: nil, reused: false, rejected_kill: false)
25
+ covered_by: [], kill_cause: nil, reused: false, rejected_kill: false, executed: nil)
22
26
  raise ArgumentError, "unknown status #{status.inspect}" unless STATUSES.include?(status)
23
27
 
24
28
  @subject = subject
@@ -30,6 +34,7 @@ module Saboteur
30
34
  @covered_by = covered_by
31
35
  @mutated_source = mutated_source
32
36
  @kill_cause = kill_cause
37
+ @executed = executed
33
38
  @rejected_kill = rejected_kill
34
39
  @reused = reused
35
40
  @ignored = nil
@@ -39,7 +44,14 @@ module Saboteur
39
44
  def as_error(reason, rejected_kill: false)
40
45
  MutantResult.new(subject: subject, mutation: mutation, status: :error, mutated_source: mutated_source,
41
46
  reason: reason, runtime: runtime, killed_by: killed_by, covered_by: covered_by,
42
- rejected_kill: rejected_kill)
47
+ executed: executed, rejected_kill: rejected_kill)
48
+ end
49
+
50
+ # True when every test planned for this mutant is known to have run.
51
+ # A killed run that stopped early, a test that called exit, or a verdict
52
+ # whose executed tests were never recorded is not complete.
53
+ def complete?
54
+ !executed.nil? && (covered_by - executed).empty?
43
55
  end
44
56
 
45
57
  def rejected_kill?
@@ -88,18 +100,25 @@ module Saboteur
88
100
  end
89
101
  end
90
102
 
103
+ # A run that stopped before every selected subject was mutated (--fail-fast).
104
+ # unreached lists the subject ids that never ran; the verdict covers only
105
+ # the subjects that did.
106
+ EarlyStop = Data.define(:reason, :unreached)
107
+
91
108
  # Everything one run produced.
92
109
  class Report
93
110
  # More than half of the executed mutants errored: the run tested
94
111
  # almost nothing and must not pass a gate.
95
112
  MAX_ERROR_FRACTION = 0.5
96
113
 
97
- attr_reader :results, :subjects, :wall_time, :invalid_mutations, :warnings, :root, :seed, :skipped_subjects
114
+ attr_reader :results, :subjects, :wall_time, :invalid_mutations, :warnings, :root, :seed, :skipped_subjects,
115
+ :generation_events, :omissions, :run_evidence, :hazard_suppressions, :early_stop, :inline_suppressions
98
116
 
99
117
  # REVIEW: diff mode; the terminal then shows one survivor per line.
100
118
  # seed: the random seed every test run started from (nil when unknown).
101
119
  def initialize(results:, subjects:, wall_time:, root:, invalid_mutations: 0, warnings: [], review: false,
102
- seed: nil, skipped_subjects: {}, diff_scope: nil)
120
+ seed: nil, skipped_subjects: {}, diff_scope: nil, generation_events: [], omissions: [],
121
+ run_evidence: nil, hazard_suppressions: 0, early_stop: nil, matrix: false, inline_suppressions: [])
103
122
  @results = results
104
123
  @subjects = subjects
105
124
  @wall_time = wall_time
@@ -110,6 +129,18 @@ module Saboteur
110
129
  @seed = seed
111
130
  @skipped_subjects = skipped_subjects
112
131
  @diff_scope = diff_scope
132
+ @generation_events = generation_events.dup.freeze
133
+ @omissions = omissions.dup.freeze
134
+ @run_evidence = run_evidence
135
+ @hazard_suppressions = hazard_suppressions
136
+ @early_stop = early_stop
137
+ @matrix = matrix
138
+ @inline_suppressions = inline_suppressions.dup.freeze
139
+ end
140
+
141
+ # A --matrix run: every covering test ran per mutant.
142
+ def matrix?
143
+ @matrix
113
144
  end
114
145
 
115
146
  # In diff mode, the changed lines (sorted) that a mutant touches; empty
@@ -119,6 +150,10 @@ module Saboteur
119
150
  @diff_scope ? @diff_scope.changed_lines_for(result.subject, result.mutation) : []
120
151
  end
121
152
 
153
+ def changed_line_distance(result)
154
+ @diff_scope&.distance_for(result.subject, result.mutation)
155
+ end
156
+
122
157
  def rejected_kills
123
158
  results.count(&:rejected_kill?)
124
159
  end
@@ -16,21 +16,87 @@ module Saboteur
16
16
  # to every test in the file of the next test that starts. Each test's
17
17
  # duration is recorded so selected tests can run fastest first.
18
18
  class CoverageMap
19
- FORMAT = 3
19
+ FORMAT = 8
20
20
 
21
- attr_reader :lines, :tests, :build_seconds, :durations
21
+ RAN_AT_LOAD = "ran at load: the code ran while the app loaded; a forked test cannot observe a change here"
22
+
23
+ attr_reader :lines, :tests, :build_seconds, :durations, :methods, :load_lines, :branches
22
24
 
23
25
  # lines: { "/abs/file.rb" => { 12 => ["id1", "id2"] } }
26
+ # methods: { "/abs/file.rb" => { 12 => ["id1", "id2"] } } keyed by the
27
+ # def's first line; line coverage never counts an endless def's body,
28
+ # so methods carry those subjects (issue #107).
24
29
  # durations: { "id1" => seconds }
25
- def initialize(lines:, tests:, build_seconds: nil, durations: {})
30
+ def initialize(lines:, tests:, build_seconds: nil, durations: {}, methods: {}, load_lines: {}, branches: {})
26
31
  @lines = lines
27
32
  @tests = tests
28
33
  @build_seconds = build_seconds
29
34
  @durations = durations
35
+ @methods = methods
36
+ @load_lines = load_lines
37
+ @branches = branches
30
38
  end
31
39
 
32
40
  def tests_for(subject)
33
- tests_for_lines(subject.file, subject.coverage_lines)
41
+ tests = tests_for_lines(subject.file, subject.coverage_lines)
42
+ return tests unless tests.empty?
43
+
44
+ by_method = methods[CoverageMap.canonical(subject.file)]
45
+ return [] unless by_method
46
+
47
+ span = subject.start_line..subject.end_line
48
+ ids = by_method.flat_map { |line, method_ids| span.cover?(line) ? method_ids : [] }.uniq
49
+ sort_test_ids(ids)
50
+ end
51
+
52
+ # Attach the parent's boot counters without losing the child's method coverage.
53
+ def with_load_lines(load_lines)
54
+ self.class.new(lines: lines, tests: tests, build_seconds: build_seconds, durations: durations,
55
+ methods: methods, load_lines: load_lines, branches: branches)
56
+ end
57
+
58
+ # A requested line ran at load but no test later hit it.
59
+ def ran_at_load?(file, line_numbers)
60
+ at_load = load_lines[CoverageMap.canonical(file)]
61
+ return false unless at_load
62
+
63
+ covered = lines[CoverageMap.canonical(file)] || {}
64
+ line_numbers.any? { |line| at_load.include?(line) && !covered.key?(line) }
65
+ end
66
+
67
+ # Tests for the lines of one edit. A branch no test executes keeps its
68
+ # "no coverage" skip, but a subject whose lines never record at all
69
+ # (an endless def) falls back to its method coverage (issue #107).
70
+ def tests_for_edit(subject, line_numbers)
71
+ tests = tests_for_lines(subject.file, line_numbers)
72
+ return tests unless tests.empty?
73
+ return [] unless tests_for_lines(subject.file, subject.coverage_lines).empty?
74
+
75
+ tests_for(subject)
76
+ end
77
+
78
+ # nil means Ruby supplied no arm evidence for this edit (including &&/||).
79
+ # An empty list means a known arm was never taken. Nested arms intersect.
80
+ def tests_for_branches(subject, edits)
81
+ arms = branches.fetch(CoverageMap.canonical(subject.file), {})
82
+ matching = edits.map do |edit|
83
+ first = position(subject.file_source, edit.start_byte)
84
+ last = position(subject.file_source, edit.end_byte)
85
+ selected = arms.select do |span, ids|
86
+ (span.first(2) <=> first) <= 0 && (last <=> span.last(2)) <= 0 &&
87
+ !ran_before_tests?(subject.file, span, ids)
88
+ end
89
+ selected.values.reduce { |ids, other| ids & other }
90
+ end
91
+ return nil if matching.include?(nil)
92
+
93
+ sort_test_ids(matching.flatten.uniq)
94
+ end
95
+
96
+ # True when neither boot nor tests recorded included code. Load-only
97
+ # evidence is valid, even though it supplies no judged mutation score.
98
+ def empty?
99
+ [lines, methods, load_lines].all? { |hits| hits.none? { |_file, values| values.any? } }
34
100
  end
35
101
 
36
102
  # Tests that executed any of the lines, fastest first (suite order breaks
@@ -46,19 +112,19 @@ module Saboteur
46
112
  by_line = lines[CoverageMap.canonical(file)]
47
113
  return [] unless by_line
48
114
 
49
- ids = line_numbers.flat_map { |line| by_line.fetch(line, []) }.uniq
50
- order = test_order
51
- ids.sort_by { |id| [durations.fetch(id, Float::INFINITY), order.fetch(id, Float::INFINITY)] }
115
+ sort_test_ids(line_numbers.flat_map { |line| by_line.fetch(line, []) }.uniq)
52
116
  end
53
117
 
54
118
  # Lines refer to tests by their index in tests, which keeps the cache
55
119
  # small for suites with long test ids.
56
120
  def to_json(*)
57
121
  index = test_order
58
- encoded = lines.transform_values do |by_line|
59
- by_line.to_h { |line, ids| [line.to_s, ids.map { |id| index.fetch(id) }] }
122
+ encode = lambda do |hits|
123
+ hits.transform_values { |by_line| by_line.to_h { |line, ids| [line.to_s, ids.map { |id| index.fetch(id) }] } }
60
124
  end
61
- JSON.generate(format: FORMAT, tests: tests, build_seconds: build_seconds, lines: encoded,
125
+ JSON.generate(format: FORMAT, tests: tests, build_seconds: build_seconds, lines: encode.call(lines),
126
+ methods: encode.call(methods), load_lines: load_lines,
127
+ branches: encode_branches(index),
62
128
  durations: tests.map { |id| durations[id]&.round(6) })
63
129
  end
64
130
 
@@ -67,11 +133,22 @@ module Saboteur
67
133
  raise Error, "coverage map format #{data['format']} is not #{FORMAT}" unless data["format"] == FORMAT
68
134
 
69
135
  tests = data["tests"]
70
- lines = data["lines"].transform_values do |by_line|
71
- by_line.to_h { |line, indexes| [line.to_i, indexes.map { |i| tests.fetch(i) }] }
136
+ decode = lambda do |hits|
137
+ hits.transform_values do |by_line|
138
+ by_line.to_h { |line, indexes| [line.to_i, indexes.map { |i| tests.fetch(i) }] }
139
+ end
72
140
  end
73
141
  durations = tests.zip(data["durations"]).to_h.compact
74
- new(lines: lines, tests: tests, build_seconds: data["build_seconds"], durations: durations)
142
+ load_lines = (data["load_lines"] || {}).transform_values { |list| list.map(&:to_i) }
143
+ new(lines: decode.call(data["lines"]), tests: tests, build_seconds: data["build_seconds"],
144
+ durations: durations, methods: decode.call(data["methods"] || {}), load_lines: load_lines,
145
+ branches: decode_branches(data.fetch("branches", {}), tests))
146
+ end
147
+
148
+ def self.decode_branches(branches, tests)
149
+ branches.transform_values do |arms|
150
+ arms.to_h { |span, ids| [span, ids.map { |id| tests.fetch(id) }] }
151
+ end
75
152
  end
76
153
 
77
154
  # Cache key over every input that can change which test runs which line.
@@ -105,6 +182,35 @@ module Saboteur
105
182
 
106
183
  private
107
184
 
185
+ def encode_branches(index)
186
+ branches.transform_values do |arms|
187
+ arms.map { |span, ids| [span, ids.map { |id| index.fetch(id) }] }
188
+ end
189
+ end
190
+
191
+ # An arm whose body the app ran while loading (a def inside an if) has no
192
+ # per-test hits, yet tests ran the lines inside it. Only lines strictly
193
+ # between the arm's first and last line count: the boundary lines can
194
+ # also hold code outside the arm. An arm with no interior line, such as
195
+ # a one-line arm, cannot be told apart, so its empty list still means
196
+ # "not taken".
197
+ def ran_before_tests?(file, span, ids)
198
+ return false unless ids.empty?
199
+
200
+ covered = lines[CoverageMap.canonical(file)] || {}
201
+ ((span[0] + 1)...span[2]).any? { |line| covered.key?(line) }
202
+ end
203
+
204
+ def position(source, offset)
205
+ prefix = source.byteslice(0, offset).b
206
+ [prefix.count("\n") + 1, offset - ((prefix.rindex("\n") || -1) + 1)]
207
+ end
208
+
209
+ def sort_test_ids(ids)
210
+ order = test_order
211
+ ids.sort_by { |id| [durations.fetch(id, Float::INFINITY), order.fetch(id, Float::INFINITY)] }
212
+ end
213
+
108
214
  def test_order
109
215
  @test_order ||= tests.each_with_index.to_h
110
216
  end