saboteur 0.2.0 → 0.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (55) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +153 -1
  3. data/README.md +526 -73
  4. data/lib/generators/saboteur/install/install_generator.rb +21 -0
  5. data/lib/saboteur/agent_skill.rb +40 -0
  6. data/lib/saboteur/baseline_health.rb +116 -0
  7. data/lib/saboteur/cli.rb +305 -30
  8. data/lib/saboteur/config.rb +65 -19
  9. data/lib/saboteur/config_file.rb +97 -0
  10. data/lib/saboteur/custom_map.rb +170 -0
  11. data/lib/saboteur/doctor.rb +88 -0
  12. data/lib/saboteur/engine/baseline_guard.rb +116 -0
  13. data/lib/saboteur/engine/boot.rb +9 -0
  14. data/lib/saboteur/engine/candidate_selection.rb +35 -0
  15. data/lib/saboteur/engine/coverage_store.rb +40 -6
  16. data/lib/saboteur/engine/data_inputs.rb +43 -0
  17. data/lib/saboteur/engine/diagnostics.rb +28 -0
  18. data/lib/saboteur/engine/diff_scope.rb +8 -0
  19. data/lib/saboteur/engine/distribution.rb +4 -2
  20. data/lib/saboteur/engine/run_fingerprint.rb +344 -0
  21. data/lib/saboteur/engine/scheduler.rb +7 -16
  22. data/lib/saboteur/engine/suite_files.rb +8 -1
  23. data/lib/saboteur/engine/verdict_check.rb +4 -2
  24. data/lib/saboteur/engine.rb +125 -25
  25. data/lib/saboteur/execution/fork_readiness.rb +46 -0
  26. data/lib/saboteur/execution/reevaluator.rb +7 -4
  27. data/lib/saboteur/execution/supervisor.rb +23 -11
  28. data/lib/saboteur/frameworks/rails.rb +329 -13
  29. data/lib/saboteur/help.rb +210 -0
  30. data/lib/saboteur/ignore_list.rb +71 -17
  31. data/lib/saboteur/install.rb +132 -0
  32. data/lib/saboteur/integration/base.rb +8 -0
  33. data/lib/saboteur/integration/rspec.rb +69 -4
  34. data/lib/saboteur/mutation_source/argument_removal.rb +86 -0
  35. data/lib/saboteur/mutation_source/builtin.rb +14 -3
  36. data/lib/saboteur/mutation_source/filter_replacement.rb +60 -0
  37. data/lib/saboteur/mutation_source/literal_replacement.rb +74 -0
  38. data/lib/saboteur/mutation_source/pair_removal.rb +101 -0
  39. data/lib/saboteur/replay.rb +33 -0
  40. data/lib/saboteur/reporter/agent.rb +87 -0
  41. data/lib/saboteur/reporter/agent_clipping.rb +29 -0
  42. data/lib/saboteur/reporter/agent_commands.rb +61 -0
  43. data/lib/saboteur/reporter/agent_incomplete.rb +37 -0
  44. data/lib/saboteur/reporter/agent_summary.rb +131 -0
  45. data/lib/saboteur/reporter/findings.rb +131 -0
  46. data/lib/saboteur/reporter/html.rb +213 -0
  47. data/lib/saboteur/reporter/replay.rb +23 -0
  48. data/lib/saboteur/reporter/stryker.rb +21 -4
  49. data/lib/saboteur/reporter/terminal.rb +76 -2
  50. data/lib/saboteur/result.rb +61 -9
  51. data/lib/saboteur/score.rb +89 -0
  52. data/lib/saboteur/version.rb +1 -1
  53. data/lib/saboteur/working_tree.rb +48 -0
  54. data/lib/saboteur.rb +7 -0
  55. metadata +30 -2
@@ -0,0 +1,213 @@
1
+ # SPDX-License-Identifier: MIT
2
+ # frozen_string_literal: true
3
+
4
+ require "cgi"
5
+ require "fileutils"
6
+
7
+ module Saboteur
8
+ module Reporter
9
+ # One self-contained HTML page: inline style, no script, no network. It is
10
+ # rendered from the same hash the JSON reporter writes (Stryker.build), so
11
+ # both reports list the same mutants with the same statuses. The scores can
12
+ # differ: the JSON schema counts NoCoverage as undetected, while this page
13
+ # shows Report#score, which leaves it out (see the "Covered mutation score"
14
+ # label on the page). The score note says what is and is not counted.
15
+ class Html
16
+ STYLE = <<~CSS
17
+ body{font:14px/1.5 system-ui,sans-serif;margin:1rem auto;max-width:72rem;padding:0 1rem;color:#1b1b1b}
18
+ h1,h2{margin:1.2em 0 .4em}table{border-collapse:collapse;width:100%;min-width:60rem}
19
+ th,td{border-bottom:1px solid #ddd;padding:.25rem .5rem;text-align:left;vertical-align:top;overflow-wrap:break-word}
20
+ td.fit{white-space:nowrap}td.wide{min-width:14rem}
21
+ th{background:#f4f4f4;font-size:12px;text-transform:uppercase}
22
+ .scroll{overflow-x:auto}.note{color:#555;font-size:12px}
23
+ .status{border-radius:3px;padding:0 .4rem;color:#fff;white-space:nowrap}
24
+ .Killed{background:#2e7d32}.Survived{background:#c62828}.Timeout{background:#b45309}
25
+ .RuntimeError{background:#6a1b9a}.NoCoverage{background:#455a64}.Ignored{background:#616161}
26
+ .Accepted{background:#8d6e63}
27
+ .warning{background:#fff8e1;border-left:4px solid #f9a825;margin:.3rem 0;padding:.3rem .6rem;overflow-wrap:anywhere}
28
+ code,pre{font-family:ui-monospace,monospace;font-size:13px}
29
+ pre{display:inline-block;vertical-align:top;margin:0;white-space:pre-wrap;overflow-wrap:anywhere}summary{cursor:pointer;font-weight:600;overflow-wrap:anywhere}
30
+ CSS
31
+
32
+ SCORE_FORMULA = "(killed + timeout) / (killed + survived + timeout)"
33
+
34
+ def initialize(path)
35
+ @path = path
36
+ end
37
+
38
+ def call(report)
39
+ FileUtils.mkdir_p(File.dirname(@path))
40
+ File.write(@path, self.class.build(report))
41
+ end
42
+
43
+ def self.build(report)
44
+ new(nil).build(report)
45
+ end
46
+
47
+ def build(report)
48
+ data = Stryker.build(report)
49
+ names = test_names(data)
50
+ results = report.results.to_h do |result|
51
+ [[Stryker.relative(result.subject.file, report.root), result.digest], result]
52
+ end
53
+ <<~HTML
54
+ <!DOCTYPE html>
55
+ <html lang="en"><head><meta charset="utf-8">
56
+ <meta name="viewport" content="width=device-width, initial-scale=1">
57
+ <title>saboteur report</title>
58
+ <style>#{STYLE}</style></head><body>
59
+ <h1>saboteur report</h1>
60
+ #{summary(report, data)}#{warnings(data[:warnings])}#{skipped_subjects(data[:skippedSubjects])}
61
+ <h2>Files</h2>
62
+ #{data[:files].map { |file, entry| file_section(file, entry, names, results) }.join("\n")}
63
+ </body></html>
64
+ HTML
65
+ end
66
+
67
+ private
68
+
69
+ def summary(report, data)
70
+ seed = data.dig(:config, :seed)
71
+ seed_note = seed ? ", seed #{seed}" : ""
72
+ counts = data[:files].values.flat_map { |entry| entry[:mutants] }.map { |mutant| mutant[:status] }.tally
73
+ <<~HTML
74
+ <p><strong>Covered mutation score: #{score_text(report.score)}</strong>
75
+ (#{report.subjects.size} subjects, #{report.results.size} mutants, #{format('%.1f', report.wall_time)}s#{seed_note})</p>
76
+ <p class="note">Covered mutation score = #{SCORE_FORMULA}. NoCoverage, errors, resource limits and skipped
77
+ mutants are not counted. Survivors accepted on the ignore list stay in the denominator. It covers only the subjects of this run, not the whole project. A tool that reads the
78
+ Stryker JSON may show a different score, because the schema counts NoCoverage as undetected.</p>
79
+ #{score_split(report)}
80
+ <p class="note">Wide tables scroll sideways.</p>
81
+ <p>#{counts.map { |status, count| "#{status_label(status)} #{count}" }.join(' ')}</p>
82
+ <p>Survivors: #{report.unignored_survivors.size} to fix, #{report.survivors.size - report.unignored_survivors.size}
83
+ accepted on the ignore list (still shown as Survived, with the reason).</p>
84
+ HTML
85
+ end
86
+
87
+ # How the score splits into assertion and crash kills, and the score
88
+ # without the whole-body mutants, which almost any test that runs the
89
+ # method kills. Each part shows only when it has something to say.
90
+ def score_split(report)
91
+ breakdown = report.score_breakdown
92
+ lines = []
93
+ lines << kill_split_line(breakdown) if breakdown.split_known?
94
+ lines << whole_body_line(report)
95
+ lines.compact.join("\n")
96
+ end
97
+
98
+ def kill_split_line(breakdown)
99
+ "<p>Of the tested mutants: #{parts_text(breakdown)}. A crash shows the code ran and broke, " \
100
+ "not that a test checked a value.</p>"
101
+ end
102
+
103
+ def parts_text(breakdown)
104
+ breakdown.parts.map { |label, share| "#{score_text(share)} #{label}" }.join(", ")
105
+ end
106
+
107
+ def whole_body_line(report)
108
+ whole = report.coarse_score_breakdown
109
+ fine = report.fine_score_breakdown
110
+ return if whole.tested.zero? || fine.tested.zero?
111
+
112
+ split = fine.split_known? ? " (#{parts_text(fine)})" : ""
113
+ "<p>Without whole-body mutants (body_nil, body_true, body_false, body_raise): #{score_text(fine.value)}" \
114
+ "#{split} over #{fine.tested} mutants. Whole-body mutants alone: #{score_text(whole.value)} " \
115
+ "over #{whole.tested}.</p>"
116
+ end
117
+
118
+ def score_text(score)
119
+ score ? format("%.1f%%", score * 100) : "n/a"
120
+ end
121
+
122
+ def warnings(messages)
123
+ return "" if messages.empty?
124
+
125
+ items = messages.map { |message| %(<div class="warning">#{escape(message)}</div>) }
126
+ "<h2>Warnings</h2>\n#{items.join("\n")}"
127
+ end
128
+
129
+ def skipped_subjects(entries)
130
+ return "" if entries.empty?
131
+
132
+ rows = entries.map do |entry|
133
+ "<tr><td>#{escape(entry[:subject])}</td><td>#{escape(entry[:file])}:#{entry[:line]}</td>" \
134
+ "<td>#{escape(entry[:reason])}</td></tr>"
135
+ end
136
+ "<h2>Skipped subjects</h2>\n<div class=\"scroll\"><table><thead><tr><th>Subject</th><th>Location</th>" \
137
+ "<th>Reason</th></tr></thead>#{rows.join}</table></div>"
138
+ end
139
+
140
+ def test_names(data)
141
+ data[:testFiles].values.flat_map { |group| group[:tests] }.to_h { |test| [test[:id], test[:name]] }
142
+ end
143
+
144
+ def file_section(file, entry, names, results)
145
+ open = entry[:mutants].any? { |mutant| mutant[:status] == "Survived" } ? " open" : ""
146
+ ordered = entry[:mutants].sort_by { |mutant| position_key(mutant) }
147
+ rows = ordered.map { |mutant| mutant_row(mutant, names, results[[file, mutant[:id]]]) }
148
+ "<details#{open}><summary>#{escape(file)} (#{entry[:mutants].size} mutants)</summary>\n" \
149
+ "<div class=\"scroll\"><table><thead><tr><th>Location</th><th>Status</th><th>Operator</th><th>Change</th>" \
150
+ "<th>Id</th><th>Notes</th></tr></thead>\n#{rows.join("\n")}</table></div></details>"
151
+ end
152
+
153
+ def position_key(mutant)
154
+ start = mutant[:location][:start]
155
+ [start[:line], start[:column], mutant[:id]]
156
+ end
157
+
158
+ def mutant_row(mutant, names, result)
159
+ start = mutant[:location][:start]
160
+ tests = Array(mutant[:killedBy]).map { |id| names.fetch(id) }.join(", ")
161
+ notes = [mutant[:statusReason], tests.empty? ? nil : "killed by #{tests}",
162
+ passing_tests_note(mutant, names, result)].compact.join("; ")
163
+ "<tr><td class=\"fit\">line #{start[:line]}, column #{start[:column]}</td><td>#{row_status(mutant,
164
+ result)}</td>" \
165
+ "<td>#{escape(mutant[:mutatorName])}</td><td class=\"wide\">#{change(mutant, result)}</td>" \
166
+ "<td class=\"fit\"><code>#{escape(mutant[:id])}</code></td><td class=\"wide\">#{escape(notes)}</td></tr>"
167
+ end
168
+
169
+ # A survivor's covering tests all passed with the mutant applied, or, for
170
+ # a reused verdict, in the earlier run it came from.
171
+ def passing_tests_note(mutant, names, result)
172
+ return unless mutant[:status] == "Survived"
173
+
174
+ passed = Array(mutant[:coveredBy]).map { |id| names.fetch(id) }
175
+ return if passed.empty?
176
+
177
+ shown = passed.first(Terminal::MAX_PASSING_TESTS).join(", ")
178
+ more = passed.size - Terminal::MAX_PASSING_TESTS
179
+ extra = more.positive? ? " (+#{more} more)" : ""
180
+ when_passed = result.reused? ? "passed in an earlier run" : "passed"
181
+ "covered by #{passed.size} #{passed.one? ? 'test' : 'tests'} that #{when_passed}: #{shown}#{extra}"
182
+ end
183
+
184
+ def row_status(mutant, result)
185
+ badge = status_label(mutant[:status])
186
+ # Only survivors are ever put on the ignore list.
187
+ result.ignored ? "#{badge} #{status_label('Accepted')}" : badge
188
+ end
189
+
190
+ # One edit shows the original text and its replacement. A mutant with
191
+ # several edits shows only the replacements, because one location cannot
192
+ # stand for all of them.
193
+ def change(mutant, result)
194
+ edits = result.mutation.edits
195
+ return "<pre>#{escape(mutant[:replacement])}</pre>" unless edits.size == 1
196
+
197
+ source = result.subject.file_source
198
+ edit = edits.first
199
+ original = source.byteslice(edit.start_byte, edit.end_byte - edit.start_byte).to_s
200
+ "<pre>#{escape(original.dup.force_encoding(Encoding::UTF_8).scrub)}</pre> becomes " \
201
+ "<pre>#{escape(edit.replacement)}</pre>"
202
+ end
203
+
204
+ def status_label(status)
205
+ %(<span class="status #{status}">#{status}</span>)
206
+ end
207
+
208
+ def escape(text)
209
+ CGI.escapeHTML(text.to_s)
210
+ end
211
+ end
212
+ end
213
+ end
@@ -0,0 +1,23 @@
1
+ # SPDX-License-Identifier: MIT
2
+ # frozen_string_literal: true
3
+
4
+ module Saboteur
5
+ module Reporter
6
+ # A replay is an inspection too, regardless of the resulting verdict.
7
+ class Replay
8
+ def initialize(out)
9
+ @out = out
10
+ end
11
+
12
+ def call(report)
13
+ report.results.each do |result|
14
+ @out.puts "\nReplay: #{result.subject.id} #{result.mutation.operator} #{result.digest} (#{result.status})"
15
+ @out.puts result.diff(root: report.root)
16
+ @out.puts "Selected tests:"
17
+ result.covered_by.each { |name| @out.puts " #{name}" }
18
+ @out.puts " none" if result.covered_by.empty?
19
+ end
20
+ end
21
+ end
22
+ end
23
+ end
@@ -16,14 +16,14 @@ module Saboteur
16
16
  @path = path
17
17
  end
18
18
 
19
- def call(report)
19
+ def call(report, exit_cause: nil)
20
20
  FileUtils.mkdir_p(File.dirname(@path))
21
- File.write(@path, JSON.generate(self.class.build(report)))
21
+ File.write(@path, JSON.generate(self.class.build(report, exit_cause: exit_cause)))
22
22
  end
23
23
 
24
24
  # Tests are listed once under testFiles with short ids; mutants refer to
25
25
  # them by id in killedBy/coveredBy instead of repeating full test names.
26
- def self.build(report)
26
+ def self.build(report, exit_cause: nil)
27
27
  test_ids = test_index(report.results)
28
28
  files = report.results.group_by { |result| result.subject.file }.to_h do |file, results|
29
29
  [relative(file, report.root), file_entry(results, test_ids)]
@@ -34,10 +34,27 @@ module Saboteur
34
34
  projectRoot: report.root,
35
35
  files: files,
36
36
  testFiles: test_files(test_ids),
37
- framework: { name: "saboteur", version: VERSION }
37
+ skippedSubjects: skipped_subjects(report),
38
+ rejectedKills: report.rejected_kills,
39
+ warnings: report.warnings,
40
+ framework: { name: "saboteur", version: VERSION },
41
+ saboteur: { findings: Findings.build(report), scores: scores(report) }
42
+ .merge(exit_cause ? { exitCause: exit_cause } : {})
38
43
  }.merge(report.seed ? { config: { seed: report.seed } } : {})
39
44
  end
40
45
 
46
+ # Score with its assertion and crash shares, for all mutants and without
47
+ # the whole-body ones (body_nil, body_true, body_false, body_raise).
48
+ def self.scores(report)
49
+ { all: report.score_breakdown.to_h, withoutWholeBodyMutants: report.fine_score_breakdown.to_h }
50
+ end
51
+
52
+ def self.skipped_subjects(report)
53
+ report.skipped_subjects.map do |subject, reason|
54
+ { subject: subject.id, file: relative(subject.file, report.root), line: subject.start_line, reason: reason }
55
+ end
56
+ end
57
+
41
58
  def self.test_index(results)
42
59
  names = results.flat_map { |result| result.covered_by + result.killed_by }.uniq
43
60
  names.each_with_index.to_h { |name, index| [name, index.to_s] }
@@ -7,6 +7,8 @@ module Saboteur
7
7
  # errors, then a unified diff for each survivor.
8
8
  class Terminal
9
9
  MAX_ERRORS = 10
10
+ MAX_PASSING_TESTS = 5
11
+ MAX_CHANGED_LINES = 3
10
12
 
11
13
  def initialize(out)
12
14
  @out = out
@@ -14,6 +16,7 @@ module Saboteur
14
16
 
15
17
  def call(report)
16
18
  summary(report)
19
+ skipped_subjects(report)
17
20
  warnings(report)
18
21
  errors(report)
19
22
  survivors(report)
@@ -39,9 +42,10 @@ module Saboteur
39
42
  end
40
43
 
41
44
  def details(report)
42
- score = report.score
43
- @out.puts "score #{score ? format('%.1f %%', score * 100) : 'n/a'} " \
45
+ breakdown = report.score_breakdown
46
+ @out.puts "score #{percent(breakdown.value)}#{score_split(breakdown)} " \
44
47
  "(killed+timeout)/(killed+survived+timeout); errors, resource limits and skips excluded"
48
+ whole_body_score(report)
45
49
  mean = report.mean_runtime
46
50
  @out.puts "mean time per executed mutant #{format('%.3f', mean)} s" if mean
47
51
  @out.puts "invalid mutations dropped (did not parse): #{report.invalid_mutations}" if
@@ -50,9 +54,35 @@ module Saboteur
50
54
  reused = report.results.count(&:reused?)
51
55
  @out.puts "reused #{reused} of #{report.results.size} verdicts from earlier runs (--incremental)" if
52
56
  reused.positive?
57
+ @out.puts "kills rejected by unchanged-method check: #{report.rejected_kills}"
53
58
  kill_causes(report)
54
59
  end
55
60
 
61
+ # " (assertion 50.0 %, crash 50.0 %)": how much of the score is a failed
62
+ # check and how much is a crash. Timeouts and kills with no recorded
63
+ # cause are named only when there are some, so the parts add up.
64
+ def score_split(breakdown)
65
+ return "" unless breakdown.split_known?
66
+
67
+ " (#{breakdown.parts.map { |label, share| "#{label} #{percent(share)}" }.join(', ')})"
68
+ end
69
+
70
+ # Whole-body mutants (body_nil, body_true, body_false, body_raise) are
71
+ # killed by almost any test that runs the method, so they get their own
72
+ # line.
73
+ def whole_body_score(report)
74
+ whole = report.coarse_score_breakdown
75
+ fine = report.fine_score_breakdown
76
+ return if whole.tested.zero? || fine.tested.zero?
77
+
78
+ @out.puts "score without whole-body mutants #{percent(fine.value)}#{score_split(fine)} " \
79
+ "(#{fine.tested} mutants); whole-body mutants alone #{percent(whole.value)} (#{whole.tested})"
80
+ end
81
+
82
+ def percent(share)
83
+ share ? format("%.1f %%", share * 100) : "n/a"
84
+ end
85
+
56
86
  # How many kills came from a failed check and how many from a crash, and
57
87
  # which operators mostly kill by crashing (their score says less).
58
88
  def kill_causes(report)
@@ -80,6 +110,15 @@ module Saboteur
80
110
  parts.join(", ")
81
111
  end
82
112
 
113
+ def skipped_subjects(report)
114
+ report.skipped_subjects.values.tally.each do |reason, count|
115
+ @out.puts "skipped subjects: #{reason}: #{count}"
116
+ end
117
+ report.skipped_subjects.each do |subject, reason|
118
+ @out.puts " #{subject.id} #{relative(subject.file, report.root)}:#{subject.start_line}: #{reason}"
119
+ end
120
+ end
121
+
83
122
  def warnings(report)
84
123
  report.warnings.each { |warning| @out.puts "warning: #{warning}" }
85
124
  end
@@ -103,12 +142,47 @@ module Saboteur
103
142
  shown(report).each do |result, hidden|
104
143
  location = "#{relative(result.subject.file, report.root)}:#{result.subject.start_line}"
105
144
  @out.puts "\n#{result.subject.id} #{result.mutation.operator} #{result.digest} #{location}"
145
+ changed_lines(report, result)
106
146
  @out.puts "ignored: #{result.ignored}" if result.ignored
107
147
  @out.puts result.diff(root: report.root)
148
+ passing_tests(result)
108
149
  @out.puts "#{hidden} more survivor#{'s' if hidden > 1} on this line in the JSON report" if hidden.positive?
109
150
  end
110
151
  end
111
152
 
153
+ # The tests that ran with the mutant applied and still passed: the
154
+ # missing assertion belongs in one of them. They are listed likeliest
155
+ # killers first, the order the run used. Line coverage can overstate which
156
+ # tests exercise the line, so the list is a lead, not proof.
157
+ def passing_tests(result)
158
+ tests = result.covered_by
159
+ return if tests.empty?
160
+
161
+ @out.puts "#{tests.size} #{tests.one? ? 'test' : 'tests'} covered this line and #{passed_phrase(result)} " \
162
+ "(the missing check may belong in one of them):"
163
+ tests.first(MAX_PASSING_TESTS).each { |name| @out.puts " #{name}" }
164
+ more = tests.size - MAX_PASSING_TESTS
165
+ @out.puts " ... #{more} more in the JSON report (coveredBy)" if more.positive?
166
+ end
167
+
168
+ # A reused verdict was not run again: its tests passed in the earlier run.
169
+ def passed_phrase(result)
170
+ result.reused? ? "passed in an earlier run (--incremental)" : "still passed"
171
+ end
172
+
173
+ # In diff mode: the changed lines this mutant touches. The location above
174
+ # is where the mutated statement starts, which can be a line or two above
175
+ # the line that changed.
176
+ def changed_lines(report, result)
177
+ lines = report.changed_lines_for(result)
178
+ return if lines.empty?
179
+
180
+ path = relative(result.subject.file, report.root)
181
+ shown = lines.first(MAX_CHANGED_LINES).map { |line| "#{path}:#{line}" }.join(", ")
182
+ more = lines.size - MAX_CHANGED_LINES
183
+ @out.puts "changed #{lines.one? ? 'line' : 'lines'}: #{shown}#{" (+#{more} more)" if more.positive?}"
184
+ end
185
+
112
186
  # [[result, how many more on the same line]]. In review mode survivors
113
187
  # on one line usually share a fate, so one is shown: live before
114
188
  # ignored, then coarse before fine, then the earliest edit.
@@ -2,6 +2,7 @@
2
2
  # frozen_string_literal: true
3
3
 
4
4
  require_relative "reporter/diff"
5
+ require_relative "score"
5
6
 
6
7
  module Saboteur
7
8
  # resource_limit: the mutant's process tree hit the memory or process cap.
@@ -17,7 +18,7 @@ module Saboteur
17
18
  attr_accessor :ignored
18
19
 
19
20
  def initialize(subject:, mutation:, status:, mutated_source:, reason: nil, runtime: nil, killed_by: [],
20
- covered_by: [], kill_cause: nil, reused: false)
21
+ covered_by: [], kill_cause: nil, reused: false, rejected_kill: false)
21
22
  raise ArgumentError, "unknown status #{status.inspect}" unless STATUSES.include?(status)
22
23
 
23
24
  @subject = subject
@@ -29,14 +30,20 @@ module Saboteur
29
30
  @covered_by = covered_by
30
31
  @mutated_source = mutated_source
31
32
  @kill_cause = kill_cause
33
+ @rejected_kill = rejected_kill
32
34
  @reused = reused
33
35
  @ignored = nil
34
36
  end
35
37
 
36
38
  # The same mutant, marked as an error: its verdict did not hold up.
37
- def as_error(reason)
39
+ def as_error(reason, rejected_kill: false)
38
40
  MutantResult.new(subject: subject, mutation: mutation, status: :error, mutated_source: mutated_source,
39
- reason: reason, runtime: runtime, killed_by: killed_by, covered_by: covered_by)
41
+ reason: reason, runtime: runtime, killed_by: killed_by, covered_by: covered_by,
42
+ rejected_kill: rejected_kill)
43
+ end
44
+
45
+ def rejected_kill?
46
+ @rejected_kill
40
47
  end
41
48
 
42
49
  def digest
@@ -53,6 +60,11 @@ module Saboteur
53
60
  status == :killed && !kill_cause.nil? && kill_cause != "assertion"
54
61
  end
55
62
 
63
+ # Killed because a test's check failed: the test looked at a value.
64
+ def assertion_kill?
65
+ status == :killed && kill_cause == "assertion"
66
+ end
67
+
56
68
  # Taken from an earlier run (--incremental) instead of run again.
57
69
  def reused?
58
70
  @reused
@@ -78,12 +90,16 @@ module Saboteur
78
90
 
79
91
  # Everything one run produced.
80
92
  class Report
81
- attr_reader :results, :subjects, :wall_time, :invalid_mutations, :warnings, :root, :seed
93
+ # More than half of the executed mutants errored: the run tested
94
+ # almost nothing and must not pass a gate.
95
+ MAX_ERROR_FRACTION = 0.5
96
+
97
+ attr_reader :results, :subjects, :wall_time, :invalid_mutations, :warnings, :root, :seed, :skipped_subjects
82
98
 
83
99
  # REVIEW: diff mode; the terminal then shows one survivor per line.
84
100
  # seed: the random seed every test run started from (nil when unknown).
85
101
  def initialize(results:, subjects:, wall_time:, root:, invalid_mutations: 0, warnings: [], review: false,
86
- seed: nil)
102
+ seed: nil, skipped_subjects: {}, diff_scope: nil)
87
103
  @results = results
88
104
  @subjects = subjects
89
105
  @wall_time = wall_time
@@ -92,6 +108,29 @@ module Saboteur
92
108
  @warnings = warnings
93
109
  @review = review
94
110
  @seed = seed
111
+ @skipped_subjects = skipped_subjects
112
+ @diff_scope = diff_scope
113
+ end
114
+
115
+ # In diff mode, the changed lines (sorted) that a mutant touches; empty
116
+ # outside diff mode, or when the mutant touches none (a whole-body mutant
117
+ # of a method whose only change is its signature).
118
+ def changed_lines_for(result)
119
+ @diff_scope ? @diff_scope.changed_lines_for(result.subject, result.mutation) : []
120
+ end
121
+
122
+ def rejected_kills
123
+ results.count(&:rejected_kill?)
124
+ end
125
+
126
+ # Errored mutants (a crash, a neutral check that failed) are outside
127
+ # the score, so a run whose executed mutants mostly errored tested
128
+ # almost nothing. Skipped mutants never ran: they are neither the
129
+ # numerator nor the denominator.
130
+ def majority_errored?
131
+ return false if executed.empty?
132
+
133
+ executed.count { |result| result.status == :error }.fdiv(executed.size) > MAX_ERROR_FRACTION
95
134
  end
96
135
 
97
136
  def review?
@@ -104,10 +143,23 @@ module Saboteur
104
143
 
105
144
  # (killed + timeout) / (killed + survived + timeout); nil with nothing tested.
106
145
  def score
107
- c = counts
108
- detected = c[:killed] + c[:timeout]
109
- tested = detected + c[:survived]
110
- tested.zero? ? nil : detected.fdiv(tested)
146
+ score_breakdown.value
147
+ end
148
+
149
+ # The score with its split into assertion kills and crash kills.
150
+ def score_breakdown
151
+ @score_breakdown ||= Score.of(results)
152
+ end
153
+
154
+ # The same breakdown without the whole-body mutants (every
155
+ # mutation marked coarse: body_nil, body_true, body_false, body_raise). Almost any test that runs the method kills
156
+ # those, so they flatter the score.
157
+ def fine_score_breakdown
158
+ @fine_score_breakdown ||= Score.of(results.reject { |result| result.mutation.coarse })
159
+ end
160
+
161
+ def coarse_score_breakdown
162
+ @coarse_score_breakdown ||= Score.of(results.select { |result| result.mutation.coarse })
111
163
  end
112
164
 
113
165
  def survivors
@@ -0,0 +1,89 @@
1
+ # SPDX-License-Identifier: MIT
2
+ # frozen_string_literal: true
3
+
4
+ module Saboteur
5
+ # The mutation score of a group of results, with the share of it that came
6
+ # from failed assertions and the share that came from crashes.
7
+ #
8
+ # value is (killed + timeout) / (killed + survived + timeout). A crash kill
9
+ # shows the code ran and broke, not that a test checked a value, so the
10
+ # split lets a reader see how much of the score is checked behavior.
11
+ # Errors, resource limits and skips are outside every number here.
12
+ class Score
13
+ def self.of(results)
14
+ new(results)
15
+ end
16
+
17
+ attr_reader :assertion_kills, :crash_kills, :unclassified_kills, :timeouts, :survivors
18
+
19
+ def initialize(results)
20
+ killed = results.select { |result| result.status == :killed }
21
+ @assertion_kills = killed.count(&:assertion_kill?)
22
+ @crash_kills = killed.count(&:crash_kill?)
23
+ @unclassified_kills = killed.size - assertion_kills - crash_kills
24
+ @timeouts = results.count { |result| result.status == :timeout }
25
+ @survivors = results.count(&:survived?)
26
+ end
27
+
28
+ def detected
29
+ assertion_kills + crash_kills + unclassified_kills + timeouts
30
+ end
31
+
32
+ def tested
33
+ detected + survivors
34
+ end
35
+
36
+ # nil with nothing tested.
37
+ def value
38
+ share(detected)
39
+ end
40
+
41
+ def assertion_share
42
+ share(assertion_kills)
43
+ end
44
+
45
+ def crash_share
46
+ share(crash_kills)
47
+ end
48
+
49
+ def unclassified_share
50
+ share(unclassified_kills)
51
+ end
52
+
53
+ def timeout_share
54
+ share(timeouts)
55
+ end
56
+
57
+ # False when no kill has a recorded cause: a split would then say
58
+ # nothing. A cause is missing for verdicts reused from a run made before
59
+ # causes were recorded, for a test that called exit, and for a failure
60
+ # outside any example (suite level).
61
+ def split_known?
62
+ assertion_kills.positive? || crash_kills.positive?
63
+ end
64
+
65
+ # [[label, share]]: assertion and crash always, then timeout and
66
+ # "cause not recorded" only when there are some, so the parts add up to
67
+ # value.
68
+ def parts
69
+ list = [["assertion", assertion_share], ["crash", crash_share]]
70
+ list << ["timeout", timeout_share] if timeouts.positive?
71
+ list << ["cause not recorded", unclassified_share] if unclassified_kills.positive?
72
+ list
73
+ end
74
+
75
+ def to_h
76
+ {
77
+ score: value, tested: tested, assertionKills: assertion_kills, crashKills: crash_kills,
78
+ unclassifiedKills: unclassified_kills, timeouts: timeouts, survivors: survivors,
79
+ assertionShare: assertion_share, crashShare: crash_share
80
+ }
81
+ end
82
+
83
+ private
84
+
85
+ def share(count)
86
+ tested.zero? ? nil : count.fdiv(tested)
87
+ end
88
+ end
89
+ end
@@ -2,5 +2,5 @@
2
2
  # frozen_string_literal: true
3
3
 
4
4
  module Saboteur
5
- VERSION = "0.2.0"
5
+ VERSION = "0.4.0"
6
6
  end