saboteur 0.4.0 → 0.5.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (90) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +167 -0
  3. data/README.md +393 -20
  4. data/exe/saboteur +6 -2
  5. data/lib/saboteur/agent_skill.rb +6 -0
  6. data/lib/saboteur/baseline/acceptance.rb +218 -0
  7. data/lib/saboteur/baseline/command.rb +80 -0
  8. data/lib/saboteur/baseline/comparison.rb +105 -0
  9. data/lib/saboteur/baseline/completeness.rb +53 -0
  10. data/lib/saboteur/baseline/evidence.rb +114 -0
  11. data/lib/saboteur/baseline/identity.rb +78 -0
  12. data/lib/saboteur/baseline/manifest.rb +164 -0
  13. data/lib/saboteur/baseline/run_evidence.rb +117 -0
  14. data/lib/saboteur/baseline/workflow.rb +121 -0
  15. data/lib/saboteur/baseline_health.rb +1 -1
  16. data/lib/saboteur/cli/exit_status.rb +93 -0
  17. data/lib/saboteur/cli/run_reporting.rb +75 -0
  18. data/lib/saboteur/cli.rb +101 -121
  19. data/lib/saboteur/config.rb +76 -6
  20. data/lib/saboteur/doctor.rb +40 -8
  21. data/lib/saboteur/engine/baseline_guard.rb +33 -11
  22. data/lib/saboteur/engine/bytecode_equivalence.rb +95 -0
  23. data/lib/saboteur/engine/canary.rb +74 -0
  24. data/lib/saboteur/engine/candidate_selection.rb +18 -5
  25. data/lib/saboteur/engine/candidates.rb +59 -7
  26. data/lib/saboteur/engine/coverage_store.rb +34 -14
  27. data/lib/saboteur/engine/diagnostics.rb +150 -4
  28. data/lib/saboteur/engine/diff_scope.rb +10 -0
  29. data/lib/saboteur/engine/distribution.rb +54 -13
  30. data/lib/saboteur/engine/result_cache.rb +10 -5
  31. data/lib/saboteur/engine/run_fingerprint.rb +6 -4
  32. data/lib/saboteur/engine/sandbox_policy.rb +108 -0
  33. data/lib/saboteur/engine/scheduler.rb +92 -20
  34. data/lib/saboteur/engine/subject_selection.rb +24 -1
  35. data/lib/saboteur/engine/test_guess.rb +52 -0
  36. data/lib/saboteur/engine/unchanged_syntax.rb +87 -0
  37. data/lib/saboteur/engine/verdict_check.rb +7 -6
  38. data/lib/saboteur/engine.rb +192 -18
  39. data/lib/saboteur/execution/protocol.rb +2 -1
  40. data/lib/saboteur/execution/reevaluator.rb +72 -10
  41. data/lib/saboteur/execution/runner.rb +120 -17
  42. data/lib/saboteur/execution/sandbox.rb +271 -0
  43. data/lib/saboteur/execution/stale_captures.rb +43 -0
  44. data/lib/saboteur/execution/supervisor.rb +101 -9
  45. data/lib/saboteur/execution/worker_pool.rb +20 -4
  46. data/lib/saboteur/execution/wrapper.rb +45 -0
  47. data/lib/saboteur/gates.rb +83 -0
  48. data/lib/saboteur/help.rb +31 -7
  49. data/lib/saboteur/ignore_list.rb +5 -0
  50. data/lib/saboteur/inline_suppression.rb +65 -0
  51. data/lib/saboteur/integration/base.rb +1 -0
  52. data/lib/saboteur/integration/focus_guard.rb +83 -0
  53. data/lib/saboteur/integration/rspec.rb +21 -1
  54. data/lib/saboteur/mutation_source/builtin.rb +22 -3
  55. data/lib/saboteur/mutation_source/chain_removal.rb +80 -0
  56. data/lib/saboteur/mutation_source/condition_negation.rb +50 -0
  57. data/lib/saboteur/mutation_source/endpoint_replacement.rb +61 -0
  58. data/lib/saboteur/mutation_source/hazard_guard.rb +136 -0
  59. data/lib/saboteur/mutation_source/operator_assignment.rb +52 -0
  60. data/lib/saboteur/mutation_source/operator_registry.rb +57 -0
  61. data/lib/saboteur/mutation_source/range_replacement.rb +43 -0
  62. data/lib/saboteur/preview.rb +106 -0
  63. data/lib/saboteur/reporter/agent_commands.rb +7 -1
  64. data/lib/saboteur/reporter/agent_incomplete.rb +24 -1
  65. data/lib/saboteur/reporter/agent_summary.rb +11 -5
  66. data/lib/saboteur/reporter/findings.rb +4 -19
  67. data/lib/saboteur/reporter/history_paths.rb +38 -0
  68. data/lib/saboteur/reporter/history_record.rb +132 -0
  69. data/lib/saboteur/reporter/html.rb +12 -1
  70. data/lib/saboteur/reporter/mutation_text.rb +36 -0
  71. data/lib/saboteur/reporter/next_steps.rb +30 -0
  72. data/lib/saboteur/reporter/palette.rb +37 -0
  73. data/lib/saboteur/reporter/score_history.rb +123 -0
  74. data/lib/saboteur/reporter/score_table.rb +57 -0
  75. data/lib/saboteur/reporter/stryker.rb +15 -2
  76. data/lib/saboteur/reporter/survivor_hints.rb +68 -0
  77. data/lib/saboteur/reporter/survivor_list.rb +126 -0
  78. data/lib/saboteur/reporter/survivor_ranking.rb +34 -0
  79. data/lib/saboteur/reporter/terminal.rb +131 -79
  80. data/lib/saboteur/reporter/test_matrix.rb +57 -0
  81. data/lib/saboteur/result.rb +41 -6
  82. data/lib/saboteur/selection/coverage_map.rb +119 -13
  83. data/lib/saboteur/selection/git_diff.rb +28 -1
  84. data/lib/saboteur/selection/recorder.rb +55 -13
  85. data/lib/saboteur/subject.rb +40 -4
  86. data/lib/saboteur/subject_finder.rb +62 -12
  87. data/lib/saboteur/version.rb +1 -1
  88. data/lib/saboteur/working_tree.rb +8 -0
  89. data/lib/saboteur.rb +6 -0
  90. metadata +43 -2
@@ -0,0 +1,132 @@
1
+ # SPDX-License-Identifier: MIT
2
+ # frozen_string_literal: true
3
+
4
+ require "json"
5
+ require "time"
6
+
7
+ module Saboteur
8
+ module Reporter
9
+ # Strict version-one observation schema. Never repairs imported evidence.
10
+ module HistoryRecord
11
+ FIELDS = %w[version date commit selection inventory score coveredScore counts breakdown incomplete].freeze
12
+ BREAKDOWN = %w[score tested assertionKills crashKills unclassifiedKills timeouts survivors
13
+ assertionShare crashShare].freeze
14
+ COUNTS = STATUSES.map(&:to_s).freeze
15
+
16
+ class UniqueObject < Hash
17
+ def []=(key, value)
18
+ raise Error, "score history contains duplicate keys" if key?(key)
19
+
20
+ super
21
+ end
22
+ end
23
+
24
+ module_function
25
+
26
+ def parse(line)
27
+ record = JSON.parse(line, object_class: UniqueObject)
28
+ validate(record)
29
+ record
30
+ rescue JSON::ParserError, ArgumentError, TypeError => e
31
+ raise Error, "score history contains an invalid record: #{e.message}"
32
+ end
33
+
34
+ def validate(record)
35
+ fields(record, FIELDS)
36
+ unless record.fetch("version").is_a?(Integer) && record.fetch("version") == 1
37
+ raise Error,
38
+ "score history contains an invalid version"
39
+ end
40
+
41
+ date(record.fetch("date"))
42
+ commit = record.fetch("commit")
43
+ raise Error, "score history contains an invalid commit" unless commit.nil? || sha?(commit, 40)
44
+
45
+ %w[selection inventory].each do |key|
46
+ raise Error, "score history contains an invalid digest" unless sha?(record.fetch(key), 64)
47
+ end
48
+ %w[score coveredScore].each { |key| fraction(record.fetch(key)) }
49
+ counts = record.fetch("counts")
50
+ fields(counts, COUNTS)
51
+ counts.each_value { |value| count(value) }
52
+ breakdown(record.fetch("breakdown"), counts, record.fetch("coveredScore"))
53
+ score_consistency(record)
54
+ reasons(record.fetch("incomplete"))
55
+ end
56
+
57
+ def date(value)
58
+ return if Time.iso8601(value).utc.iso8601 == value
59
+
60
+ raise Error,
61
+ "score history contains an invalid date"
62
+ rescue ArgumentError, TypeError, NoMethodError => e
63
+ raise Error, "score history contains an invalid date: #{e.message}"
64
+ end
65
+
66
+ def reasons(value)
67
+ unless value.is_a?(Array) && value.all? do |reason|
68
+ reason.is_a?(String) && !reason.empty?
69
+ end && value.uniq == value
70
+ raise Error, "score history contains invalid incomplete reasons"
71
+ end
72
+ end
73
+
74
+ def score_consistency(record)
75
+ value = record.fetch("breakdown")
76
+ detected = value.values_at("assertionKills", "crashKills", "unclassifiedKills", "timeouts").sum
77
+ tested = value.fetch("tested")
78
+ expected = tested.zero? ? nil : detected.fdiv(tested)
79
+ raise Error, "score history contains an inconsistent score" unless record.fetch("coveredScore") == expected
80
+
81
+ %w[assertion crash].each do |kind|
82
+ expected_share = tested.zero? ? nil : value.fetch("#{kind}Kills").fdiv(tested)
83
+ unless value.fetch("#{kind}Share") == expected_share
84
+ raise Error,
85
+ "score history contains an inconsistent share"
86
+ end
87
+ end
88
+ overall = record.fetch("score")
89
+ return unless overall && ((record.fetch("coveredScore").nil? && !overall.zero?) ||
90
+ (record.fetch("coveredScore") && overall > record.fetch("coveredScore")))
91
+
92
+ raise Error, "score history contains an inconsistent overall score"
93
+ end
94
+
95
+ def fields(object, expected)
96
+ return if object.is_a?(Hash) && object.keys.sort == expected.sort
97
+
98
+ raise Error,
99
+ "score history contains an invalid record schema"
100
+ end
101
+
102
+ def sha?(value, length)
103
+ value.is_a?(String) && value.match?(/\A[0-9a-f]{#{length}}\z/)
104
+ end
105
+
106
+ def fraction(value)
107
+ return if value.nil? || (0..1).cover?(value)
108
+
109
+ raise Error,
110
+ "score history contains an invalid score"
111
+ end
112
+
113
+ def count(value)
114
+ raise Error, "score history contains an invalid count" unless value.is_a?(Integer) && value >= 0
115
+ end
116
+
117
+ def breakdown(value, counts, covered)
118
+ fields(value, BREAKDOWN)
119
+ %w[score assertionShare crashShare].each { |key| fraction(value.fetch(key)) }
120
+ (BREAKDOWN - %w[score assertionShare crashShare]).each { |key| count(value.fetch(key)) }
121
+ kills = value.values_at("assertionKills", "crashKills", "unclassifiedKills").sum
122
+ tested = counts.values_at("killed", "survived", "timeout").sum
123
+ unless kills == counts.fetch("killed") && value.fetch("tested") == tested &&
124
+ value.fetch("timeouts") == counts.fetch("timeout") &&
125
+ value.fetch("survivors") == counts.fetch("survived") &&
126
+ value.fetch("score") == covered
127
+ raise Error, "score history contains inconsistent counts"
128
+ end
129
+ end
130
+ end
131
+ end
132
+ end
@@ -57,7 +57,7 @@ module Saboteur
57
57
  <title>saboteur report</title>
58
58
  <style>#{STYLE}</style></head><body>
59
59
  <h1>saboteur report</h1>
60
- #{summary(report, data)}#{warnings(data[:warnings])}#{skipped_subjects(data[:skippedSubjects])}
60
+ #{summary(report, data)}#{inline_suppressions(report)}#{warnings(data[:warnings])}#{skipped_subjects(data[:skippedSubjects])}
61
61
  <h2>Files</h2>
62
62
  #{data[:files].map { |file, entry| file_section(file, entry, names, results) }.join("\n")}
63
63
  </body></html>
@@ -66,6 +66,17 @@ module Saboteur
66
66
 
67
67
  private
68
68
 
69
+ def inline_suppressions(report)
70
+ return "" if report.inline_suppressions.empty?
71
+
72
+ items = report.inline_suppressions.map do |entry|
73
+ path = entry[:file].delete_prefix("#{report.root}/")
74
+ "<li>#{escape(path)}:#{entry[:line]} #{escape(entry[:subject])} " \
75
+ "#{escape(entry[:operator])} <code>#{escape(entry[:digest])}</code>: #{escape(entry[:reason])}</li>"
76
+ end
77
+ "<h2>Inline suppressions (#{items.size}, outside the score)</h2><ul>#{items.join}</ul>"
78
+ end
79
+
69
80
  def summary(report, data)
70
81
  seed = data.dig(:config, :seed)
71
82
  seed_note = seed ? ", seed #{seed}" : ""
@@ -0,0 +1,36 @@
1
+ # SPDX-License-Identifier: MIT
2
+ # frozen_string_literal: true
3
+
4
+ module Saboteur
5
+ module Reporter
6
+ # The original and replacement text of one mutant, and the line it
7
+ # changes, for compact one-line output.
8
+ module MutationText
9
+ module_function
10
+
11
+ def line(result)
12
+ result.subject.line_of(sorted_edits(result).first.start_byte)
13
+ end
14
+
15
+ def original(result)
16
+ join(result) { |edit| result.subject.file_source.byteslice(edit.start_byte, edit.end_byte - edit.start_byte) }
17
+ end
18
+
19
+ def replacement(result)
20
+ join(result, &:replacement)
21
+ end
22
+
23
+ def sorted_edits(result)
24
+ result.mutation.edits.sort_by(&:start_byte)
25
+ end
26
+
27
+ # Whole-body mutants carry a long text; one line keeps the output compact.
28
+ def join(result, &pick)
29
+ text = sorted_edits(result).map { |edit| pick.call(edit).to_s.dup.force_encoding(Encoding::UTF_8).scrub }
30
+ .join(" | ")
31
+ flat = text.gsub(/\s*\n\s*/, " ").strip
32
+ flat.length > 60 ? "#{flat[0, 57]}..." : flat
33
+ end
34
+ end
35
+ end
36
+ end
@@ -0,0 +1,30 @@
1
+ # SPDX-License-Identifier: MIT
2
+ # frozen_string_literal: true
3
+
4
+ module Saboteur
5
+ module Reporter
6
+ # The footer under a run that left survivors: how to look at one, how to
7
+ # accept one, and how to hand the run to an agent. Silent when nothing
8
+ # survived, so a clean run stays short.
9
+ class NextSteps
10
+ def initialize(out, palette)
11
+ @out = out
12
+ @palette = palette
13
+ end
14
+
15
+ def call(report)
16
+ survivor = report.unignored_survivors.first
17
+ return unless survivor
18
+
19
+ @out.puts "\n#{@palette.paint(:bold, 'Next steps:')}"
20
+ seed = " --seed #{report.seed}" if report.seed
21
+ @out.puts " replay one survivor: saboteur run --id #{survivor.digest}#{seed}"
22
+ @out.puts " (add this run's own --config, --since, --subject, --test and source options)"
23
+ @out.puts " accept an equivalent one: add to your ignore file (c.ignore_file, default #{Config::IGNORE_FILE}):"
24
+ entry = %(i.mutation "#{survivor.digest}", subject: "#{survivor.subject.id}", reason: "why it is equivalent")
25
+ @out.puts " #{entry}"
26
+ @out.puts " let an agent close the gaps: saboteur run --agent, with the same options"
27
+ end
28
+ end
29
+ end
30
+ end
@@ -0,0 +1,37 @@
1
+ # SPDX-License-Identifier: MIT
2
+ # frozen_string_literal: true
3
+
4
+ module Saboteur
5
+ module Reporter
6
+ # Wraps text in ANSI colour codes when colour is on, and returns it
7
+ # unchanged when it is off. Colour is on only when asked for or when the
8
+ # output is a terminal and NO_COLOR is not set (https://no-color.org).
9
+ class Palette
10
+ CODES = { red: 31, green: 32, yellow: 33, magenta: 35, cyan: 36, dim: 2, bold: 1 }.freeze
11
+
12
+ # mode: :auto, :always or :never.
13
+ def self.for(out, mode: :auto, environment: ENV)
14
+ enabled = case mode
15
+ when :always then true
16
+ when :never then false
17
+ else out.respond_to?(:tty?) && out.tty? && environment["NO_COLOR"].to_s.empty?
18
+ end
19
+ new(enabled: enabled)
20
+ end
21
+
22
+ def initialize(enabled:)
23
+ @enabled = enabled
24
+ end
25
+
26
+ def enabled?
27
+ @enabled
28
+ end
29
+
30
+ def paint(color, text)
31
+ return text unless @enabled
32
+
33
+ "\e[#{CODES.fetch(color)}m#{text}\e[0m"
34
+ end
35
+ end
36
+ end
37
+ end
@@ -0,0 +1,123 @@
1
+ # SPDX-License-Identifier: MIT
2
+ # frozen_string_literal: true
3
+
4
+ require "json"
5
+ require "time"
6
+ require "digest"
7
+ require "fileutils"
8
+ require_relative "agent_incomplete"
9
+ require_relative "history_record"
10
+
11
+ module Saboteur
12
+ module Reporter
13
+ # Opt-in append-only score observations, not a baseline or a CI gate.
14
+ class ScoreHistory
15
+ DIFFERENT_RUN = "score history: change unavailable (different selection/inventory or incomplete run)"
16
+ MISSING_SCORE = "score history: change unavailable (no tested mutants)"
17
+ MAX_RECORD_BYTES = 64 * 1024
18
+ OPEN_FLAGS = File::RDWR | File::CREAT | File::NONBLOCK | File::NOFOLLOW
19
+
20
+ def initialize(path)
21
+ @path = path
22
+ end
23
+
24
+ def append(report, head:, scope:)
25
+ record = build(report, head, scope)
26
+ HistoryRecord.validate(record)
27
+ line = "#{JSON.generate(record)}\n"
28
+ raise Error, "score history record exceeds byte limit" if line.bytesize > MAX_RECORD_BYTES
29
+
30
+ FileUtils.mkdir_p(File.dirname(@path))
31
+ File.open(@path, OPEN_FLAGS, 0o600) do |file|
32
+ regular_file(file)
33
+ lock(file, File::LOCK_EX)
34
+
35
+ previous = previous_record(file)
36
+ file.seek(0, IO::SEEK_END)
37
+ file.write(line)
38
+ file.flush
39
+ change(previous, record)
40
+ end
41
+ rescue SystemCallError, IOError, JSON::ParserError, KeyError, TypeError => e
42
+ raise Error, "score history was not recorded: #{e.message}"
43
+ end
44
+
45
+ def validate_existing
46
+ return unless File.exist?(@path) || File.symlink?(@path)
47
+
48
+ File.open(@path, File::RDONLY | File::NONBLOCK | File::NOFOLLOW) do |file|
49
+ regular_file(file)
50
+ lock(file, File::LOCK_SH)
51
+
52
+ previous_record(file)
53
+ end
54
+ rescue SystemCallError, IOError => e
55
+ raise Error, "score history cannot be read: #{e.message}"
56
+ end
57
+
58
+ private
59
+
60
+ def build(report, head, scope)
61
+ { "version" => 1, "date" => Time.now.utc.iso8601, "commit" => head,
62
+ "selection" => Digest::SHA256.hexdigest(JSON.generate(scope.sort_by { |key, _| key.to_s }.to_h)),
63
+ "inventory" => Digest::SHA256.hexdigest(JSON.generate(report.results.map(&:digest).sort)),
64
+ "score" => overall_score(report), "coveredScore" => report.score,
65
+ "counts" => report.counts.transform_keys(&:to_s),
66
+ "breakdown" => report.score_breakdown.to_h.transform_keys(&:to_s),
67
+ "incomplete" => incompleteness(report) }
68
+ end
69
+
70
+ def overall_score(report)
71
+ score = report.score_breakdown
72
+ missing = report.results.count { |result| result.status == :skipped && result.reason == "no coverage" }
73
+ denominator = score.tested + missing
74
+ denominator.zero? ? nil : score.detected.fdiv(denominator)
75
+ end
76
+
77
+ def incompleteness(report)
78
+ reasons = Agent::Incomplete.reasons(report).map { |reason| reason[:reason] }
79
+ reasons << "baseline_incomplete" if report.run_evidence && !report.run_evidence["complete"]
80
+ reasons << "omissions" unless report.omissions.empty?
81
+ reasons.uniq
82
+ end
83
+
84
+ def lock(file, mode)
85
+ deadline = Process.clock_gettime(Process::CLOCK_MONOTONIC) + 2
86
+ until file.flock(mode | File::LOCK_NB)
87
+ if Process.clock_gettime(Process::CLOCK_MONOTONIC) >= deadline
88
+ raise Error,
89
+ "score history lock was not acquired"
90
+ end
91
+
92
+ sleep 0.01
93
+ end
94
+ end
95
+
96
+ def regular_file(file)
97
+ raise Error, "score history must be a regular file" unless file.stat.file?
98
+ end
99
+
100
+ def previous_record(file)
101
+ while (line = file.gets(MAX_RECORD_BYTES + 1))
102
+ raise Error, "score history record exceeds byte limit" if line.bytesize > MAX_RECORD_BYTES
103
+ raise Error, "score history contains a truncated record" unless line.end_with?("\n")
104
+
105
+ previous = HistoryRecord.parse(line)
106
+ end
107
+ previous
108
+ end
109
+
110
+ def change(previous, current)
111
+ return "score history: first record" unless previous
112
+ return DIFFERENT_RUN unless
113
+ previous["selection"] == current["selection"] && previous["inventory"] == current["inventory"] &&
114
+ previous["incomplete"].empty? && current["incomplete"].empty?
115
+ return MISSING_SCORE if
116
+ previous["coveredScore"].nil? || current["coveredScore"].nil?
117
+
118
+ delta = (current["coveredScore"] - previous["coveredScore"]) * 100
119
+ format("score history: covered score change %+.2f percentage points (same configured selection)", delta)
120
+ end
121
+ end
122
+ end
123
+ end
@@ -0,0 +1,57 @@
1
+ # SPDX-License-Identifier: MIT
2
+ # frozen_string_literal: true
3
+
4
+ require_relative "../score"
5
+
6
+ module Saboteur
7
+ module Reporter
8
+ # One row per source file, worst score first, with survivor counts, and a
9
+ # one-line legend for the statuses in the summary.
10
+ class ScoreTable
11
+ MAX_ROWS = 15
12
+ LEGEND = ["killed: a test failed survived: every selected test passed",
13
+ "timeout: exceeded its deadline",
14
+ "error, resource limit, skipped and the other statuses: not scored"].freeze
15
+
16
+ def initialize(out, palette)
17
+ @out = out
18
+ @palette = palette
19
+ end
20
+
21
+ def call(report)
22
+ rows = rows(report)
23
+ return if rows.size < 2
24
+
25
+ @out.puts "\nScore by file (worst first):"
26
+ width = rows.first(MAX_ROWS).map { |row| row[:path].length }.max
27
+ rows.first(MAX_ROWS).each { |row| @out.puts line(row, width) }
28
+ @out.puts " ... #{rows.size - MAX_ROWS} more files" if rows.size > MAX_ROWS
29
+ end
30
+
31
+ def legend
32
+ LEGEND.each_with_index.map do |text, index|
33
+ @palette.paint(:dim, "#{index.zero? ? 'legend: ' : ' '}#{text}")
34
+ end.join("\n")
35
+ end
36
+
37
+ private
38
+
39
+ def rows(report)
40
+ by_file = report.results.group_by { |result| result.subject.file }
41
+ rows = by_file.map { |file, results| row(report, file, results) }
42
+ rows.reject { |row| row[:tested].zero? }.sort_by { |row| [row[:value], -row[:survivors], row[:path]] }
43
+ end
44
+
45
+ def row(report, file, results)
46
+ breakdown = Score.of(results)
47
+ { path: file.delete_prefix("#{report.root}/"), value: breakdown.value,
48
+ survivors: results.count(&:unignored_survivor?), tested: breakdown.tested }
49
+ end
50
+
51
+ def line(row, width)
52
+ score = format("%5.1f %%", row[:value] * 100)
53
+ " #{row[:path].ljust(width)} #{score} #{row[:survivors]} survivor#{'s' unless row[:survivors] == 1}"
54
+ end
55
+ end
56
+ end
57
+ end
@@ -35,10 +35,14 @@ module Saboteur
35
35
  files: files,
36
36
  testFiles: test_files(test_ids),
37
37
  skippedSubjects: skipped_subjects(report),
38
+ earlyStop: early_stop(report),
38
39
  rejectedKills: report.rejected_kills,
39
40
  warnings: report.warnings,
40
41
  framework: { name: "saboteur", version: VERSION },
41
- saboteur: { findings: Findings.build(report), scores: scores(report) }
42
+ saboteur: { findings: Findings.build(report), scores: scores(report),
43
+ inlineSuppressions: report.inline_suppressions.map do |entry|
44
+ entry.merge(file: relative(entry[:file], report.root))
45
+ end }
42
46
  .merge(exit_cause ? { exitCause: exit_cause } : {})
43
47
  }.merge(report.seed ? { config: { seed: report.seed } } : {})
44
48
  end
@@ -49,6 +53,12 @@ module Saboteur
49
53
  { all: report.score_breakdown.to_h, withoutWholeBodyMutants: report.fine_score_breakdown.to_h }
50
54
  end
51
55
 
56
+ # nil for a run that covered its whole selection.
57
+ def self.early_stop(report)
58
+ stop = report.early_stop
59
+ stop && { reason: stop.reason, unreachedSubjects: stop.unreached }
60
+ end
61
+
52
62
  def self.skipped_subjects(report)
53
63
  report.skipped_subjects.map do |subject, reason|
54
64
  { subject: subject.id, file: relative(subject.file, report.root), line: subject.start_line, reason: reason }
@@ -101,7 +111,10 @@ module Saboteur
101
111
  when :survived then "Survived"
102
112
  when :timeout then "Timeout"
103
113
  when :error, :resource_limit then "RuntimeError"
104
- when :skipped then result.reason == "no coverage" ? "NoCoverage" : "Ignored"
114
+ when :skipped then result.reason&.start_with?("no coverage") ? "NoCoverage" : "Ignored"
115
+ # The schema has no status for "ran, but no test can observe a
116
+ # change here". Ignored with the reason is the honest mapping.
117
+ when :ran_at_load, :unreachable then "Ignored"
105
118
  end
106
119
  end
107
120
 
@@ -0,0 +1,68 @@
1
+ # SPDX-License-Identifier: MIT
2
+ # frozen_string_literal: true
3
+
4
+ module Saboteur
5
+ module Reporter
6
+ # Suggests a check that might kill a surviving mutant, by operator. It does
7
+ # not say what is missing: the survivor can also be an equivalent mutant.
8
+ module SurvivorHints
9
+ BY_OPERATOR = {
10
+ "comparison" => "try an example at the boundary: equal values, or values one apart",
11
+ "arithmetic" => "try asserting the exact computed number",
12
+ "boolean_logic" => "try an example where each side of the condition alone decides the result",
13
+ "negation_removal" => "try an example where the negated value changes the result",
14
+ "literal_number" => "try asserting something that depends on this number",
15
+ "literal_boolean" => "try an example that depends on this true/false value",
16
+ "literal_string" => "try asserting on this text",
17
+ "statement_removal" => "try asserting the effect of this statement",
18
+ "or_assign" => "try an example where the variable is already set (||= vs =)",
19
+ "safe_navigation" => "try an example where the receiver is nil",
20
+ "argument_removal" => "try have_received(...).with(...), or assert a result that depends on this argument",
21
+ "pair_removal" => "try have_received(...).with(...), or assert the hash content under this key",
22
+ "endpoint_replacement" => "try a collection with 2 or more elements, to tell first from last",
23
+ "filter_replacement" => "try items that match and items that do not, to tell select from reject"
24
+ }.freeze
25
+
26
+ COARSE = "try asserting this method's result or its effect"
27
+
28
+ # Printed once after the survivors: a hint is a place to look, not a diagnosis.
29
+ CAVEAT = "Hints are suggestions, not diagnoses: a survivor can also be an equivalent mutant, or a change " \
30
+ "the selected tests cannot observe."
31
+
32
+ # Comparison operator swaps where the two operators give the same answer
33
+ # except in one named case, so a test that hits that case can tell them
34
+ # apart. Anything else gets the plain text hint.
35
+ EQUAL_CASE = [%w[< <=], %w[> >=]].map(&:to_set).freeze
36
+ UNEQUAL_CASE = [%w[< >], %w[<= >=]].map(&:to_set).freeze
37
+
38
+ module_function
39
+
40
+ def for(operator, coarse: false)
41
+ BY_OPERATOR.fetch(operator) { coarse ? COARSE : nil }
42
+ end
43
+
44
+ # The hint as data for `--agent`: { kind:, text: } and, for a comparison
45
+ # swap, case: the relation between the two operands that separates the
46
+ # original from the mutant ("left == right"). It is where to look, not a
47
+ # promise that a test there will kill the mutant. nil when there is none.
48
+ def structured(operator:, original:, replacement:, coarse: false)
49
+ text = self.for(operator, coarse: coarse)
50
+ return unless text
51
+
52
+ operands = boundary_case(operator, original, replacement)
53
+ return { kind: "boundary", case: operands, text: text } if operands
54
+
55
+ { kind: coarse ? "method_result" : "assertion", text: text }
56
+ end
57
+
58
+ def boundary_case(operator, original, replacement)
59
+ return unless operator == "comparison"
60
+
61
+ pair = Set[original, replacement]
62
+ return "left == right" if EQUAL_CASE.include?(pair)
63
+
64
+ "left != right" if UNEQUAL_CASE.include?(pair)
65
+ end
66
+ end
67
+ end
68
+ end
@@ -0,0 +1,126 @@
1
+ # SPDX-License-Identifier: MIT
2
+ # frozen_string_literal: true
3
+
4
+ require_relative "mutation_text"
5
+ require_relative "survivor_hints"
6
+
7
+ module Saboteur
8
+ module Reporter
9
+ # Survivors grouped by file, then method, then line. The source line and
10
+ # the tests that covered it are printed once; each mutation is one compact
11
+ # line: operator, original -> replacement, digest.
12
+ class SurvivorList
13
+ MAX_PASSING_TESTS = 5
14
+
15
+ def initialize(out, palette)
16
+ @out = out
17
+ @palette = palette
18
+ end
19
+
20
+ # Every survivor, in report order; the grouping keeps them readable.
21
+ def call(report, survivors)
22
+ return if survivors.empty?
23
+
24
+ @out.puts "Survivors (#{survivors.size}):"
25
+ survivors.group_by { |result| result.subject.file }.sort_by { |file, _| file }.each do |file, in_file|
26
+ file_section(report, relative(file, report.root), in_file)
27
+ end
28
+ @out.puts "\n#{@palette.paint(:dim, SurvivorHints::CAVEAT)}" if survivors.any? { |result| hint_for(result) }
29
+ end
30
+
31
+ private
32
+
33
+ def file_section(report, path, in_file)
34
+ @out.puts "\n#{@palette.paint(:bold, path)}"
35
+ in_file.group_by { |result| result.subject.id }.each do |subject_id, in_subject|
36
+ @out.puts " #{subject_id}"
37
+ lines(report, path, in_subject)
38
+ end
39
+ end
40
+
41
+ def lines(report, path, in_subject)
42
+ in_subject.group_by { |result| MutationText.line(result) }.sort.each do |line, on_line|
43
+ first = on_line.first
44
+ @out.puts " #{path}:#{line} in #{first.subject.id}"
45
+ @out.puts " #{@palette.paint(:dim, "#{line}:")} #{source_line(first, line)}"
46
+ evidence_groups(report, on_line).each { |group| evidence_group(report, path, group) }
47
+ end
48
+ end
49
+
50
+ # Mutants on one line share the printed evidence only when it is the
51
+ # same for all of them: covering tests (in order), reuse and changed
52
+ # lines can each differ per mutant, so each run of neighbours with the
53
+ # same tuple is one group and shares evidence above its own mutations.
54
+ # Evidence that repeats after another group is printed again, which
55
+ # keeps the ranked order.
56
+ def evidence_groups(report, on_line)
57
+ on_line.chunk { |result| [result.covered_by, result.reused?, report.changed_lines_for(result)] }
58
+ .map(&:last)
59
+ end
60
+
61
+ def evidence_group(report, path, group)
62
+ first = group.first
63
+ changed_lines(report, path, first)
64
+ passing_tests(first)
65
+ group.each { |result| mutation(result) }
66
+ end
67
+
68
+ def source_line(result, line)
69
+ result.subject.file_source.lines[line - 1].to_s.strip
70
+ end
71
+
72
+ def mutation(result)
73
+ text = " #{@palette.paint(:red, '-')} #{result.mutation.operator} " \
74
+ "#{MutationText.original(result)} -> #{@palette.paint(:green, MutationText.replacement(result))} " \
75
+ "#{result.digest}"
76
+ text += " (ignored: #{result.ignored})" if result.ignored
77
+ @out.puts text
78
+ hint(result)
79
+ end
80
+
81
+ def hint(result)
82
+ text = hint_for(result)
83
+ @out.puts " #{@palette.paint(:cyan, 'hint:')} #{text}" if text
84
+ end
85
+
86
+ def hint_for(result)
87
+ return nil if result.ignored
88
+
89
+ SurvivorHints.for(result.mutation.operator, coarse: result.mutation.coarse)
90
+ end
91
+
92
+ # The tests that ran with the mutant applied and still passed: the
93
+ # missing assertion belongs in one of them. They are listed likeliest
94
+ # killers first, the order the run used. Line coverage can overstate which
95
+ # tests exercise the line, so the list is a lead, not proof.
96
+ def passing_tests(result)
97
+ tests = result.covered_by
98
+ return if tests.empty?
99
+
100
+ @out.puts " #{tests.size} #{tests.one? ? 'test' : 'tests'} covered this line and " \
101
+ "#{passed_phrase(result)} (the missing check may belong in one of them):"
102
+ tests.first(MAX_PASSING_TESTS).each { |name| @out.puts " #{name}" }
103
+ more = tests.size - MAX_PASSING_TESTS
104
+ @out.puts " ... #{more} more in the JSON report (coveredBy)" if more.positive?
105
+ end
106
+
107
+ def passed_phrase(result)
108
+ result.reused? ? "passed in an earlier run (--incremental)" : "still passed"
109
+ end
110
+
111
+ # In diff mode: the changed lines this mutant touches.
112
+ def changed_lines(report, path, result)
113
+ lines = report.changed_lines_for(result)
114
+ return if lines.empty?
115
+
116
+ shown = lines.first(3).map { |line| "#{path}:#{line}" }.join(", ")
117
+ more = lines.size - 3
118
+ @out.puts " changed #{lines.one? ? 'line' : 'lines'}: #{shown}#{" (+#{more} more)" if more.positive?}"
119
+ end
120
+
121
+ def relative(file, root)
122
+ file.delete_prefix("#{root}/")
123
+ end
124
+ end
125
+ end
126
+ end