saboteur 0.6.0 → 0.8.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (133) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +1024 -0
  3. data/README.md +199 -952
  4. data/exe/saboteur +34 -7
  5. data/lib/saboteur/baseline/workflow.rb +2 -1
  6. data/lib/saboteur/baseline_health.rb +40 -6
  7. data/lib/saboteur/cli/exit_status.rb +80 -4
  8. data/lib/saboteur/cli/help_options.rb +17 -0
  9. data/lib/saboteur/cli/history_command.rb +135 -0
  10. data/lib/saboteur/cli/presets.rb +40 -0
  11. data/lib/saboteur/cli/run_budget_options.rb +37 -0
  12. data/lib/saboteur/cli/run_reporting.rb +74 -5
  13. data/lib/saboteur/cli/selection_options.rb +31 -0
  14. data/lib/saboteur/cli/shard_options.rb +17 -0
  15. data/lib/saboteur/cli/source_set.rb +53 -0
  16. data/lib/saboteur/cli/survivors_command.rb +157 -0
  17. data/lib/saboteur/cli.rb +136 -98
  18. data/lib/saboteur/config.rb +163 -15
  19. data/lib/saboteur/config_file.rb +8 -0
  20. data/lib/saboteur/constant_reads.rb +182 -0
  21. data/lib/saboteur/constant_subject.rb +70 -0
  22. data/lib/saboteur/constant_subject_finder.rb +281 -0
  23. data/lib/saboteur/coverage_reads.rb +29 -0
  24. data/lib/saboteur/crash_report.rb +154 -0
  25. data/lib/saboteur/custom_map.rb +3 -1
  26. data/lib/saboteur/dirty_tree_warning.rb +128 -0
  27. data/lib/saboteur/doctor.rb +49 -1
  28. data/lib/saboteur/engine/baseline_guard.rb +136 -14
  29. data/lib/saboteur/engine/boot.rb +16 -1
  30. data/lib/saboteur/engine/canary.rb +41 -10
  31. data/lib/saboteur/engine/candidate_selection.rb +28 -1
  32. data/lib/saboteur/engine/candidates.rb +43 -3
  33. data/lib/saboteur/engine/coverage_guard.rb +29 -0
  34. data/lib/saboteur/engine/coverage_store.rb +79 -15
  35. data/lib/saboteur/engine/defined_order_prefix.rb +43 -0
  36. data/lib/saboteur/engine/diagnostics.rb +41 -2
  37. data/lib/saboteur/engine/diff_scope.rb +24 -5
  38. data/lib/saboteur/engine/distribution.rb +134 -8
  39. data/lib/saboteur/engine/early_stop_recording.rb +47 -0
  40. data/lib/saboteur/engine/gated_run.rb +35 -0
  41. data/lib/saboteur/engine/load_coverage.rb +45 -0
  42. data/lib/saboteur/engine/out_of_scope_guard.rb +21 -0
  43. data/lib/saboteur/engine/pipelines.rb +107 -0
  44. data/lib/saboteur/engine/result_cache.rb +67 -9
  45. data/lib/saboteur/engine/run_budget.rb +90 -0
  46. data/lib/saboteur/engine/run_fingerprint.rb +4 -0
  47. data/lib/saboteur/engine/sandbox_policy.rb +56 -11
  48. data/lib/saboteur/engine/scheduler.rb +222 -20
  49. data/lib/saboteur/engine/selection_announcement.rb +54 -0
  50. data/lib/saboteur/engine/shard.rb +69 -0
  51. data/lib/saboteur/engine/subject_selection.rb +373 -19
  52. data/lib/saboteur/engine/suite_files.rb +3 -1
  53. data/lib/saboteur/engine/test_guess.rb +438 -25
  54. data/lib/saboteur/engine/thread_warnings.rb +53 -0
  55. data/lib/saboteur/engine/unchanged_syntax.rb +17 -2
  56. data/lib/saboteur/engine/unscoped_suite_warning.rb +41 -0
  57. data/lib/saboteur/engine/verdict_check.rb +54 -18
  58. data/lib/saboteur/engine.rb +202 -158
  59. data/lib/saboteur/execution/external_runner.rb +471 -0
  60. data/lib/saboteur/execution/marker_sweep.rb +79 -0
  61. data/lib/saboteur/execution/peak_memory.rb +28 -0
  62. data/lib/saboteur/execution/process_table.rb +12 -0
  63. data/lib/saboteur/execution/process_tree.rb +65 -5
  64. data/lib/saboteur/execution/protocol.rb +2 -1
  65. data/lib/saboteur/execution/reevaluator.rb +68 -12
  66. data/lib/saboteur/execution/runner.rb +225 -24
  67. data/lib/saboteur/execution/sandbox.rb +21 -3
  68. data/lib/saboteur/execution/seatbelt.rb +226 -0
  69. data/lib/saboteur/execution/supervisor.rb +45 -13
  70. data/lib/saboteur/execution/worker_pool.rb +110 -8
  71. data/lib/saboteur/frameworks/sqlite_memory_database.rb +49 -0
  72. data/lib/saboteur/gates.rb +43 -2
  73. data/lib/saboteur/help.rb +108 -13
  74. data/lib/saboteur/ignore_list.rb +40 -7
  75. data/lib/saboteur/install.rb +4 -3
  76. data/lib/saboteur/integration/base.rb +7 -2
  77. data/lib/saboteur/integration/rspec.rb +175 -9
  78. data/lib/saboteur/mutation_source/argument_removal.rb +1 -1
  79. data/lib/saboteur/mutation_source/builtin.rb +44 -14
  80. data/lib/saboteur/mutation_source/chain_removal.rb +1 -1
  81. data/lib/saboteur/mutation_source/condition_negation.rb +1 -1
  82. data/lib/saboteur/mutation_source/constant_literals.rb +66 -0
  83. data/lib/saboteur/mutation_source/element_removal.rb +73 -0
  84. data/lib/saboteur/mutation_source/endpoint_replacement.rb +1 -1
  85. data/lib/saboteur/mutation_source/entry_edits.rb +44 -0
  86. data/lib/saboteur/mutation_source/filter_replacement.rb +1 -1
  87. data/lib/saboteur/mutation_source/hazard_guard.rb +129 -3
  88. data/lib/saboteur/mutation_source/interpolation_removal.rb +41 -0
  89. data/lib/saboteur/mutation_source/literal_replacement.rb +1 -1
  90. data/lib/saboteur/mutation_source/method_swap.rb +193 -0
  91. data/lib/saboteur/mutation_source/mutation.rb +5 -3
  92. data/lib/saboteur/mutation_source/operand_replacement.rb +54 -0
  93. data/lib/saboteur/mutation_source/operator_assignment.rb +1 -1
  94. data/lib/saboteur/mutation_source/operator_examples.rb +6 -1
  95. data/lib/saboteur/mutation_source/operator_registry.rb +7 -2
  96. data/lib/saboteur/mutation_source/operator_visit.rb +39 -0
  97. data/lib/saboteur/mutation_source/pair_removal.rb +13 -31
  98. data/lib/saboteur/mutation_source/range_replacement.rb +1 -1
  99. data/lib/saboteur/mutation_source/regex_replacement.rb +93 -0
  100. data/lib/saboteur/preview.rb +122 -13
  101. data/lib/saboteur/refinement_activations.rb +60 -0
  102. data/lib/saboteur/reporter/agent.rb +9 -1
  103. data/lib/saboteur/reporter/agent_commands.rb +47 -15
  104. data/lib/saboteur/reporter/agent_incomplete.rb +38 -2
  105. data/lib/saboteur/reporter/agent_summary.rb +4 -3
  106. data/lib/saboteur/reporter/findings.rb +5 -2
  107. data/lib/saboteur/reporter/history_policy.rb +11 -0
  108. data/lib/saboteur/reporter/history_reader.rb +47 -0
  109. data/lib/saboteur/reporter/html.rb +26 -4
  110. data/lib/saboteur/reporter/mutation_text.rb +6 -0
  111. data/lib/saboteur/reporter/next_steps.rb +14 -4
  112. data/lib/saboteur/reporter/score_history.rb +5 -1
  113. data/lib/saboteur/reporter/stryker.rb +40 -7
  114. data/lib/saboteur/reporter/survivor_classification.rb +101 -0
  115. data/lib/saboteur/reporter/survivor_hints.rb +3 -0
  116. data/lib/saboteur/reporter/survivor_list.rb +15 -9
  117. data/lib/saboteur/reporter/terminal.rb +74 -17
  118. data/lib/saboteur/result.rb +38 -7
  119. data/lib/saboteur/selection/coverage_map.rb +107 -11
  120. data/lib/saboteur/selection/dependency_shield.rb +80 -0
  121. data/lib/saboteur/selection/external_test_map.rb +27 -0
  122. data/lib/saboteur/selection/git_diff.rb +105 -13
  123. data/lib/saboteur/selection/layout_only_lines.rb +80 -0
  124. data/lib/saboteur/selection/recorder.rb +22 -4
  125. data/lib/saboteur/selection/source_scope.rb +22 -0
  126. data/lib/saboteur/selection/untracked_changes.rb +126 -0
  127. data/lib/saboteur/subject.rb +53 -7
  128. data/lib/saboteur/subject_finder.rb +173 -29
  129. data/lib/saboteur/text.rb +42 -0
  130. data/lib/saboteur/version.rb +1 -1
  131. data/lib/saboteur/working_tree.rb +38 -15
  132. data/lib/saboteur.rb +29 -1
  133. metadata +50 -2
@@ -4,6 +4,8 @@
4
4
  require "cgi"
5
5
  require "fileutils"
6
6
 
7
+ require_relative "survivor_classification"
8
+
7
9
  module Saboteur
8
10
  module Reporter
9
11
  # One self-contained HTML page: inline style, no script, no network. It is
@@ -23,7 +25,9 @@ module Saboteur
23
25
  .status{border-radius:3px;padding:0 .4rem;color:#fff;white-space:nowrap}
24
26
  .Killed{background:#2e7d32}.Survived{background:#c62828}.Timeout{background:#b45309}
25
27
  .RuntimeError{background:#6a1b9a}.NoCoverage{background:#455a64}.Ignored{background:#616161}
26
- .Accepted{background:#8d6e63}
28
+ .Accepted,.Triage_accepted_equivalent{background:#8d6e63}
29
+ .Triage_wont_fix{background:#616161}.Triage_todo{background:#b45309}
30
+ .classification{border-radius:3px;padding:0 .4rem;background:#eceff1;color:#37474f;white-space:nowrap}
27
31
  .warning{background:#fff8e1;border-left:4px solid #f9a825;margin:.3rem 0;padding:.3rem .6rem;overflow-wrap:anywhere}
28
32
  code,pre{font-family:ui-monospace,monospace;font-size:13px}
29
33
  pre{display:inline-block;vertical-align:top;margin:0;white-space:pre-wrap;overflow-wrap:anywhere}summary{cursor:pointer;font-weight:600;overflow-wrap:anywhere}
@@ -195,12 +199,30 @@ module Saboteur
195
199
  def row_status(mutant, result)
196
200
  badge = status_label(mutant[:status])
197
201
  # Only survivors are ever put on the ignore list.
198
- result.ignored ? "#{badge} #{status_label('Accepted')}" : badge
202
+ if result.ignored || result.triage_state
203
+ # An entry without a state predates triage states: render Accepted.
204
+ badge = if result.triage_state.nil?
205
+ "#{badge} #{status_label('Accepted')}"
206
+ else
207
+ "#{badge} #{status_label(triage_badge(result), css_class: "Triage_#{result.triage_state}")}"
208
+ end
209
+ end
210
+ # The survivor's classification, as a label only (#192).
211
+ label = SurvivorClassification.label(result)
212
+ label ? "#{badge} <span class=\"classification\">#{escape(label)}</span>" : badge
199
213
  end
200
214
 
201
215
  # One edit shows the original text and its replacement. A mutant with
202
216
  # several edits shows only the replacements, because one location cannot
203
217
  # stand for all of them.
218
+ def triage_badge(result)
219
+ case result.triage_state
220
+ when "wont_fix" then "Won\u2019t fix"
221
+ when "todo" then "Todo"
222
+ else "Accepted"
223
+ end
224
+ end
225
+
204
226
  def change(mutant, result)
205
227
  edits = result.mutation.edits
206
228
  return "<pre>#{escape(mutant[:replacement])}</pre>" unless edits.size == 1
@@ -212,8 +234,8 @@ module Saboteur
212
234
  "<pre>#{escape(edit.replacement)}</pre>"
213
235
  end
214
236
 
215
- def status_label(status)
216
- %(<span class="status #{status}">#{status}</span>)
237
+ def status_label(status, css_class: status)
238
+ %(<span class="status #{css_class}">#{status}</span>)
217
239
  end
218
240
 
219
241
  def escape(text)
@@ -20,6 +20,12 @@ module Saboteur
20
20
  join(result, &:replacement)
21
21
  end
22
22
 
23
+ # A mutant that only deletes: every edit's replacement is empty.
24
+ # Rendered as "removes X" rather than "X -> " with a blank side.
25
+ def removal?(result)
26
+ sorted_edits(result).all? { |edit| edit.replacement.empty? }
27
+ end
28
+
23
29
  def sorted_edits(result)
24
30
  result.mutation.edits.sort_by(&:start_byte)
25
31
  end
@@ -7,9 +7,14 @@ module Saboteur
7
7
  # accept one, and how to hand the run to an agent. Silent when nothing
8
8
  # survived, so a clean run stays short.
9
9
  class NextSteps
10
- def initialize(out, palette)
10
+ # replay_command: digest -> the full, copy-pasteable replay command
11
+ # carrying the run's own selection options, however long it gets
12
+ # (#261). When no builder is wired the digest-only hint (with its
13
+ # reassembly note) is the fallback.
14
+ def initialize(out, palette, replay_command: nil)
11
15
  @out = out
12
16
  @palette = palette
17
+ @replay_command = replay_command
13
18
  end
14
19
 
15
20
  def call(report)
@@ -17,9 +22,14 @@ module Saboteur
17
22
  return unless survivor
18
23
 
19
24
  @out.puts "\n#{@palette.paint(:bold, 'Next steps:')}"
20
- seed = " --seed #{report.seed}" if report.seed
21
- @out.puts " replay one survivor: saboteur run --id #{survivor.digest}#{seed}"
22
- @out.puts " (add this run's own --config, --since, --subject, --test and source options)"
25
+ command = @replay_command&.call(survivor.digest)
26
+ if command
27
+ @out.puts " replay one survivor: #{command}"
28
+ else
29
+ seed = " --seed #{report.seed}" if report.seed
30
+ @out.puts " replay one survivor: saboteur run --id #{survivor.digest}#{seed}"
31
+ @out.puts " (add this run's own --config, --since, --subject, --test and source options)"
32
+ end
23
33
  @out.puts " accept an equivalent one: add to your ignore file (c.ignore_file, default #{Config::IGNORE_FILE}):"
24
34
  entry = %(i.mutation "#{survivor.digest}", subject: "#{survivor.subject.id}", reason: "why it is equivalent")
25
35
  @out.puts " #{entry}"
@@ -39,7 +39,11 @@ module Saboteur
39
39
  file.flush
40
40
  change(previous, record)
41
41
  end
42
- rescue SystemCallError, IOError, JSON::ParserError, KeyError, TypeError => e
42
+ # SystemCallError/IOError from the write itself propagate unwrapped on
43
+ # purpose: the CLI treats those as a soft artifact failure (warn and
44
+ # exit by the results), while validation, integrity, record-size and
45
+ # lock failures stay hard errors (#296).
46
+ rescue JSON::ParserError, KeyError, TypeError => e
43
47
  raise Error, "score history was not recorded: #{e.message}"
44
48
  end
45
49
 
@@ -39,12 +39,24 @@ module Saboteur
39
39
  rejectedKills: report.rejected_kills,
40
40
  warnings: report.warnings,
41
41
  framework: { name: "saboteur", version: VERSION },
42
- saboteur: { findings: Findings.build(report), scores: scores(report),
43
- inlineSuppressions: inline_audit(report) }
44
- .merge(operatorPolicy: report.operator_policy).merge(exit_cause ? { exitCause: exit_cause } : {})
42
+ saboteur: saboteur_enrichment(report, exit_cause)
45
43
  }.merge(report.seed ? { config: { seed: report.seed } } : {})
46
44
  end
47
45
 
46
+ # saboteur-namespaced keys beyond Stryker's schema: findings and
47
+ # scores, then the optional policy, shard, exit-cause and memory keys.
48
+ def self.saboteur_enrichment(report, exit_cause)
49
+ { findings: Findings.build(report), scores: scores(report),
50
+ inlineSuppressions: inline_audit(report) }
51
+ .merge(operatorPolicy: report.operator_policy).merge(shard(report))
52
+ .merge(exit_cause ? { exitCause: exit_cause } : {})
53
+ .merge(report.memory ? { memory: report.memory } : {})
54
+ end
55
+
56
+ def self.shard(report)
57
+ report.shard ? { shard: report.shard.to_h } : {}
58
+ end
59
+
48
60
  def self.inline_audit(report)
49
61
  report.inline_suppressions.map { |entry| entry.merge(file: relative(entry[:file], report.root)) }
50
62
  end
@@ -58,7 +70,14 @@ module Saboteur
58
70
  # nil for a run that covered its whole selection.
59
71
  def self.early_stop(report)
60
72
  stop = report.early_stop
61
- stop && { reason: stop.reason, unreachedSubjects: stop.unreached }
73
+ return unless stop
74
+
75
+ { reason: stop.reason, unreachedSubjects: stop.unreached, truncatedSubjects: truncated_subjects(stop),
76
+ unverifiedSubjects: stop.unverified }
77
+ end
78
+
79
+ def self.truncated_subjects(stop)
80
+ stop.truncated.map { |entry| { subject: entry[:subject], omittedMutants: entry[:omitted] } }
62
81
  end
63
82
 
64
83
  def self.skipped_subjects(report)
@@ -100,7 +119,14 @@ module Saboteur
100
119
 
101
120
  def self.optional_fields(result, test_ids)
102
121
  {
122
+ # A rejected kill is a verdict, not a tool error: the tests failed
123
+ # but also fail with the method unchanged. Flag it per mutant;
124
+ # the top-level rejectedKills stays a bare count (#375).
125
+ rejectedKill: (true if result.rejected_kill?),
103
126
  statusReason: status_reason(result),
127
+ # Why the survivor is worth a look, as a label only: scores and
128
+ # the exit code never change (#192).
129
+ classification: SurvivorClassification.label(result),
104
130
  killedBy: result.killed_by.map { |name| test_ids.fetch(name) }.then { |ids| ids.empty? ? nil : ids },
105
131
  coveredBy: result.covered_by.map { |name| test_ids.fetch(name) }.then { |ids| ids.empty? ? nil : ids },
106
132
  duration: result.runtime && (result.runtime * 1000).round
@@ -115,17 +141,24 @@ module Saboteur
115
141
  when :error, :resource_limit then "RuntimeError"
116
142
  when :skipped then result.reason&.start_with?("no coverage") ? "NoCoverage" : "Ignored"
117
143
  # The schema has no status for "ran, but no test can observe a
118
- # change here". Ignored with the reason is the honest mapping.
119
- when :ran_at_load, :unreachable then "Ignored"
144
+ # change here", nor for a verdict the budget left unchecked.
145
+ # Ignored with the reason is the honest mapping.
146
+ when :ran_at_load, :unreachable, :unverified then "Ignored"
120
147
  end
121
148
  end
122
149
 
123
150
  def self.status_reason(result)
151
+ return "todo: #{result.triage_reason}" if result.triage_state == "todo"
152
+ # The "ignored: " prefix is a public contract consumers classify by
153
+ # (#304 CI): accepted_equivalent stays byte-identical to a plain
154
+ # ignore, and only wont_fix adds a suffix after the reason.
155
+ return "ignored: #{result.ignored} (wont_fix)" if result.triage_state == "wont_fix"
124
156
  return "ignored: #{result.ignored}" if result.ignored
125
157
  return "resource limit: #{result.reason}" if result.status == :resource_limit
126
158
  return result.crash_kill? ? "crash: #{result.kill_cause}" : result.kill_cause if result.kill_cause
127
159
 
128
- result.reason.to_s.lines.first&.strip
160
+ reason = result.reason.to_s.strip
161
+ reason.empty? ? nil : reason
129
162
  end
130
163
 
131
164
  def self.location(edit, source)
@@ -0,0 +1,101 @@
1
+ # SPDX-License-Identifier: MIT
2
+ # frozen_string_literal: true
3
+
4
+ require_relative "../engine/bytecode_equivalence"
5
+ require_relative "../execution/external_runner"
6
+ require_relative "../mutation_source/builtin"
7
+
8
+ module Saboteur
9
+ module Reporter
10
+ # Why a survivor is worth a look, as a LABEL ONLY (#192): the label
11
+ # lands in reports so a reader can sort real gaps from noise. Every
12
+ # label is a HEURISTIC, not a diagnosis - inspect the survivor before
13
+ # acting on it, and never add a test or ignore a mutant from the label
14
+ # alone. Labels never change scores or the exit code.
15
+ #
16
+ # One label per survivor, first match wins:
17
+ # likely_equivalent - the mutant compiles to the same instructions as
18
+ # the original (the #151/#169 bytecode filter): strong evidence of
19
+ # a no-op mutant, worth checking against the source. First: a
20
+ # no-op mutant's survival says nothing about the suite.
21
+ # arid - every line an edit touches matches an arid pattern
22
+ # (logging and friends, #111/#187). The builtin hint list decides;
23
+ # a configured arid policy does not change the label.
24
+ # missing_test - a test covered the line and passed, which MAY mean
25
+ # the suite reaches the code without pinning this behaviour. The
26
+ # covering list is only the PLANNED coverage: whole-body (coarse)
27
+ # mutants carry method-level guesses, so they never get this
28
+ # label, and neither does anything without real coverage.
29
+ # tool_limit - reserved for survivors saboteur could not observe
30
+ # faithfully (re-eval refusal, external runner without coverage). No
31
+ # result records that signal yet, so the label is never emitted.
32
+ #
33
+ # A survivor with no matching signal stays unlabeled.
34
+ module SurvivorClassification
35
+ LABELS = %w[likely_equivalent arid missing_test tool_limit].freeze
36
+
37
+ class << self
38
+ def label(result, arid: MutationSource::Builtin::DEFAULT_ARID)
39
+ return nil unless result.status == :survived
40
+ return "likely_equivalent" if equivalent?(result)
41
+ return "arid" if arid?(result, arid)
42
+ return "missing_test" if possible_test_gap?(result)
43
+
44
+ nil
45
+ end
46
+
47
+ private
48
+
49
+ # The mutant's own def text, compiled side by side with the
50
+ # original: equal instruction sequences mean behaviour cannot
51
+ # change. Edits index the whole file, so they shift by the def's
52
+ # start byte before splicing into the def text.
53
+ def equivalent?(result)
54
+ subject = result.subject
55
+ original = subject.source
56
+ mutated = result.mutation.edits.sort_by(&:start_byte).reverse.inject(original) do |text, edit|
57
+ from = edit.start_byte - subject.start_byte
58
+ to = edit.end_byte - subject.start_byte
59
+ text.byteslice(0, from) + edit.replacement + text.byteslice(to..)
60
+ end
61
+ Engine::BytecodeEquivalence.new.equivalent?(original, mutated,
62
+ frozen_string_literal: subject.frozen_literals)
63
+ end
64
+
65
+ # covered_by is a PLANNED covering list, not a verdict about what
66
+ # the suite asserts: coarse (whole-body) mutants carry method-level
67
+ # guesses, and a --test-command run records only the pseudo-test id
68
+ # for every mutant, so neither reads as real coverage (#305 QA).
69
+ def possible_test_gap?(result)
70
+ real_coverage = result.covered_by - [Execution::ExternalRunner::PSEUDO_TEST]
71
+ !result.mutation.coarse && real_coverage.any?
72
+ end
73
+
74
+ # Arid only when EVERY line an edit touches reads as arid: an edit
75
+ # that starts on a logging line but removes a composite body is not
76
+ # arid. The operator never decides: removing a logging statement is
77
+ # arid whatever removed it.
78
+ def arid?(result, patterns)
79
+ result.mutation.edits.all? do |edit|
80
+ lines = edited_lines(result.subject, edit)
81
+ lines.any? && lines.all? do |line|
82
+ patterns.any? { |pattern| line.strip.match?(Regexp.new(pattern)) }
83
+ end
84
+ end
85
+ end
86
+
87
+ # Edit offsets are BYTES, so the line lookup runs on the binary
88
+ # view (index/rindex are byte-exact there) and only the extracted
89
+ # lines are decoded, scrubbed so a partial multibyte sequence at an
90
+ # edit boundary cannot raise on the pattern match.
91
+ def edited_lines(subject, edit)
92
+ bytes = subject.file_source.b
93
+ last = [edit.end_byte - 1, edit.start_byte].max
94
+ line_start = (bytes.byteslice(0, edit.start_byte).rindex("\n") || -1) + 1
95
+ line_end = bytes.index("\n", last) || bytes.bytesize
96
+ bytes.byteslice(line_start, line_end - line_start).force_encoding(Encoding::UTF_8).scrub.lines
97
+ end
98
+ end
99
+ end
100
+ end
101
+ end
@@ -19,6 +19,9 @@ module Saboteur
19
19
  "safe_navigation" => "try an example where the receiver is nil",
20
20
  "argument_removal" => "try have_received(...).with(...), or assert a result that depends on this argument",
21
21
  "pair_removal" => "try have_received(...).with(...), or assert the hash content under this key",
22
+ "chain_removal" => "try the shorter chain's value against the longer one; if you confirm the " \
23
+ "shorter chain is equivalent across the intended inputs (for example a duplicated " \
24
+ "column), simplify the code or ignore the mutant",
22
25
  "endpoint_replacement" => "try a collection with 2 or more elements, to tell first from last",
23
26
  "filter_replacement" => "try items that match and items that do not, to tell select from reject"
24
27
  }.freeze
@@ -47,14 +47,16 @@ module Saboteur
47
47
  end
48
48
  end
49
49
 
50
- # Mutants on one line share the printed evidence only when it is the
51
- # same for all of them: covering tests (in order), reuse and changed
52
- # lines can each differ per mutant, so each run of neighbours with the
53
- # same tuple is one group and shares evidence above its own mutations.
54
- # Evidence that repeats after another group is printed again, which
55
- # keeps the ranked order.
50
+ # Mutants on one line share the printed evidence when it names the
51
+ # same tests: the covering-test list is compared as a set (#502), so
52
+ # one line's mutants no longer reprint an identical block only because
53
+ # the run measured the killer order per mutant. Reuse and changed
54
+ # lines still group exactly. Each run of neighbours with the same key
55
+ # is one group and shares evidence above its own mutations; evidence
56
+ # that repeats after another group is printed again, which keeps the
57
+ # ranked order. The block prints in the first mutant's run order.
56
58
  def evidence_groups(report, on_line)
57
- on_line.chunk { |result| [result.covered_by, result.reused?, report.changed_lines_for(result)] }
59
+ on_line.chunk { |result| [result.covered_by.uniq.sort, result.reused?, report.changed_lines_for(result)] }
58
60
  .map(&:last)
59
61
  end
60
62
 
@@ -70,9 +72,13 @@ module Saboteur
70
72
  end
71
73
 
72
74
  def mutation(result)
75
+ change = if MutationText.removal?(result)
76
+ "removes #{MutationText.original(result)}"
77
+ else
78
+ "#{MutationText.original(result)} -> #{@palette.paint(:green, MutationText.replacement(result))}"
79
+ end
73
80
  text = " #{@palette.paint(:red, '-')} #{result.mutation.operator} " \
74
- "#{MutationText.original(result)} -> #{@palette.paint(:green, MutationText.replacement(result))} " \
75
- "#{result.digest}"
81
+ "#{change} #{result.digest}"
76
82
  text += " (ignored: #{result.ignored})" if result.ignored
77
83
  @out.puts text
78
84
  hint(result)
@@ -17,12 +17,12 @@ module Saboteur
17
17
  MAX_ERRORS = 10
18
18
  MAX_PASSING_TESTS = SurvivorList::MAX_PASSING_TESTS
19
19
 
20
- def initialize(out, verbose: false, color: :auto)
20
+ def initialize(out, verbose: false, color: :auto, replay_command: nil)
21
21
  @out = out
22
22
  @verbose = verbose
23
23
  @palette = Palette.for(out, mode: color)
24
24
  @score_table = ScoreTable.new(out, @palette)
25
- @next_steps = NextSteps.new(out, @palette)
25
+ @next_steps = NextSteps.new(out, @palette, replay_command: replay_command)
26
26
  end
27
27
 
28
28
  def call(report)
@@ -91,6 +91,7 @@ module Saboteur
91
91
  seed = report.seed && ", seed #{report.seed} (repeat with --seed #{report.seed})"
92
92
  @out.puts "saboteur #{VERSION}: #{report.subjects.size} subjects, #{report.results.size} mutants, " \
93
93
  "wall #{format('%.2f', report.wall_time)} s#{seed}"
94
+ @out.puts report.shard.line if report.shard
94
95
  @out.puts status_line(report)
95
96
  details(report)
96
97
  @score_table.call(report)
@@ -103,12 +104,15 @@ module Saboteur
103
104
  survived = "survived #{c[:survived]}"
104
105
  ignored = report.survivors.size - report.unignored_survivors.size
105
106
  survived += " (#{ignored} ignored)" if ignored.positive?
107
+ rejected = report.rejected_kills
108
+ rejected_note = " (#{rejected} rejected kill#{'s' if rejected > 1})" if rejected.positive?
106
109
  line = "#{@palette.paint(:green,
107
110
  "killed #{c[:killed]}")} #{@palette.paint(:red,
108
111
  survived)} timeout #{c[:timeout]} " \
109
- "resource limit #{c[:resource_limit]} error #{c[:error]} skipped #{c[:skipped]}"
112
+ "resource limit #{c[:resource_limit]} error #{c[:error]}#{rejected_note} skipped #{c[:skipped]}"
110
113
  line += " ran at load #{c[:ran_at_load]}" if c[:ran_at_load].positive?
111
114
  line += " unreachable #{c[:unreachable]}" if c[:unreachable].positive?
115
+ line += " unverified #{c[:unverified]}" if c[:unverified].positive?
112
116
  line
113
117
  end
114
118
 
@@ -121,11 +125,19 @@ module Saboteur
121
125
 
122
126
  def details(report)
123
127
  breakdown = report.score_breakdown
124
- @out.puts "score #{percent(breakdown.value)}" \
125
- "#{if @verbose
126
- "#{score_split(breakdown)} (killed+timeout)/(killed+survived+timeout); " \
127
- 'errors, resource limits and skips excluded'
128
- end}"
128
+ # The share covers scored mutants only, so an error-dominated run
129
+ # must not print it as a bare score: "score 100.0 %" above "the run
130
+ # tested almost nothing" reads as a contradiction (#557).
131
+ if report.majority_errored?
132
+ @out.puts "score n/a (#{breakdown.tested} of #{report.executed.size} mutants scored, " \
133
+ "#{percent(breakdown.value)} of scored)"
134
+ else
135
+ @out.puts "score #{percent(breakdown.value)}" \
136
+ "#{if @verbose
137
+ "#{score_split(breakdown)} (killed+timeout)/(killed+survived+timeout); " \
138
+ 'errors, resource limits and skips excluded'
139
+ end}"
140
+ end
129
141
  report.skip_reasons.each { |reason, count| @out.puts "skipped: #{reason}: #{count}" }
130
142
  reused = report.results.count(&:reused?)
131
143
  @out.puts "reused #{reused} of #{report.results.size} verdicts from earlier runs (--incremental)" if
@@ -223,17 +235,51 @@ module Saboteur
223
235
  end
224
236
  end
225
237
 
226
- # The verdict covers only the subjects that ran, so say it where the
238
+ # The verdict covers only the completed work, so say it where the
227
239
  # score is read.
228
240
  def early_stop(report)
229
241
  stop = report.early_stop
230
242
  return unless stop
231
243
 
232
- @out.puts @palette.paint(:yellow, "stopped early (--fail-fast): #{stop.unreached.size} selected " \
233
- "subject#{'s' unless stop.unreached.one?} did not run; the score below " \
234
- "covers only the subjects that ran")
235
- stop.unreached.first(MAX_ERRORS).each { |id| @out.puts " not run: #{id}" }
236
- @out.puts " ... #{stop.unreached.size - MAX_ERRORS} more" if stop.unreached.size > MAX_ERRORS
244
+ label = stop.reason == "fail_fast" ? "--fail-fast" : "--max-duration spent"
245
+ @out.puts @palette.paint(:yellow, "stopped early (#{label}): #{early_stop_summary(stop)}; the score below " \
246
+ "covers only completed work")
247
+ budget_shortfall(stop, report.results.size)
248
+ list(stop.unreached) { |id| " not run: #{id}" }
249
+ list(stop.truncated) { |entry| " stopped mid-way: #{entry[:subject]} (#{entry[:omitted]} mutants omitted)" }
250
+ list(stop.unverified) { |id| " unverified: #{id}" }
251
+ end
252
+
253
+ def list(entries)
254
+ entries.first(MAX_ERRORS).each { |entry| @out.puts yield(entry) }
255
+ @out.puts " ... #{entries.size - MAX_ERRORS} more" if entries.size > MAX_ERRORS
256
+ end
257
+
258
+ def budget_shortfall(stop, completed)
259
+ return unless stop.reason == "max_duration"
260
+
261
+ line = if stop.planned_mutants
262
+ "partial: #{stop.planned_mutants - completed} of #{stop.planned_mutants} mutants not run"
263
+ else
264
+ "partial: mutant count unknown; the budget was spent before planning finished"
265
+ end
266
+ @out.puts @palette.paint(:yellow, line)
267
+ end
268
+
269
+ def early_stop_summary(stop)
270
+ parts = []
271
+ if stop.unreached.any?
272
+ parts << "#{stop.unreached.size} selected subject#{'s' unless stop.unreached.one?} did not run"
273
+ end
274
+ if stop.truncated.any?
275
+ omitted = stop.truncated.sum { |entry| entry[:omitted] }
276
+ parts << "#{stop.truncated.size} subject#{'s' unless stop.truncated.one?} stopped mid-way " \
277
+ "(#{omitted} mutants omitted)"
278
+ end
279
+ if stop.unverified.any?
280
+ parts << "#{stop.unverified.size} subject#{'s' unless stop.unverified.one?} left with unverified results"
281
+ end
282
+ parts.join("; ")
237
283
  end
238
284
 
239
285
  def warnings(report)
@@ -241,13 +287,24 @@ module Saboteur
241
287
  end
242
288
 
243
289
  def errors(report)
244
- errored = report.results.select { |result| result.status == :error }
290
+ # Rejected kills are verdicts, not tool errors: the tests failed,
291
+ # but they also fail with the method unchanged (#375).
292
+ rejected, errored = report.results.select { |result| result.status == :error }.partition(&:rejected_kill?)
293
+ unless rejected.empty?
294
+ @out.puts "\nRejected kills (tests also fail with the method unchanged; not counted as killed):"
295
+ print_error_lines(rejected)
296
+ end
245
297
  return if errored.empty?
246
298
 
247
299
  @out.puts "\nErrors (not counted as killed):"
300
+ print_error_lines(errored)
301
+ end
302
+
303
+ def print_error_lines(errored)
248
304
  errored.first(MAX_ERRORS).each do |result|
249
- @out.puts " #{result.subject.id} #{result.mutation.operator} #{result.digest}: " \
250
- "#{result.reason.to_s.lines.first&.strip}"
305
+ lines = result.reason.to_s.lines.map(&:strip)
306
+ @out.puts " #{result.subject.id} #{result.mutation.operator} #{result.digest}: #{lines.first}"
307
+ lines.drop(1).each { |line| @out.puts " #{line}" }
251
308
  end
252
309
  @out.puts " ... #{errored.size - MAX_ERRORS} more" if errored.size > MAX_ERRORS
253
310
  end
@@ -12,7 +12,10 @@ module Saboteur
12
12
  # test can observe a change there, so it is outside the score.
13
13
  # unreachable: a Method object captured before re-evaluation still calls
14
14
  # the original body, so no test can observe the mutant. Outside the score.
15
- STATUSES = %i[killed survived timeout resource_limit error skipped ran_at_load unreachable].freeze
15
+ # unverified: the --max-duration budget ran out before the verdict check
16
+ # or timeout calibration could run, so the kill or timeout proves
17
+ # nothing. Outside the score; the run reports budget-partial.
18
+ STATUSES = %i[killed survived timeout resource_limit error skipped ran_at_load unreachable unverified].freeze
16
19
 
17
20
  # Outcome of one mutation. reason carries the skip reason, the error
18
21
  # message or the timeout limit. ignored is the ignore-list reason, if any.
@@ -20,6 +23,11 @@ module Saboteur
20
23
  attr_reader :subject, :mutation, :status, :reason, :runtime, :killed_by, :covered_by, :mutated_source,
21
24
  :kill_cause, :executed
22
25
  attr_accessor :ignored
26
+ # The ignore list's triage for this survivor (#155): a state from
27
+ # IgnoreList::STATES and its reason. accepted_equivalent and wont_fix
28
+ # suppress the exit failure (the survivor still counts in the score); a
29
+ # todo survivor keeps counting as an unignored survivor.
30
+ attr_accessor :triage_state, :triage_reason
23
31
 
24
32
  def initialize(subject:, mutation:, status:, mutated_source:, reason: nil, runtime: nil, killed_by: [],
25
33
  covered_by: [], kill_cause: nil, reused: false, rejected_kill: false, executed: nil)
@@ -38,6 +46,17 @@ module Saboteur
38
46
  @rejected_kill = rejected_kill
39
47
  @reused = reused
40
48
  @ignored = nil
49
+ # triage_state/triage_reason stay unset until the ignore list writes
50
+ # them: the accessors read as nil before that, and there is no
51
+ # statement for a mutation to remove (#304 dogfood).
52
+ end
53
+
54
+ # The same mutant, marked unverified: the budget ran out before its
55
+ # verdict could be checked, so it must not count as detected (#296).
56
+ def as_unverified(reason)
57
+ MutantResult.new(subject: subject, mutation: mutation, status: :unverified, mutated_source: mutated_source,
58
+ reason: reason, runtime: runtime, killed_by: killed_by, covered_by: covered_by,
59
+ kill_cause: kill_cause, executed: executed)
41
60
  end
42
61
 
43
62
  # The same mutant, marked as an error: its verdict did not hold up.
@@ -100,10 +119,17 @@ module Saboteur
100
119
  end
101
120
  end
102
121
 
103
- # A run that stopped before every selected subject was mutated (--fail-fast).
104
- # unreached lists the subject ids that never ran; the verdict covers only
105
- # the subjects that did.
106
- EarlyStop = Data.define(:reason, :unreached)
122
+ # A run that stopped before every selected subject was mutated (--fail-fast,
123
+ # --max-duration), or whose budget ran out before every verdict check.
124
+ # unreached lists the subject ids that never ran; truncated lists the
125
+ # subjects that started but stopped mid-way as {subject:, omitted:}
126
+ # entries; unverified lists the subjects whose kills the budget left
127
+ # unchecked. The verdict covers only completed, verified work.
128
+ EarlyStop = Data.define(:reason, :unreached, :truncated, :unverified, :planned_mutants) do
129
+ def initialize(reason:, unreached:, truncated:, unverified:, planned_mutants: nil)
130
+ super
131
+ end
132
+ end
107
133
 
108
134
  # Everything one run produced.
109
135
  class Report
@@ -113,14 +139,14 @@ module Saboteur
113
139
 
114
140
  attr_reader :results, :subjects, :wall_time, :invalid_mutations, :warnings, :root, :seed, :skipped_subjects,
115
141
  :generation_events, :omissions, :run_evidence, :hazard_suppressions, :early_stop, :inline_suppressions,
116
- :operator_policy
142
+ :operator_policy, :discovery_failures, :memory, :shard
117
143
 
118
144
  # REVIEW: diff mode; the terminal then shows one survivor per line.
119
145
  # seed: the random seed every test run started from (nil when unknown).
120
146
  def initialize(results:, subjects:, wall_time:, root:, invalid_mutations: 0, warnings: [], review: false,
121
147
  seed: nil, skipped_subjects: {}, diff_scope: nil, generation_events: [], omissions: [],
122
148
  run_evidence: nil, hazard_suppressions: 0, early_stop: nil, matrix: false, inline_suppressions: [],
123
- operator_policy: nil)
149
+ operator_policy: nil, discovery_failures: [], memory: nil, shard: nil)
124
150
  @results = results
125
151
  @subjects = subjects
126
152
  @wall_time = wall_time
@@ -139,6 +165,11 @@ module Saboteur
139
165
  @matrix = matrix
140
166
  @inline_suppressions = inline_suppressions.dup.freeze
141
167
  @operator_policy = operator_policy
168
+ @discovery_failures = discovery_failures.dup.freeze
169
+ # Per-worker and supervisor peak RSS in MB, when the platform measured
170
+ # it (#202); nil elsewhere, so JSON reports omit the section.
171
+ @memory = memory.dup.freeze
172
+ @shard = shard
142
173
  end
143
174
 
144
175
  # A --matrix run: every covering test ran per mutant.