saboteur 0.12.1 → 0.12.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (52) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +86 -0
  3. data/README.md +18 -3
  4. data/lib/saboteur/baseline/command.rb +6 -1
  5. data/lib/saboteur/baseline/comparison.rb +8 -0
  6. data/lib/saboteur/baseline/completeness.rb +14 -4
  7. data/lib/saboteur/baseline/evidence.rb +8 -1
  8. data/lib/saboteur/baseline/run_evidence.rb +10 -1
  9. data/lib/saboteur/baseline/workflow.rb +17 -5
  10. data/lib/saboteur/cli/color_option.rb +26 -0
  11. data/lib/saboteur/cli/doctor_command.rb +1 -0
  12. data/lib/saboteur/cli/exit_status.rb +4 -3
  13. data/lib/saboteur/cli/explain_command.rb +2 -1
  14. data/lib/saboteur/cli/failure_output.rb +13 -2
  15. data/lib/saboteur/cli/help_command.rb +5 -0
  16. data/lib/saboteur/cli/history_command.rb +2 -0
  17. data/lib/saboteur/cli/output_options.rb +1 -5
  18. data/lib/saboteur/cli/run_command.rb +1 -1
  19. data/lib/saboteur/cli/run_reporting.rb +1 -9
  20. data/lib/saboteur/cli/run_validation.rb +0 -1
  21. data/lib/saboteur/cli/survivors_command.rb +10 -6
  22. data/lib/saboteur/cli/whole_suite_map_lookup.rb +33 -0
  23. data/lib/saboteur/cli.rb +6 -2
  24. data/lib/saboteur/engine/coverage_store.rb +7 -4
  25. data/lib/saboteur/engine/diagnostics.rb +7 -0
  26. data/lib/saboteur/engine/pipelines.rb +9 -2
  27. data/lib/saboteur/engine/result_cache.rb +8 -5
  28. data/lib/saboteur/engine/scheduler.rb +3 -2
  29. data/lib/saboteur/engine/survivor_confirmation.rb +75 -11
  30. data/lib/saboteur/engine/test_guess.rb +2 -1
  31. data/lib/saboteur/engine.rb +34 -16
  32. data/lib/saboteur/execution/runner.rb +14 -4
  33. data/lib/saboteur/execution/site_probe.rb +161 -0
  34. data/lib/saboteur/gates.rb +1 -1
  35. data/lib/saboteur/integration/rspec.rb +30 -0
  36. data/lib/saboteur/reporter/default_findings.rb +1 -1
  37. data/lib/saboteur/reporter/findings.rb +2 -1
  38. data/lib/saboteur/reporter/gap_kinds.rb +18 -0
  39. data/lib/saboteur/reporter/next_steps.rb +3 -1
  40. data/lib/saboteur/reporter/review_comments.rb +4 -1
  41. data/lib/saboteur/reporter/score_table.rb +6 -11
  42. data/lib/saboteur/reporter/stryker.rb +12 -2
  43. data/lib/saboteur/reporter/survivor_classification.rb +53 -6
  44. data/lib/saboteur/reporter/survivor_hints.rb +13 -9
  45. data/lib/saboteur/reporter/survivor_list.rb +6 -6
  46. data/lib/saboteur/reporter/terminal.rb +4 -2
  47. data/lib/saboteur/reporter/unconfirmed_section.rb +44 -0
  48. data/lib/saboteur/result.rb +46 -4
  49. data/lib/saboteur/selection/coverage_map.rb +11 -1
  50. data/lib/saboteur/version.rb +1 -1
  51. data/lib/saboteur.rb +5 -0
  52. metadata +5 -1
@@ -57,7 +57,7 @@ module Saboteur
57
57
  def list_lines(report, live)
58
58
  groups(report, live).map do |(path, line, subject_id), results|
59
59
  first, *rest = results
60
- label = SurvivorClassification.label(first)
60
+ label = SurvivorClassification.group_label(results)
61
61
  text = "#{path}:#{line} #{subject_id}#{" [#{label}]" if label} - #{first.digest}"
62
62
  rest.any? ? "#{text} (+#{rest.size} more)" : text
63
63
  end
@@ -71,7 +71,8 @@ module Saboteur
71
71
  edits: entries,
72
72
  coarse: result.mutation.coarse,
73
73
  # Why the survivor is worth a look, as a label only (#192).
74
- classification: SurvivorClassification.label(result)
74
+ classification: SurvivorClassification.label(result),
75
+ gapKind: SurvivorClassification.kind(result)
75
76
  }.compact
76
77
  entry = with_changed_lines(report, result, entry)
77
78
  result.ignored ? entry.merge(reason: result.ignored) : entry
@@ -58,6 +58,12 @@ module Saboteur
58
58
 
59
59
  FALLBACK = "result unchecked"
60
60
 
61
+ # Which kind names a statement when its surviving mutants disagree
62
+ # (#P4): the first one listed that any of them carries. The broadest
63
+ # unchecked thing wins, so a branch condition is not called a
64
+ # boundary because one of its mutants happened to flip a comparison.
65
+ PRECEDENCE = ["result unchecked", "error path", "branch", "boundary", "effect", "number"].freeze
66
+
61
67
  # The deterministic per-kind reason printed next to a survivor's rank.
62
68
  REASONS = {
63
69
  "boundary" => "the covering specs pass without pinning this boundary",
@@ -68,6 +74,18 @@ module Saboteur
68
74
  "result unchecked" => "the covering specs pass without pinning this result"
69
75
  }.freeze
70
76
 
77
+ # What an observed label (input_gap/assertion_gap, #1290) replaces
78
+ # the kind with, and the advice the kind gave: kept in the reason.
79
+ OBSERVED = %w[input_gap assertion_gap].freeze
80
+ ADVICE = {
81
+ "boundary" => "exercise both sides of this boundary",
82
+ "branch" => "exercise both branches",
83
+ "effect" => "assert the effect of this statement",
84
+ "number" => "no test depends on this exact number",
85
+ "error path" => "exercise the error path",
86
+ "result unchecked" => "assert this result"
87
+ }.freeze
88
+
71
89
  module_function
72
90
 
73
91
  def kind(operator, comparison_edit: false)
@@ -20,7 +20,9 @@ module Saboteur
20
20
  end
21
21
 
22
22
  def call(report)
23
- survivor = report.unignored_survivors.first
23
+ # The replay example is the #1 survivor of the ranked list, not
24
+ # whichever one the results hold first.
25
+ survivor = SurvivorRanking.ordered(report, report.unignored_survivors).first
24
26
  return unless survivor
25
27
 
26
28
  @out.puts "\n#{@palette.paint(:head, 'Next steps:')}"
@@ -32,7 +32,10 @@ module Saboteur
32
32
  TOOL_LIMIT_HINT = "Saboteur could not observe this mutant faithfully. Check it by hand."
33
33
  HINTS = GapKinds::KINDS.to_h { |label| [label, GAP_HINT] }
34
34
  .merge("tool_limit" => TOOL_LIMIT_HINT,
35
- "input_gap" => "No test reaches this line. Exercise it with a varied input.").freeze
35
+ "input_gap" => "No test input makes this change differ. " \
36
+ "Exercise it with a varied input.",
37
+ "assertion_gap" => "A test reaches this line, the value changes, and it still " \
38
+ "passes. Add an assertion that pins this behaviour.").freeze
36
39
  DEFAULT_HINT = "No test fails with this change. Check whether a test should pin it."
37
40
 
38
41
  Plan = Struct.new(:post, :delete, :summary, keyword_init: true)
@@ -15,7 +15,6 @@ module Saboteur
15
15
  # it. Untested subjects are never dropped silently, even at 100 %.
16
16
  class ScoreTable
17
17
  MAX_ROWS = 15
18
- MAX_UNTESTED_NAMES = 3
19
18
  # The skip reason CandidateSelection gives a subject no test covers.
20
19
  NO_COVERAGE = "no coverage"
21
20
 
@@ -63,7 +62,7 @@ module Saboteur
63
62
  detected: breakdown.detected, tested: breakdown.tested, untested: untested }
64
63
  end
65
64
 
66
- # "cart.rb 100.0% (2/2) · n/a - 2 subjects - no test executes them: ...": the
65
+ # "cart.rb 100.0% (2/2) · n/a - 2 subjects - no test executes them": the
67
66
  # tested score and the untested count stand next to each other; a
68
67
  # file with nothing tested gets no percentage at all.
69
68
  def line(row, width)
@@ -75,18 +74,14 @@ module Saboteur
75
74
  note ? "#{base} #{@palette.glyph(:middot)} #{note}" : base
76
75
  end
77
76
 
78
- # "n/a - 2 subjects - no test executes them: Cart#total, Cart#biggest":
79
- # names the subjects the score cannot speak for, capped so a long
80
- # list does not swallow the row; the Not scored block lists them all.
81
- # Wording owns the claim (#1145): a stubbed-away subject has tests,
82
- # they just never execute it.
77
+ # "n/a - 2 subjects - no test executes them": only the count. The
78
+ # Not scored block below names every subject once (P6), so the table
79
+ # does not repeat the names. Wording owns the claim (#1145): a
80
+ # stubbed-away subject has tests, they just never execute it.
83
81
  def untested_note(untested)
84
82
  return if untested.empty?
85
83
 
86
- names = untested.map(&:id)
87
- shown = names.first(MAX_UNTESTED_NAMES)
88
- shown << "+#{names.size - MAX_UNTESTED_NAMES} more" if names.size > MAX_UNTESTED_NAMES
89
- @palette.paint(:warn, "n/a - #{Wording.untested_note(untested.size)}: #{shown.join(', ')}")
84
+ @palette.paint(:warn, "n/a - #{Wording.untested_note(untested.size)}")
90
85
  end
91
86
  end
92
87
  end
@@ -98,7 +98,7 @@ module Saboteur
98
98
  end
99
99
 
100
100
  def self.test_index(results)
101
- names = results.flat_map { |result| result.covered_by + result.killed_by + result.executed.to_a }.uniq
101
+ names = results.flat_map { |result| result.covered_by + result.killed_by + result.executed.to_a }.uniq.sort
102
102
  names.each_with_index.to_h { |name, index| [name, index.to_s] }
103
103
  end
104
104
 
@@ -139,6 +139,15 @@ module Saboteur
139
139
  # Why the survivor is worth a look, as a label only: scores and
140
140
  # the exit code never change (#192).
141
141
  classification: SurvivorClassification.label(result),
142
+ # Additive (#1290): the gap kind an observed label replaced.
143
+ gapKind: SurvivorClassification.kind(result),
144
+ # Additive (#1286): confirmed, killed_by_confirmation or unconfirmed.
145
+ # An unconfirmed survivor reports Ignored: outside the score.
146
+ confirmation: result.confirmation_state,
147
+ # Additive (#1290): whether the mutated expression ever produced a
148
+ # different value from the original; absent when not probed.
149
+ valueDiffered: result.value_differed,
150
+ creditedTest: result.credited_test&.then { |name| test_ids.fetch(name) },
142
151
  killedBy: result.killed_by.map { |name| test_ids.fetch(name) }.then { |ids| ids.empty? ? nil : ids },
143
152
  coveredBy: result.covered_by.map { |name| test_ids.fetch(name) }.then { |ids| ids.empty? ? nil : ids },
144
153
  # Which of the covering tests actually ran against the mutant; absent
@@ -151,7 +160,7 @@ module Saboteur
151
160
  def self.status(result)
152
161
  case result.status
153
162
  when :killed then "Killed"
154
- when :survived then "Survived"
163
+ when :survived then result.unconfirmed? ? "Ignored" : "Survived"
155
164
  when :timeout then "Timeout"
156
165
  when :error, :resource_limit then "RuntimeError"
157
166
  when :skipped then result.reason&.start_with?("no coverage") ? "NoCoverage" : "Ignored"
@@ -169,6 +178,7 @@ module Saboteur
169
178
  # ignore, and only wont_fix adds a suffix after the reason.
170
179
  return "ignored: #{result.ignored} (wont_fix)" if result.triage_state == "wont_fix"
171
180
  return "ignored: #{result.ignored}" if result.ignored
181
+ return "unconfirmed: #{result.unconfirmed_reason}" if result.unconfirmed?
172
182
  return "resource limit: #{result.reason}" if result.status == :resource_limit
173
183
  return result.crash_kill? ? "crash: #{result.kill_cause}" : result.kill_cause if result.kill_cause
174
184
 
@@ -29,11 +29,14 @@ module Saboteur
29
29
  # operator and the edit shape only. The covering list is only the
30
30
  # PLANNED coverage: whole-body (coarse) mutants carry method-level
31
31
  # guesses, so they never get a gap kind.
32
- # input_gap - no real test covered the mutant at all, so the input
33
- # never varied (D6). Coarse mutants stay unlabeled: their planned
34
- # coverage is a method-level guess, not an observation. The
35
- # deterministic version (the runner records the mutated value
36
- # actually differing) is #1064 follow-up.
32
+ # input_gap - the tests never gave the mutated expression an input
33
+ # where its value differs (the runner recorded it, #1290), or no
34
+ # real test covered the mutant at all (D6). Coarse mutants stay
35
+ # unlabeled: their planned coverage is a method-level guess.
36
+ # assertion_gap - the mutated expression produced a different value
37
+ # while the covering tests ran, and they still passed: a test
38
+ # reaches the code and no assertion pins it (#1290). Only side-effect
39
+ # free sites are observed; an unprobed site keeps its gap kind.
37
40
  # tool_limit - survivors saboteur could not observe faithfully: an
38
41
  # external-runner run records only the pseudo-test id, never real
39
42
  # coverage (#1071). A re-eval refusal records no signal on the
@@ -41,7 +44,10 @@ module Saboteur
41
44
  #
42
45
  # A survivor with no matching signal stays unlabeled.
43
46
  module SurvivorClassification
44
- LABELS = %w[likely_equivalent arid tool_limit input_gap].concat(GapKinds::KINDS).freeze
47
+ LABELS = %w[likely_equivalent arid tool_limit input_gap assertion_gap].concat(GapKinds::KINDS).freeze
48
+ # assertion_gap decides a statement's label ahead of every gap kind: the
49
+ # value changed and the tests passed.
50
+ GROUP_ORDER = (["assertion_gap"] + GapKinds::PRECEDENCE).freeze
45
51
 
46
52
  class << self
47
53
  def label(result, arid: MutationSource::Builtin::DEFAULT_ARID)
@@ -49,12 +55,44 @@ module Saboteur
49
55
  return "likely_equivalent" if equivalent?(result)
50
56
  return "arid" if arid?(result, arid)
51
57
  return "tool_limit" if tool_limited?(result)
58
+
59
+ observed = observed_gap(result)
60
+ return observed if observed
52
61
  return gap_kind(result) if possible_test_gap?(result)
53
62
  return "input_gap" unless result.mutation.coarse
54
63
 
55
64
  nil
56
65
  end
57
66
 
67
+ # The label of one statement's surviving mutants as a whole, the
68
+ # same whichever mutant sorts first. Any gap kind decides, by
69
+ # GapKinds::PRECEDENCE, so one equivalent, logging or input_gap
70
+ # sibling never hides a real survivor. With no gap kind, the first
71
+ # label present stands: likely_equivalent, arid or tool_limit then
72
+ # describe the statement when all its mutants carry it.
73
+ #
74
+ # Among gap kinds, result unchecked wins because it is the strictly
75
+ # larger gap: nothing pins the statement's value at all, which
76
+ # subsumes a missing branch, boundary or effect check on it.
77
+ def group_label(results)
78
+ labels = results.map { |result| label(result) }.compact
79
+ GROUP_ORDER.find { |kind| labels.include?(kind) } || labels.first
80
+ end
81
+
82
+ # The gap kind an observed label (input_gap/assertion_gap) replaced:
83
+ # what the edit shape says is unchecked. nil unless the observation
84
+ # decided the label, so a plain kind label stays its own kind.
85
+ def kind(result)
86
+ return nil unless GapKinds::OBSERVED.include?(observed_gap(result)) && possible_test_gap?(result)
87
+
88
+ gap_kind(result)
89
+ end
90
+
91
+ def group_kind(results)
92
+ kinds = results.map { |result| kind(result) }.compact
93
+ GapKinds::PRECEDENCE.find { |kind| kinds.include?(kind) }
94
+ end
95
+
58
96
  private
59
97
 
60
98
  # The mutant's own def text, compiled side by side with the
@@ -76,6 +114,15 @@ module Saboteur
76
114
  # An external runner reports one pseudo id for every mutant and no
77
115
  # coverage map, so a survivor there was never observed against real
78
116
  # tests. Pseudo id next to a real id keeps the gap kind (#305).
117
+ # The runner's observation (#1290): the mutated expression never gave
118
+ # a different value (input_gap), or gave one and the tests passed
119
+ # (assertion_gap). nil when the site was not probed.
120
+ def observed_gap(result)
121
+ return nil if result.mutation.coarse || result.value_differed.nil?
122
+
123
+ result.value_differed ? "assertion_gap" : "input_gap"
124
+ end
125
+
79
126
  def tool_limited?(result)
80
127
  result.covered_by.any? && result.covered_by.all? { |id| id == Execution::ExternalRunner::PSEUDO_TEST }
81
128
  end
@@ -48,9 +48,8 @@ module Saboteur
48
48
  UNEQUAL_CASE = [%w[< >], %w[<= >=]].map(&:to_set).freeze
49
49
 
50
50
  # Labels that never get a hint: a generated assertion on an equivalent
51
- # mutant is the worst kind of hint noise. The assertion_gap/input_gap
52
- # split (#1064) narrows this further; until it lands, every other
53
- # label keeps the operator hint unchanged.
51
+ # mutant is the worst kind of hint noise. Every other label, including
52
+ # input_gap and assertion_gap, keeps the operator hint.
54
53
  NO_HINT_LABELS = %w[likely_equivalent].freeze
55
54
 
56
55
  # One deterministic template sentence per operator: what the change
@@ -91,13 +90,14 @@ module Saboteur
91
90
 
92
91
  # The deterministic per-label reason printed next to a survivor's rank.
93
92
  # Labels come from SurvivorClassification; they are heuristics, not
94
- # diagnoses. #1064 lands the assertion_gap label: add its reason here
95
- # and its gate in NO_HINT_LABELS when it does.
93
+ # diagnoses. input_gap and assertion_gap keep the operator hint: it is
94
+ # the right place to look in both cases (#1290).
96
95
  LABEL_REASONS = {
97
96
  "likely_equivalent" => "compiles to the same instructions as the original",
98
97
  "arid" => "every edited line matches an arid pattern",
99
98
  "tool_limit" => "the tool could not observe this mutant faithfully",
100
- "input_gap" => "no test reached this mutant, so its input never varied"
99
+ "input_gap" => "no test input made this mutation change the value, so its input never varied",
100
+ "assertion_gap" => "a test ran this mutant, the value changed, and no assertion noticed"
101
101
  }.merge(GapKinds::REASONS).freeze
102
102
 
103
103
  module_function
@@ -114,9 +114,13 @@ module Saboteur
114
114
  WHY_IT_MATTERS.fetch(operator, WHY_FALLBACK)
115
115
  end
116
116
 
117
- # The per-label reason sentence; nil when the label has none.
118
- def label_reason(label)
119
- LABEL_REASONS[label]
117
+ # The per-label reason sentence; nil when the label has none. An
118
+ # observed label (input_gap/assertion_gap) keeps the advice of the gap
119
+ # kind it replaced, so the kind is not lost behind the observation.
120
+ def label_reason(label, kind: nil)
121
+ reason = LABEL_REASONS[label]
122
+ advice = GapKinds::ADVICE[kind]
123
+ advice && GapKinds::OBSERVED.include?(label) ? "#{reason}; #{advice}" : reason
120
124
  end
121
125
 
122
126
  def for(operator, coarse: false)
@@ -77,8 +77,8 @@ module Saboteur
77
77
  line = MutationText.line(first)
78
78
  path = relative(first.subject.file, report.root)
79
79
  @out.puts "\n#{@palette.paint(:head, "#{rank}. #{path}:#{line}")} #{first.subject.id} " \
80
- "#{@palette.paint(:dim, label_suffix(first, rank, total))}".rstrip
81
- reason = SurvivorHints.label_reason(@classifier.label(first))
80
+ "#{@palette.paint(:dim, label_suffix(group, rank, total))}".rstrip
81
+ reason = SurvivorHints.label_reason(@classifier.group_label(group), kind: @classifier.group_kind(group))
82
82
  @out.puts " #{@palette.paint(:dim, reason)}" if reason
83
83
  fence(first, line)
84
84
  @out.puts " #{@palette.paint(:dim, "why it matters: #{why(first.mutation.operator)}")}"
@@ -105,10 +105,10 @@ module Saboteur
105
105
  # siblings read "#3 of 14 blocks", never blocks against mutants (#1147).
106
106
  # Label and rank stay on the header line; the reason moves to its own
107
107
  # dim line so the fixed text cannot push the header past 80 columns.
108
- def label_suffix(result, rank, total)
109
- label = @classifier.label(result)
108
+ def label_suffix(group, rank, total)
109
+ label = @classifier.group_label(group)
110
110
  unit = total == 1 ? "block" : "blocks"
111
- parts = [("coarse" if result.mutation.coarse), label, "##{rank} of #{total} #{unit}"].compact
111
+ parts = [("coarse" if group.first.mutation.coarse), label, "##{rank} of #{total} #{unit}"].compact
112
112
  "[#{parts.join(' · ')}]"
113
113
  end
114
114
 
@@ -226,7 +226,7 @@ module Saboteur
226
226
 
227
227
  whole = @whole_suite_map&.tests_for(group.first.subject)
228
228
  if whole.nil?
229
- @out.puts @palette.paint(:dim, " covering tests unknown - run a full coverage build")
229
+ @out.puts @palette.paint(:dim, " covering list is this run's selection only; run `saboteur run --all`")
230
230
  return
231
231
  end
232
232
 
@@ -6,6 +6,7 @@ require_relative "wording"
6
6
  require_relative "baseline_tail"
7
7
  require_relative "next_steps"
8
8
  require_relative "kill_causes"
9
+ require_relative "unconfirmed_section"
9
10
  require_relative "survivor_ranking"
10
11
  require_relative "palette"
11
12
  require_relative "notice"
@@ -62,6 +63,7 @@ module Saboteur
62
63
  return quiet_call(report) if @quiet
63
64
 
64
65
  findings(report)
66
+ UnconfirmedSection.new(@out, @palette).call(report)
65
67
  errors(report)
66
68
  skipped_subjects(report) if @verbose
67
69
  test_command_gaps(report)
@@ -94,7 +96,7 @@ module Saboteur
94
96
 
95
97
  def quiet_survivor_line(path, line, subject_id, results)
96
98
  first, *rest = results
97
- label = SurvivorClassification.label(first)
99
+ label = SurvivorClassification.group_label(results)
98
100
  text = "#{path}:#{line} #{subject_id}#{" [#{label}]" if label} - #{first.digest}"
99
101
  rest.any? ? "#{text} (+#{rest.size} more)" : text
100
102
  end
@@ -346,7 +348,7 @@ module Saboteur
346
348
  untested = untested_subjects(report).size
347
349
  [counted(counts[:killed], "killed", plural: false, color: :pass),
348
350
  counted(live, "survived", plural: false, color: :fail),
349
- counted(report.survivors.size - live, "accepted", plural: false),
351
+ counted(report.survivors.size - live, "accepted", plural: false), UnconfirmedSection.part(report),
350
352
  counted(counts[:timeout], "timeout", color: :warn), counted(counts[:resource_limit], "resource limit"),
351
353
  error_part(report, counts[:error]), counted(counts[:unverified], "unverified", plural: false),
352
354
  ("#{untested} subject#{'s' unless untested == 1} untested" if untested.positive?)].compact
@@ -0,0 +1,44 @@
1
+ # SPDX-License-Identifier: MIT
2
+ # frozen_string_literal: true
3
+
4
+ require_relative "mutation_text"
5
+ require_relative "survivor_ranking"
6
+
7
+ module Saboteur
8
+ module Reporter
9
+ # Survivors whose covering tests were not all run (#1286): outside the
10
+ # score and the exit code, so they get their own short list instead of
11
+ # a place among the findings.
12
+ class UnconfirmedSection
13
+ LIMIT = 10
14
+
15
+ # The count for the one-line (--quiet) tally, nil when there is none.
16
+ def self.part(report)
17
+ count = report.unconfirmed_survivors.size
18
+ "#{count} unconfirmed" if count.positive?
19
+ end
20
+
21
+ def initialize(out, palette)
22
+ @out = out
23
+ @palette = palette
24
+ end
25
+
26
+ def call(report)
27
+ pending = report.unconfirmed_survivors
28
+ return if pending.empty?
29
+
30
+ @out.puts @palette.paint(:head, "unconfirmed (covering tests not run) - #{pending.size}, not in the score")
31
+ SurvivorRanking.ordered(report, pending).first(LIMIT).each { |result| @out.puts line(report, result) }
32
+ @out.puts " ... #{pending.size - LIMIT} more" if pending.size > LIMIT
33
+ @out.puts
34
+ end
35
+
36
+ private
37
+
38
+ def line(report, result)
39
+ path = result.subject.file.delete_prefix("#{report.root}/")
40
+ " #{path}:#{MutationText.line(result)} #{result.subject.id} - #{result.digest} (#{result.unconfirmed_reason})"
41
+ end
42
+ end
43
+ end
44
+ end
@@ -35,9 +35,13 @@ module Saboteur
35
35
  # status keeps its meaning either way.
36
36
  attr_reader :confirmation
37
37
 
38
+ # Whether the mutated expression ever produced a value different from the
39
+ # original one while the covering tests ran; nil when not probed (#1290).
40
+ attr_reader :value_differed
41
+
38
42
  def initialize(subject:, mutation:, status:, mutated_source:, reason: nil, runtime: nil, killed_by: [],
39
43
  covered_by: [], kill_cause: nil, reused: false, rejected_kill: false, executed: nil,
40
- confirmation: nil)
44
+ confirmation: nil, value_differed: nil)
41
45
  raise ArgumentError, "unknown status #{status.inspect}" unless STATUSES.include?(status)
42
46
 
43
47
  @subject = subject
@@ -53,6 +57,7 @@ module Saboteur
53
57
  @rejected_kill = rejected_kill
54
58
  @reused = reused
55
59
  @confirmation = confirmation
60
+ @value_differed = value_differed
56
61
  @ignored = nil
57
62
  # triage_state/triage_reason stay unset until the ignore list writes
58
63
  # them: the accessors read as nil before that, and there is no
@@ -69,7 +74,7 @@ module Saboteur
69
74
  MutantResult.new(subject: subject, mutation: mutation, status: status, mutated_source: mutated_source,
70
75
  reason: reason, runtime: runtime, killed_by: killed_by, covered_by: covered_by,
71
76
  kill_cause: kill_cause, reused: reused?, rejected_kill: rejected_kill?,
72
- executed: executed, confirmation: confirmation)
77
+ executed: executed, confirmation: confirmation, value_differed: value_differed)
73
78
  end
74
79
 
75
80
  # The reason a --flaky-check disagreement starts with: unverified, but
@@ -112,8 +117,37 @@ module Saboteur
112
117
  @digest ||= mutation.digest(subject_start: subject.start_byte, original_source: subject.file_source)
113
118
  end
114
119
 
120
+ # A survivor whose confirmation run could not happen (budget spent, or the
121
+ # run failed) is outside the score and the exit code until it is
122
+ # confirmed (#1286). A survivor the whole-suite map cannot speak for
123
+ # ("unconfirmed_map(...)") stays counted: a stale map never lowers the gate.
115
124
  def survived?
116
- status == :survived
125
+ status == :survived && !unconfirmed?
126
+ end
127
+
128
+ def unconfirmed?
129
+ status == :survived && @confirmation.to_s.start_with?("unconfirmed(")
130
+ end
131
+
132
+ # The additive report field: confirmed, killed_by_confirmation or
133
+ # unconfirmed; nil when confirmation never applied (#1286).
134
+ def confirmation_state
135
+ return nil if @confirmation.nil?
136
+
137
+ label = @confirmation.to_s
138
+ label.start_with?("unconfirmed") ? "unconfirmed" : label
139
+ end
140
+
141
+ # The test a confirmation kill is credited to.
142
+ def credited_test
143
+ killed_by.first if confirmation_state == "killed_by_confirmation"
144
+ end
145
+
146
+ # Why the survivor stayed unconfirmed: the text inside unconfirmed(...).
147
+ def unconfirmed_reason
148
+ return unless unconfirmed?
149
+
150
+ @confirmation.delete_prefix("unconfirmed(").delete_suffix(")")
117
151
  end
118
152
 
119
153
  # Killed because a test's error, not its check, fired: NoMethodError and
@@ -280,7 +314,9 @@ module Saboteur
280
314
  end
281
315
 
282
316
  def counts
283
- STATUSES.to_h { |status| [status, results.count { |result| result.status == status }] }
317
+ STATUSES.to_h do |status|
318
+ [status, results.count { |result| status == :survived ? result.survived? : result.status == status }]
319
+ end
284
320
  end
285
321
 
286
322
  # (killed + timeout) / (killed + survived + timeout); nil with nothing tested.
@@ -308,6 +344,12 @@ module Saboteur
308
344
  results.select(&:survived?)
309
345
  end
310
346
 
347
+ # Survivors whose covering tests were not all run: outside the score and
348
+ # the exit code, listed on their own (#1286).
349
+ def unconfirmed_survivors
350
+ results.select(&:unconfirmed?)
351
+ end
352
+
311
353
  def unignored_survivors
312
354
  results.select(&:unignored_survivor?)
313
355
  end
@@ -443,9 +443,19 @@ module Saboteur
443
443
  [prefix.count("\n") + 1, offset - ((prefix.rindex("\n") || -1) + 1)]
444
444
  end
445
445
 
446
+ # Durations are compared to the millisecond. Two tests within a
447
+ # millisecond of each other swap places from one map build to the next,
448
+ # and with fail-fast the first failing test names a mutant's killer and
449
+ # kill cause, so sub-millisecond noise must not decide the order (#1274).
450
+ # Suite order breaks the tie instead.
446
451
  def sort_test_ids(ids)
447
452
  order = test_order
448
- ids.sort_by { |id| [durations.fetch(id, Float::INFINITY), order.fetch(id, Float::INFINITY)] }
453
+ ids.sort_by { |id| [duration_bucket(id), order.fetch(id, Float::INFINITY)] }
454
+ end
455
+
456
+ def duration_bucket(id)
457
+ seconds = durations[id]
458
+ seconds ? (seconds * 1000).ceil : Float::INFINITY
449
459
  end
450
460
 
451
461
  def test_order
@@ -2,5 +2,5 @@
2
2
  # frozen_string_literal: true
3
3
 
4
4
  module Saboteur
5
- VERSION = "0.12.1"
5
+ VERSION = "0.12.3"
6
6
  end
data/lib/saboteur.rb CHANGED
@@ -6,6 +6,10 @@ require_relative "saboteur/version"
6
6
  module Saboteur
7
7
  class Error < StandardError; end
8
8
 
9
+ # A spec path from the cached coverage map that is not a spec file inside
10
+ # the project: the map is untrusted input (#427, #1286).
11
+ class RefusedSpecPath < Error; end
12
+
9
13
  # A phase timeout that --max-duration shortened: budget exhaustion, not a
10
14
  # broken suite. The engine turns it into the budget partial-report path
11
15
  # instead of a phase error (#296).
@@ -66,6 +70,7 @@ require_relative "saboteur/execution/marker_sweep"
66
70
  require_relative "saboteur/execution/supervisor"
67
71
  require_relative "saboteur/execution/reevaluator"
68
72
  require_relative "saboteur/execution/wrapper"
73
+ require_relative "saboteur/execution/site_probe"
69
74
  require_relative "saboteur/execution/runner"
70
75
  require_relative "saboteur/execution/bundle_gemfile_pin"
71
76
  require_relative "saboteur/execution/gemfile_path_sources"
metadata CHANGED
@@ -1,7 +1,7 @@
1
1
  --- !ruby/object:Gem::Specification
2
2
  name: saboteur
3
3
  version: !ruby/object:Gem::Version
4
- version: 0.12.1
4
+ version: 0.12.3
5
5
  platform: ruby
6
6
  authors:
7
7
  - Vamsi Pavan Mahesh Gunturu
@@ -63,6 +63,7 @@ files:
63
63
  - lib/saboteur/cli.rb
64
64
  - lib/saboteur/cli/agent_provenance.rb
65
65
  - lib/saboteur/cli/baseline_gate.rb
66
+ - lib/saboteur/cli/color_option.rb
66
67
  - lib/saboteur/cli/dirty_tree_notice.rb
67
68
  - lib/saboteur/cli/doctor_command.rb
68
69
  - lib/saboteur/cli/error_cause.rb
@@ -89,6 +90,7 @@ files:
89
90
  - lib/saboteur/cli/shard_options.rb
90
91
  - lib/saboteur/cli/source_set.rb
91
92
  - lib/saboteur/cli/survivors_command.rb
93
+ - lib/saboteur/cli/whole_suite_map_lookup.rb
92
94
  - lib/saboteur/config.rb
93
95
  - lib/saboteur/config_file.rb
94
96
  - lib/saboteur/constant_reads.rb
@@ -162,6 +164,7 @@ files:
162
164
  - lib/saboteur/execution/runner.rb
163
165
  - lib/saboteur/execution/sandbox.rb
164
166
  - lib/saboteur/execution/seatbelt.rb
167
+ - lib/saboteur/execution/site_probe.rb
165
168
  - lib/saboteur/execution/stale_captures.rb
166
169
  - lib/saboteur/execution/supervisor.rb
167
170
  - lib/saboteur/execution/supervisor_batch.rb
@@ -253,6 +256,7 @@ files:
253
256
  - lib/saboteur/reporter/survivor_ranking.rb
254
257
  - lib/saboteur/reporter/terminal.rb
255
258
  - lib/saboteur/reporter/test_matrix.rb
259
+ - lib/saboteur/reporter/unconfirmed_section.rb
256
260
  - lib/saboteur/reporter/wording.rb
257
261
  - lib/saboteur/result.rb
258
262
  - lib/saboteur/score.rb