saboteur 0.11.1 → 0.12.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (46) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +64 -0
  3. data/README.md +1 -1
  4. data/lib/saboteur/cli/exit_status.rb +6 -2
  5. data/lib/saboteur/cli/explain_command.rb +51 -0
  6. data/lib/saboteur/cli/flaky_exit.rb +23 -0
  7. data/lib/saboteur/cli/output_options.rb +4 -0
  8. data/lib/saboteur/cli/run_command.rb +19 -0
  9. data/lib/saboteur/cli/run_output.rb +10 -0
  10. data/lib/saboteur/cli/run_reporting.rb +14 -3
  11. data/lib/saboteur/cli/survivors_command.rb +2 -1
  12. data/lib/saboteur/cli.rb +4 -1
  13. data/lib/saboteur/config.rb +25 -3
  14. data/lib/saboteur/doctor.rb +8 -1
  15. data/lib/saboteur/engine/coverage_store.rb +54 -3
  16. data/lib/saboteur/engine/diagnostics.rb +4 -3
  17. data/lib/saboteur/engine/distribution.rb +6 -2
  18. data/lib/saboteur/engine/pipelines.rb +14 -2
  19. data/lib/saboteur/engine/scheduler.rb +1 -1
  20. data/lib/saboteur/engine/selection_announcement.rb +5 -4
  21. data/lib/saboteur/engine/subject_selection.rb +10 -0
  22. data/lib/saboteur/engine.rb +70 -12
  23. data/lib/saboteur/help.rb +11 -0
  24. data/lib/saboteur/ignore_list.rb +22 -2
  25. data/lib/saboteur/mutation_source/builtin.rb +1 -0
  26. data/lib/saboteur/mutation_source/operator_explain.rb +181 -0
  27. data/lib/saboteur/reporter/agent_commands.rb +23 -6
  28. data/lib/saboteur/reporter/agent_incomplete.rb +9 -2
  29. data/lib/saboteur/reporter/agent_summary.rb +2 -1
  30. data/lib/saboteur/reporter/default_findings.rb +83 -0
  31. data/lib/saboteur/reporter/gap_kinds.rb +80 -0
  32. data/lib/saboteur/reporter/html.rb +19 -4
  33. data/lib/saboteur/reporter/next_steps.rb +1 -0
  34. data/lib/saboteur/reporter/review_comments.rb +8 -5
  35. data/lib/saboteur/reporter/score_table.rb +8 -7
  36. data/lib/saboteur/reporter/speed_footer.rb +74 -0
  37. data/lib/saboteur/reporter/survivor_classification.rb +31 -8
  38. data/lib/saboteur/reporter/survivor_hints.rb +5 -3
  39. data/lib/saboteur/reporter/survivor_list.rb +78 -7
  40. data/lib/saboteur/reporter/terminal.rb +52 -36
  41. data/lib/saboteur/reporter/wording.rb +37 -0
  42. data/lib/saboteur/result.rb +28 -2
  43. data/lib/saboteur/survivor_guidance.rb +2 -2
  44. data/lib/saboteur/text.rb +6 -0
  45. data/lib/saboteur/version.rb +1 -1
  46. metadata +8 -1
@@ -169,6 +169,11 @@ module Saboteur
169
169
  return missed.map { |pattern| missed_pattern_message(pattern) }.join("; ")
170
170
  end
171
171
 
172
+ if @config["since"]
173
+ return "--subject #{missed.join(', ')} matches no subject in the --since #{@config['since']} " \
174
+ "diff scope (the scope limits the search to changed files; drop --since or pass --all)"
175
+ end
176
+
172
177
  "--subject #{missed.join(', ')} matches no subject (patterns are names like " \
173
178
  "Klass, Klass#meth or Klass.meth, not globs)"
174
179
  end
@@ -180,6 +185,11 @@ module Saboteur
180
185
  macro = attribute_macro_for(pattern)
181
186
  return "--subject #{pattern} is defined by #{macro} and has no body to mutate" if macro
182
187
 
188
+ if @config["since"]
189
+ return "--subject #{pattern} matches no subject in the --since #{@config['since']} diff scope " \
190
+ "(the scope limits the search to changed files)"
191
+ end
192
+
183
193
  "fix the --subject pattern \"#{pattern}\": it matches no subject (patterns are names like " \
184
194
  "Klass, Klass#meth or Klass.meth, not globs)"
185
195
  end
@@ -4,6 +4,7 @@
4
4
  require "fileutils"
5
5
  require "json"
6
6
  require_relative "replay"
7
+ require_relative "text"
7
8
  require_relative "mutation_source/hazard_guard"
8
9
  require_relative "baseline/run_evidence"
9
10
  require_relative "engine/early_stop_recording"
@@ -116,6 +117,24 @@ module Saboteur
116
117
  warnings + map.duplicate_keys(path).map { |key| "duplicate custom map key #{key}: Ruby keeps the last" }
117
118
  end
118
119
 
120
+ # The map this run selected tests from: survivor presentation compares
121
+ # it against the cached whole-suite map (review C1).
122
+ def run_coverage_map
123
+ @coverage_store&.fetched_map
124
+ end
125
+
126
+ # The cached whole-suite map on this tree, for survivor presentation
127
+ # in scoped runs; nil when caching is off or no full build ran (review C1).
128
+ def whole_suite_coverage_map
129
+ @coverage_store&.whole_suite_map
130
+ end
131
+
132
+ # False when caching is off, where no coverage build can ever leave a
133
+ # whole-suite map to consult, so advising one would be wrong (review C1).
134
+ def whole_suite_map_lookup?
135
+ config["cache"] ? true : false
136
+ end
137
+
119
138
  private
120
139
 
121
140
  # Inferred test selection, the default since 0.7.0: scope the run to the
@@ -180,7 +199,8 @@ module Saboteur
180
199
  matrix: config["matrix"], inline_suppressions: @diagnostics.inline_suppressions,
181
200
  operator_policy: config.operator_policy, discovery_failures: @diagnostics.discovery_failures,
182
201
  test_command: config["test_command"], reuse_label: reuse_label, since: config["since"],
183
- shard: @shard, memory: @memory_report, referenced: @guessed_referenced || {})
202
+ shard: @shard, memory: @memory_report, referenced: @guessed_referenced || {},
203
+ phase_timings: @phase_timings)
184
204
  end
185
205
 
186
206
  def complete_evidence(results, subjects)
@@ -207,14 +227,13 @@ module Saboteur
207
227
 
208
228
  def mutate(subjects, files, integration, scope, ignore_list, load_lines: {}, load_methods: {})
209
229
  store = coverage_store(integration, load_lines, load_methods)
210
- full_passes = config["baseline_passes"] + (store.reusable?(files) ? 0 : 1)
211
- warn_unscoped(integration, full_passes)
212
- announce_quiet_start(store, files)
213
- check_baseline(integration, full_passes)
230
+ map_reusable = store.reusable?(files)
231
+ full_passes = config["baseline_passes"] + (map_reusable ? 0 : 1)
232
+ baseline_preflight(store, integration, files, full_passes)
214
233
  refuse_unaffordable_coverage_build(store, files)
215
234
  return [] if abandon_for_budget?(subjects)
216
235
 
217
- map = store.fetch(files)
236
+ map, map_seconds = timed_map_fetch(store, files)
218
237
  inform_about_run(subjects, map)
219
238
  cache = result_cache(map, files)
220
239
  scheduler = scheduler_for(map, integration, cache)
@@ -227,13 +246,41 @@ module Saboteur
227
246
  # mutant when serial. Pack the heap now that the map and candidates
228
247
  # are retained, to reduce later copying of shared pages.
229
248
  Execution::ForkReadiness.prepare_heap
230
- results = run_and_confirm(runnable_subjects, runnable_lists, scheduler, cache, ignore_list)
249
+ results = run_and_confirm_timed(runnable_subjects, runnable_lists, scheduler, cache, ignore_list,
250
+ map_cached: map_reusable, map_seconds:)
231
251
  cache&.save
232
252
  results
233
253
  rescue BudgetPhaseTimeout => e
234
254
  budget_phase_stop(e, subjects)
235
255
  end
236
256
 
257
+ # The map phase's monotonic split, for the speed footer.
258
+ def timed_map_fetch(store, files)
259
+ started = monotonic
260
+ [store.fetch(files), monotonic - started]
261
+ end
262
+
263
+ # Baseline phase: announce, then verify the unchanged tree passes.
264
+ def baseline_preflight(store, integration, files, full_passes)
265
+ warn_unscoped(integration, full_passes)
266
+ announce_quiet_start(store, files)
267
+ check_baseline(integration, full_passes)
268
+ end
269
+
270
+ # The mutant phase plus the run's three disjoint timing intervals
271
+ # (baseline inside check_baseline, map build, mutants) and the worker
272
+ # count, recorded for the terminal's speed footer.
273
+ def run_and_confirm_timed(subjects, lists, scheduler, cache, ignore_list, map_cached:, map_seconds:)
274
+ started = monotonic
275
+ results = run_and_confirm(subjects, lists, scheduler, cache, ignore_list)
276
+ # @baseline_elapsed is a Float from initialize on, so no fallback is needed.
277
+ @phase_timings = PhaseTimings.new(baseline_seconds: @baseline_elapsed,
278
+ map_seconds:, map_cached:,
279
+ mutants_seconds: monotonic - started,
280
+ jobs: config["jobs"])
281
+ results
282
+ end
283
+
237
284
  # The candidate run plus the opt-in post-run confirmation pass (#1063):
238
285
  # confirmation runs after the last result, before the verdict cache is
239
286
  # written, so a confirmed kill overwrites the survivor verdict the run
@@ -312,16 +359,27 @@ module Saboteur
312
359
  tests: map.tests.size, cached: @map_cached)
313
360
  @diagnostics.quiet_inference(@quiet_spec_count)
314
361
  else
315
- @diagnostics.inform("#{subjects.size} subjects, #{map.tests.size} tests in coverage map")
362
+ @diagnostics.inform("#{Text.counted(subjects.size, 'subject')}, " \
363
+ "#{Text.counted(map.tests.size, 'test')} in coverage map")
316
364
  end
317
365
  ThreadWarnings.warn(@diagnostics, rails: config["rails"])
318
366
  end
319
367
 
368
+ # This run's map is the whole-suite map only when nothing scoped it:
369
+ # no --since, no subjects, and tests either unset or inferred over every
370
+ # included file (never an explicit --test list) (review C1).
371
+ def whole_suite_run?
372
+ config["since"].nil? && config["subjects"].empty? &&
373
+ (config["tests"].empty? || !@guessed_tests.nil?)
374
+ end
375
+
320
376
  def coverage_store(integration, load_lines, load_methods = {})
321
- CoverageStore.new(config: config, integration: integration, hooks: hooks, load_lines:, load_methods:,
322
- supervisor: supervisor(memory_factor: 4), diagnostics: @diagnostics,
323
- timeout_cap: method(:budget_capped_timeout),
324
- parts: config["coverage_parts"])
377
+ @coverage_store ||=
378
+ CoverageStore.new(config: config, integration: integration, hooks: hooks, load_lines:, load_methods:,
379
+ whole_suite_run: whole_suite_run?,
380
+ supervisor: supervisor(memory_factor: 4), diagnostics: @diagnostics,
381
+ timeout_cap: method(:budget_capped_timeout),
382
+ parts: config["coverage_parts"])
325
383
  end
326
384
 
327
385
  def check_baseline(integration, full_passes)
data/lib/saboteur/help.rb CHANGED
@@ -152,6 +152,17 @@ module Saboteur
152
152
  ["Diff, replay command and ignore template for one mutant",
153
153
  "saboteur survivors .saboteur/report.json --show 1a2b3c4d5e6f7a8b"]]
154
154
  ),
155
+ Command.new(
156
+ name: "explain",
157
+ summary: "Explain one mutation operator: definition, example and kill guidance",
158
+ description: "Prints what one operator does, a before/after example of the mutation, the test " \
159
+ "shapes that let it survive, and the assertion that kills it. Covers every operator " \
160
+ "the registry can emit.",
161
+ takes_options: false,
162
+ exit_codes: { 0 => "Explanation printed.",
163
+ 2 => "Unknown or missing operator name; the valid names are listed." },
164
+ examples: [["Why did this mutant survive?", "saboteur explain operand_replacement"]]
165
+ ),
155
166
  Command.new(
156
167
  name: "badge",
157
168
  summary: "Print a shields.io endpoint badge over the opt-in score history",
@@ -125,7 +125,7 @@ module Saboteur
125
125
  # unscoped_digests (subject id => every digest generated before a diff
126
126
  # scope cut) keeps a scoped run from calling a scoped-out entry stale
127
127
  # (#284).
128
- def apply(results, unscoped_digests: nil)
128
+ def apply(results, subjects: nil, unscoped_digests: nil, full_run: false)
129
129
  seen = {}
130
130
  results.each do |result|
131
131
  key = [result.subject.id, result.digest]
@@ -139,7 +139,8 @@ module Saboteur
139
139
  result.triage_reason = entry.reason
140
140
  result.ignored = entry.reason if entry.state != "todo"
141
141
  end
142
- duplicate_warnings + stale_warnings(results, seen, unscoped_digests)
142
+ duplicate_warnings + stale_warnings(results, seen, unscoped_digests) +
143
+ missing_subject_warnings(results, subjects, full_run: full_run)
143
144
  end
144
145
 
145
146
  private
@@ -159,6 +160,25 @@ module Saboteur
159
160
  .map { |(subject, mutation), _| "duplicate ignore entry #{subject} #{mutation}: the last entry wins" }
160
161
  end
161
162
 
163
+ # A full run examines every in-scope subject, so an entry whose subject
164
+ # never appears points at code that was renamed or removed - say so, but
165
+ # only then: a scoped (--since/--subject) run cannot tell "renamed" from
166
+ # "not in this diff" (#1206). The run's subject list is the source of
167
+ # truth when given: an examined subject with no results (uncovered, or
168
+ # nothing mutable left) is not missing.
169
+ def missing_subject_warnings(results, subjects, full_run:)
170
+ return [] unless full_run
171
+
172
+ examined = (subjects || results).to_set { |item| item.respond_to?(:id) ? item.id : item.subject.id }
173
+ entries.reject { |entry| examined.include?(entry.subject) }
174
+ .map { |entry| missing_subject_message(entry) }
175
+ end
176
+
177
+ def missing_subject_message(entry)
178
+ "stale ignore entry #{entry.subject} #{entry.mutation}: subject not found " \
179
+ "(renamed, removed, or its file is excluded?) - remove the entry or fix the subject name"
180
+ end
181
+
162
182
  def stale_message(entry)
163
183
  "stale ignore entry #{entry.subject} #{entry.mutation}: no such mutation " \
164
184
  "- remove the entry or refresh its digest (saboteur survivors --show)"
@@ -13,6 +13,7 @@ require_relative "pair_removal"
13
13
  require_relative "element_removal"
14
14
  require_relative "endpoint_replacement"
15
15
  require_relative "operator_registry"
16
+ require_relative "operator_explain"
16
17
  require_relative "range_replacement"
17
18
  require_relative "operator_assignment"
18
19
  require_relative "condition_negation"
@@ -0,0 +1,181 @@
1
+ # SPDX-License-Identifier: MIT
2
+ # frozen_string_literal: true
3
+
4
+ require_relative "operator_examples"
5
+ require_relative "operator_registry"
6
+
7
+ module Saboteur
8
+ module MutationSource
9
+ # Curated text behind `saboteur explain NAME`. The definition and the
10
+ # example stay shared with --list-operators (one wording source); the
11
+ # survival and kill hints below are the only text owned here. Every
12
+ # registry operator needs an entry: the coverage spec fails the build
13
+ # when one ships without it.
14
+ module OperatorExplain
15
+ Entry = Data.define(:name, :definition, :example, :survives, :kill)
16
+
17
+ GUIDANCE = {
18
+ "body_nil" => {
19
+ survives: "the method is called for side effects and no test asserts the return value.",
20
+ kill: "an assertion on the exact return value, or on a caller behaviour that depends on it."
21
+ },
22
+ "body_true" => {
23
+ survives: "tests never check the predicate outcome, or only take the path true already forces.",
24
+ kill: "an example whose input makes the predicate false, asserting that path is taken."
25
+ },
26
+ "body_false" => {
27
+ survives: "tests never check the predicate outcome, or only take the path false already forces.",
28
+ kill: "an example whose input makes the predicate true, asserting that path is taken."
29
+ },
30
+ "body_raise" => {
31
+ survives: "tests stub or skip the method, or wrap the call in a broad rescue.",
32
+ kill: "any test that calls the method and lets it complete; the raise fails that example."
33
+ },
34
+ "comparison" => {
35
+ survives: "tests never use values at the boundary the change moves, such as equal values for < vs <=.",
36
+ kill: "a boundary example: the equal case, or the two values just either side of the operator."
37
+ },
38
+ "arithmetic" => {
39
+ survives: "tests assert the result loosely (type, presence, sign) instead of its exact value.",
40
+ kill: "an assertion on the exact computed value with operands where the two operators differ."
41
+ },
42
+ "boolean_logic" => {
43
+ survives: "tests never exercise the dropped operand deciding the outcome on its own.",
44
+ kill: "one example per operand where that operand alone flips the result, asserting each outcome."
45
+ },
46
+ "condition_true" => {
47
+ survives: "tests only exercise the branch the forced condition takes.",
48
+ kill: "an example with a falsy input, asserting the else branch still runs."
49
+ },
50
+ "condition_false" => {
51
+ survives: "tests only exercise the branch the forced condition takes.",
52
+ kill: "an example with a truthy input, asserting the then branch still runs."
53
+ },
54
+ "negation_removal" => {
55
+ survives: "tests assert loosely, or never cover the input the negation inverts.",
56
+ kill: "assert the behaviour for both a truthy and a falsy input through the public API."
57
+ },
58
+ "literal_number" => {
59
+ survives: "the literal flows into nothing a test asserts, or every fixture value works for both.",
60
+ kill: "an assertion on the exact value the literal feeds (a total, a limit, an index)."
61
+ },
62
+ "or_assign" => {
63
+ survives: "tests only cover the nil or false case, where both forms assign.",
64
+ kill: "an example where the variable is already truthy and must keep its value."
65
+ },
66
+ "and_assign" => {
67
+ survives: "tests only cover the truthy case, where both forms assign.",
68
+ kill: "an example where the variable is falsy and must stay untouched."
69
+ },
70
+ "safe_navigation" => {
71
+ survives: "the receiver is never nil in any covering test.",
72
+ kill: "an example with a nil receiver asserting nil comes back instead of NoMethodError."
73
+ },
74
+ "statement_removal" => {
75
+ survives: "the removed statement's effect is never observed: an unasserted side effect or return.",
76
+ kill: "assert the side effect (a spy, a state change) or the value the statement produced."
77
+ },
78
+ "copy_replacement" => {
79
+ survives: "tests never mutate the returned object, so the aliasing goes unnoticed.",
80
+ kill: "mutate what the method returns and assert the original is unchanged (or assert a copy)."
81
+ },
82
+ "filter_replacement" => {
83
+ survives: "tests assert loosely, or the fixture collection is empty or all one kind.",
84
+ kill: "assert the exact surviving elements of a mixed collection, in order."
85
+ },
86
+ "literal_string" => {
87
+ survives: "no test asserts the string's content, only that something was returned.",
88
+ kill: "an exact string equality assertion on the value that carries the literal."
89
+ },
90
+ "literal_boolean" => {
91
+ survives: "the boolean is never asserted, or both values pass the covering assertions.",
92
+ kill: "an example whose expected behaviour flips with the boolean."
93
+ },
94
+ "argument_swap" => {
95
+ survives: "tests use equal or interchangeable arguments, or assert the result loosely.",
96
+ kill: "asymmetric arguments with an assertion on the exact result order."
97
+ },
98
+ "argument_removal" => {
99
+ survives: "the removed argument changes nothing the covering tests observe.",
100
+ kill: "a case where that argument changes the result, asserted exactly."
101
+ },
102
+ "pair_removal" => {
103
+ survives: "tests never read the removed key, directly or through a consumer.",
104
+ kill: "assert the value stored under the removed key, or the whole hash."
105
+ },
106
+ "element_removal" => {
107
+ survives: "tests never exercise the removed element of the array.",
108
+ kill: "an example per element whose behaviour changes when that element is missing."
109
+ },
110
+ "endpoint_replacement" => {
111
+ survives: "fixture collections have one element, or the wrong end is never asserted.",
112
+ kill: "a multi-element collection with an assertion on exactly which end comes back."
113
+ },
114
+ "chain_removal" => {
115
+ survives: "in every tested case the skipped call returns an equivalent value.",
116
+ kill: "a case where the removed call changes the value, asserted exactly."
117
+ },
118
+ "range" => {
119
+ survives: "tests never use the endpoint the exclusive range drops.",
120
+ kill: "an example whose input is the boundary value, asserting it is included."
121
+ },
122
+ "operator_assignment" => {
123
+ survives: "tests check the variable changed, not its exact new value.",
124
+ kill: "assert the exact value after the assignment, where += and -= disagree."
125
+ },
126
+ "condition_negation" => {
127
+ survives: "tests only cover the side of the predicate the negation keeps.",
128
+ kill: "assert both branches with inputs on each side of the predicate."
129
+ },
130
+ "method_swap" => {
131
+ survives: "tests assert loosely, or min and max coincide in every fixture.",
132
+ kill: "a fixture where the two methods differ, asserting the exact expected one."
133
+ },
134
+ "operand_replacement" => {
135
+ survives: "tests assert the result loosely (present, positive, non-nil) or never exercise " \
136
+ "the removed operand.",
137
+ kill: "an assertion on the exact value through the public API, with inputs where the " \
138
+ "removed operand changes the result."
139
+ },
140
+ "interpolation_removal" => {
141
+ survives: "no test asserts the full interpolated content of the string.",
142
+ kill: "an exact string assertion with a known value in the interpolation."
143
+ },
144
+ "jump_value" => {
145
+ survives: "callers ignore the returned value, or tests never take that return path.",
146
+ kill: "an example that takes the early return and asserts the exact value it hands back."
147
+ },
148
+ "rescue_removal" => {
149
+ survives: "tests never raise the error the handler rescues.",
150
+ kill: "an example raising that error, asserting the handler's fallback behaviour."
151
+ },
152
+ "ensure_removal" => {
153
+ survives: "the cleanup's effect is never observed on the tested paths.",
154
+ kill: "assert the cleanup ran (resource released, flag reset), including on the error path."
155
+ },
156
+ "when_removal" => {
157
+ survives: "tests never hit the removed branch of the case.",
158
+ kill: "one example per when branch, asserting its exact outcome."
159
+ },
160
+ "unary_removal" => {
161
+ survives: "fixtures use zero or symmetric values, or tests assert loosely.",
162
+ kill: "an assertion on the exact signed or complemented value with a non-trivial input."
163
+ },
164
+ "regex_replacement" => {
165
+ survives: "every test input matches under both the original and the loosened regex.",
166
+ kill: "an input the anchor or quantifier excludes, asserting it is rejected."
167
+ }
168
+ }.freeze
169
+
170
+ # nil for an unknown name; the command turns that into the exit-2 error.
171
+ def self.for(name)
172
+ return unless OperatorRegistry::DESCRIPTIONS.key?(name)
173
+
174
+ guidance = GUIDANCE.fetch(name)
175
+ Entry.new(name:, definition: OperatorRegistry::DESCRIPTIONS.fetch(name),
176
+ example: OperatorExamples::EXAMPLES.fetch(name),
177
+ survives: guidance.fetch(:survives), kill: guidance.fetch(:kill))
178
+ end
179
+ end
180
+ end
181
+ end
@@ -45,14 +45,30 @@ module Saboteur
45
45
 
46
46
  # Replay one survivor by digest in the same selection. capped: false
47
47
  # skips the protocol byte cap, for callers that print to a terminal.
48
- def replay(provenance, digest, capped: true)
49
- words = ["saboteur", "run", "--id", digest, *selection(provenance, replay: true), "--seed",
50
- provenance.seed.to_s]
48
+ # report: narrows the replay to the mutant's covering spec files
49
+ # instead of the run's full inferred list (#1148).
50
+ def replay(provenance, digest, capped: true, report: nil)
51
+ words = ["saboteur", "run", "--id", digest,
52
+ *selection(provenance, replay: true, tests: report && covering_files(report, digest)),
53
+ "--seed", provenance.seed.to_s]
51
54
  finish(words, provenance, capped: capped)
52
55
  end
53
56
 
54
57
  private
55
58
 
59
+ # A replay needs only the specs that cover the mutant, not every
60
+ # inferred file (#1148): covered_by holds RSpec ids like
61
+ # "./spec/fee_spec.rb[1:2]", so strip the example suffix. nil when
62
+ # the survivor has no covering ids - then the whole selection
63
+ # stays, since dropping it would widen the replay to inference.
64
+ def covering_files(report, digest)
65
+ result = report.results.find { |entry| entry.digest == digest }
66
+ return unless result
67
+
68
+ files = result.covered_by.filter_map { |id| id[%r{\A\.?/?(.+?\.rb)\[}, 1] }.uniq
69
+ files unless files.empty?
70
+ end
71
+
56
72
  def finish(words, provenance, capped: true)
57
73
  words += ["--", *provenance.sources] unless provenance.sources.empty?
58
74
  command = Shellwords.join(words)
@@ -93,13 +109,14 @@ module Saboteur
93
109
 
94
110
  # Whole-suite scope is a bare --all, only when no tests resolved;
95
111
  # --all never combines with --test.
96
- def selection(provenance, replay: false)
112
+ def selection(provenance, replay: false, tests: nil)
113
+ tests ||= provenance.tests
97
114
  words = []
98
115
  words.push("--config", provenance.config_path) if provenance.config_path
99
116
  words.push("--since", provenance.ref) if provenance.ref
100
117
  provenance.subjects.each { |subject| words.push("--subject", subject) }
101
- words << "--all" if provenance.settings[:all] && provenance.tests.empty?
102
- words.push(*test_words(provenance.tests))
118
+ words << "--all" if provenance.settings[:all] && tests.empty?
119
+ words.push(*test_words(tests))
103
120
  words.concat(setting_words(provenance, replay: replay))
104
121
  words.concat(mode_words(provenance))
105
122
  words.push("--id", provenance.digest) if provenance.digest && !replay
@@ -40,8 +40,7 @@ module Saboteur
40
40
  add(reasons, "mutant_cap", omitted_by_cap(report), "mutants omitted by max_mutants_per_method")
41
41
  add(reasons, "errors", report.counts[:error], "mutants errored; their verdict did not hold up")
42
42
  add(reasons, "resource_limit", report.counts[:resource_limit], "mutants hit the memory or process cap")
43
- add(reasons, "unverified", report.counts[:unverified],
44
- "mutants the --max-duration budget left unverified; their kills are not confirmed")
43
+ unverified_reasons(reasons, report)
45
44
  discovery_failures(reasons, report)
46
45
  early_stop_reason(reasons, report)
47
46
  inline_scope(reasons, report)
@@ -70,6 +69,14 @@ module Saboteur
70
69
  "included untracked files could not be read; their subjects were not covered")
71
70
  end
72
71
 
72
+ def unverified_reasons(reasons, report)
73
+ flaky = report.results.count(&:flaky?)
74
+ add(reasons, "unverified", report.counts[:unverified] - flaky,
75
+ "mutants the --max-duration budget left unverified; their kills are not confirmed")
76
+ add(reasons, "flaky", flaky,
77
+ "mutants gave a different verdict on the --flaky-check reordered rerun; they are unverified")
78
+ end
79
+
73
80
  def early_stop_reason(reasons, report)
74
81
  stop = report.early_stop
75
82
  return unless stop
@@ -123,7 +123,8 @@ module Saboteur
123
123
  steps = {}
124
124
  add_command(steps, :rerun, Commands.rerun(provenance))
125
125
  first = Findings.build(report)[:survivors].first
126
- add_command(steps, :replay, Commands.replay(provenance, first[:digest])) if first
126
+ # report: narrows the replay to covering specs, as on the run screen (#1148).
127
+ add_command(steps, :replay, Commands.replay(provenance, first[:digest], report: report)) if first
127
128
  steps[:fullReport] = provenance.json_report if short_path?(provenance.json_report)
128
129
  steps
129
130
  end
@@ -0,0 +1,83 @@
1
+ # SPDX-License-Identifier: MIT
2
+ # frozen_string_literal: true
3
+
4
+ require_relative "mutation_text"
5
+ require_relative "survivor_classification"
6
+ require_relative "survivor_list"
7
+
8
+ module Saboteur
9
+ module Reporter
10
+ # Default whole-project findings (D1): the one-line-per-finding list the
11
+ # --quiet report popularised, then full blocks for the top
12
+ # SurvivorRanking picks only, then a dim closer naming the rest. A
13
+ # --since run or --details keeps the full block list.
14
+ class DefaultFindings
15
+ # Blocks a default run expands; the closer counts what is left out.
16
+ TOP_BLOCKS = 3
17
+
18
+ # Two lines: the fixed text alone is 87 columns at rest; keep it inside 80.
19
+ CLOSER = "... %<more>d more %<findings>s - saboteur survivors .saboteur/report.json,\nor re-run with --details"
20
+
21
+ def initialize(out, palette, all_mutants: false, details: false,
22
+ whole_suite_map: SurvivorList::NO_WHOLE_SUITE_LOOKUP, run_map: nil)
23
+ @out = out
24
+ @palette = palette
25
+ @all_mutants = all_mutants
26
+ @details = details
27
+ @whole_suite_map = whole_suite_map
28
+ @run_map = run_map
29
+ end
30
+
31
+ def call(report, ranked)
32
+ live = ranked.select(&:unignored_survivor?)
33
+ return full(report, ranked) if @details || report.diff? || live.empty?
34
+
35
+ @out.puts(list_lines(report, live))
36
+ @out.puts
37
+ SurvivorList.new(@out, @palette, all_mutants: @all_mutants, limit: TOP_BLOCKS,
38
+ whole_suite_map: @whole_suite_map, run_map: @run_map).call(report, ranked)
39
+ more = finding_count(report, live) - TOP_BLOCKS
40
+ findings = "finding#{'s' unless more == 1}"
41
+ @out.puts @palette.paint(:dim, format(CLOSER, more:, findings:)) if more.positive?
42
+ @out.puts
43
+ end
44
+
45
+ private
46
+
47
+ # --details and --since runs: every finding expanded, exactly the
48
+ # pre-D1 default.
49
+ def full(report, ranked)
50
+ SurvivorList.new(@out, @palette, all_mutants: @all_mutants,
51
+ whole_suite_map: @whole_suite_map, run_map: @run_map).call(report, ranked)
52
+ @out.puts unless report.survivors.empty?
53
+ end
54
+
55
+ # The quiet list's line shape: path:line subject, label and digest,
56
+ # sibling mutants on the statement collapsed to a count (#1126/#1133).
57
+ def list_lines(report, live)
58
+ groups(report, live).map do |(path, line, subject_id), results|
59
+ first, *rest = results
60
+ label = SurvivorClassification.label(first)
61
+ text = "#{path}:#{line} #{subject_id}#{" [#{label}]" if label} - #{first.digest}"
62
+ rest.any? ? "#{text} (+#{rest.size} more)" : text
63
+ end
64
+ end
65
+
66
+ # --all-mutants gives every survivor its own block, so the closer
67
+ # counts mutants rather than statements.
68
+ def finding_count(report, live)
69
+ @all_mutants ? live.size : groups(report, live).size
70
+ end
71
+
72
+ def groups(report, live)
73
+ live.group_by do |result|
74
+ [relative(result.subject.file, report.root), MutationText.line(result), result.subject.id]
75
+ end
76
+ end
77
+
78
+ def relative(file, root)
79
+ file.delete_prefix("#{root}/")
80
+ end
81
+ end
82
+ end
83
+ end