saboteur 0.8.0 → 0.9.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (128) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +215 -0
  3. data/README.md +44 -6
  4. data/exe/saboteur +6 -2
  5. data/lib/saboteur/attribute_finder.rb +116 -0
  6. data/lib/saboteur/baseline/run_evidence.rb +2 -1
  7. data/lib/saboteur/cli/agent_provenance.rb +59 -0
  8. data/lib/saboteur/cli/dirty_tree_notice.rb +28 -0
  9. data/lib/saboteur/cli/doctor_command.rb +41 -0
  10. data/lib/saboteur/cli/error_cause.rb +40 -0
  11. data/lib/saboteur/cli/execution_options.rb +46 -0
  12. data/lib/saboteur/cli/exit_status.rb +12 -14
  13. data/lib/saboteur/cli/explicit_file_options.rb +21 -0
  14. data/lib/saboteur/cli/explicit_sources.rb +46 -0
  15. data/lib/saboteur/cli/failure_output.rb +35 -0
  16. data/lib/saboteur/cli/help_command.rb +23 -0
  17. data/lib/saboteur/cli/matrix_option.rb +17 -0
  18. data/lib/saboteur/cli/output_options.rb +27 -0
  19. data/lib/saboteur/cli/presets.rb +41 -1
  20. data/lib/saboteur/cli/preview_command.rb +26 -0
  21. data/lib/saboteur/cli/run_budget_options.rb +6 -1
  22. data/lib/saboteur/cli/run_command.rb +33 -0
  23. data/lib/saboteur/cli/run_output.rb +38 -0
  24. data/lib/saboteur/cli/run_reporting.rb +6 -5
  25. data/lib/saboteur/cli/run_validation.rb +54 -0
  26. data/lib/saboteur/cli/source_set.rb +0 -18
  27. data/lib/saboteur/cli.rb +29 -305
  28. data/lib/saboteur/config.rb +76 -7
  29. data/lib/saboteur/config_file.rb +12 -8
  30. data/lib/saboteur/constant_subject_finder.rb +3 -0
  31. data/lib/saboteur/custom_map.rb +24 -5
  32. data/lib/saboteur/deep_source.rb +49 -0
  33. data/lib/saboteur/dirty_tree_warning.rb +6 -4
  34. data/lib/saboteur/doctor.rb +10 -0
  35. data/lib/saboteur/engine/baseline_guard.rb +118 -15
  36. data/lib/saboteur/engine/baseline_parts.rb +51 -0
  37. data/lib/saboteur/engine/boot.rb +11 -3
  38. data/lib/saboteur/engine/bytecode_equivalence.rb +17 -1
  39. data/lib/saboteur/engine/candidate_selection.rb +16 -5
  40. data/lib/saboteur/engine/candidates.rb +4 -1
  41. data/lib/saboteur/engine/coverage_inputs_snapshot.rb +146 -0
  42. data/lib/saboteur/engine/coverage_miss_note.rb +86 -0
  43. data/lib/saboteur/engine/coverage_partial.rb +48 -0
  44. data/lib/saboteur/engine/coverage_parts.rb +50 -0
  45. data/lib/saboteur/engine/coverage_shadow.rb +92 -0
  46. data/lib/saboteur/engine/coverage_store.rb +296 -21
  47. data/lib/saboteur/engine/diagnostics.rb +24 -7
  48. data/lib/saboteur/engine/distribution.rb +2 -2
  49. data/lib/saboteur/engine/gated_run.rb +4 -1
  50. data/lib/saboteur/engine/load_coverage.rb +72 -8
  51. data/lib/saboteur/engine/partial_invalidation.rb +115 -0
  52. data/lib/saboteur/engine/partial_merge.rb +40 -0
  53. data/lib/saboteur/engine/pipelines.rb +11 -3
  54. data/lib/saboteur/engine/reference_specs.rb +71 -0
  55. data/lib/saboteur/engine/result_cache.rb +68 -5
  56. data/lib/saboteur/engine/resume_manifest.rb +110 -0
  57. data/lib/saboteur/engine/run_budget.rb +10 -7
  58. data/lib/saboteur/engine/run_fingerprint.rb +2 -1
  59. data/lib/saboteur/engine/sandbox_policy.rb +26 -5
  60. data/lib/saboteur/engine/scheduler.rb +33 -25
  61. data/lib/saboteur/engine/selection_announcement.rb +9 -3
  62. data/lib/saboteur/engine/shadow_comparison.rb +121 -0
  63. data/lib/saboteur/engine/shadow_run.rb +38 -0
  64. data/lib/saboteur/engine/subject_selection.rb +144 -64
  65. data/lib/saboteur/engine/suite_partition.rb +22 -0
  66. data/lib/saboteur/engine/test_guess.rb +186 -32
  67. data/lib/saboteur/engine/unchanged_syntax.rb +10 -2
  68. data/lib/saboteur/engine.rb +44 -11
  69. data/lib/saboteur/environment_hints.rb +55 -0
  70. data/lib/saboteur/execution/bundle_gemfile_pin.rb +72 -0
  71. data/lib/saboteur/execution/external_runner.rb +123 -1
  72. data/lib/saboteur/execution/fiddle_loader.rb +34 -0
  73. data/lib/saboteur/execution/gemfile_path_sources.rb +166 -0
  74. data/lib/saboteur/execution/marker_sweep.rb +2 -1
  75. data/lib/saboteur/execution/process_table.rb +7 -3
  76. data/lib/saboteur/execution/reevaluator.rb +7 -0
  77. data/lib/saboteur/execution/sandbox.rb +39 -20
  78. data/lib/saboteur/execution/seatbelt.rb +10 -3
  79. data/lib/saboteur/execution/supervisor.rb +42 -8
  80. data/lib/saboteur/execution/supervisor_batch.rb +105 -0
  81. data/lib/saboteur/execution/test_command_shape.rb +149 -0
  82. data/lib/saboteur/execution/worker_pool.rb +48 -4
  83. data/lib/saboteur/frameworks/rails.rb +2 -5
  84. data/lib/saboteur/frameworks/sqlite_memory_database.rb +32 -2
  85. data/lib/saboteur/gates.rb +121 -28
  86. data/lib/saboteur/install.rb +6 -1
  87. data/lib/saboteur/integration/base.rb +19 -2
  88. data/lib/saboteur/integration/rspec.rb +145 -18
  89. data/lib/saboteur/line_hits.rb +81 -0
  90. data/lib/saboteur/method_calls.rb +91 -0
  91. data/lib/saboteur/mutation_source/argument_swap.rb +54 -0
  92. data/lib/saboteur/mutation_source/builtin.rb +33 -21
  93. data/lib/saboteur/mutation_source/condition_negation.rb +3 -12
  94. data/lib/saboteur/mutation_source/hazard_guard.rb +94 -13
  95. data/lib/saboteur/mutation_source/jump_value.rb +59 -0
  96. data/lib/saboteur/mutation_source/literal_replacement.rb +6 -23
  97. data/lib/saboteur/mutation_source/operator_examples.rb +6 -0
  98. data/lib/saboteur/mutation_source/operator_registry.rb +6 -0
  99. data/lib/saboteur/mutation_source/operator_visit.rb +7 -1
  100. data/lib/saboteur/mutation_source/rescue_removal.rb +69 -0
  101. data/lib/saboteur/mutation_source/unary_removal.rb +47 -0
  102. data/lib/saboteur/mutation_source/when_removal.rb +51 -0
  103. data/lib/saboteur/preview.rb +13 -0
  104. data/lib/saboteur/refinement_activations.rb +22 -10
  105. data/lib/saboteur/reporter/agent.rb +2 -2
  106. data/lib/saboteur/reporter/agent_incomplete.rb +31 -5
  107. data/lib/saboteur/reporter/agent_summary.rb +16 -10
  108. data/lib/saboteur/reporter/review_comments.rb +151 -0
  109. data/lib/saboteur/reporter/safe_out.rb +32 -0
  110. data/lib/saboteur/reporter/stryker.rb +8 -6
  111. data/lib/saboteur/reporter/survivor_classification.rb +41 -3
  112. data/lib/saboteur/reporter/survivor_hints.rb +5 -0
  113. data/lib/saboteur/reporter/terminal.rb +45 -5
  114. data/lib/saboteur/result.rb +14 -4
  115. data/lib/saboteur/selection/coverage_map.rb +150 -9
  116. data/lib/saboteur/selection/external_test_map.rb +4 -0
  117. data/lib/saboteur/selection/git_diff.rb +24 -2
  118. data/lib/saboteur/selection/layout_only_lines.rb +45 -1
  119. data/lib/saboteur/selection/recorder.rb +52 -22
  120. data/lib/saboteur/selection/trace_recorder.rb +83 -0
  121. data/lib/saboteur/selection/untracked_changes.rb +1 -2
  122. data/lib/saboteur/subject.rb +16 -0
  123. data/lib/saboteur/subject_finder.rb +91 -6
  124. data/lib/saboteur/text.rb +31 -0
  125. data/lib/saboteur/version.rb +1 -1
  126. data/lib/saboteur/working_tree.rb +7 -2
  127. data/lib/saboteur.rb +9 -0
  128. metadata +48 -16
@@ -60,59 +60,152 @@ module Saboteur
60
60
  !@limits.fetch("max_errors").nil?
61
61
  end
62
62
 
63
- # Prints each failed gate to err and remembers the first one's cause for the JSON report.
64
- def failed?(report, err)
65
- vacuous_gates(report).each { |note| err.puts("saboteur: #{note}") }
63
+ # Prints the gate notes and each failed gate to err. The first
64
+ # violation's cause is recorded for the JSON report unless the caller
65
+ # already has a cause that must win (record_cause: false).
66
+ def failed?(report, err, record_cause: true)
67
+ overlap_notes(report).each { |note| err.puts("saboteur: #{note}") }
66
68
  found = violations(report)
67
69
  found.each { |violation| err.puts("saboteur: #{violation.message}") }
68
- @cause = found.first&.cause
69
- !@cause.nil?
70
+ @cause = found.first&.cause if record_cause
71
+ found.any?
70
72
  end
71
73
 
72
74
  def violations(report)
73
75
  SETTINGS.filter_map do |key, (status, cause, flag)|
74
76
  limit = @limits.fetch(key)
77
+ next if limit.nil?
78
+
75
79
  count = report.counts.fetch(status)
76
- next if limit.nil? || !over?(count, limit, report.executed.size)
80
+ next unless over?(count, limit, report.executed.size)
77
81
 
78
82
  Violation.new(cause: cause, message: "#{count} #{status} mutants, above #{flag} #{limit}")
79
83
  end + score_gate_violations(report)
80
84
  end
81
85
 
86
+ # A machine-readable summary per score gate for the JSON report: the
87
+ # floor, the mutants scoring under it, its score and pass/fail state,
88
+ # the failing subjects and the matched files. nil without score gates.
89
+ def score_gate_breakdowns(report)
90
+ return if @score_gates.empty?
91
+
92
+ assignment = gate_assignment(report)
93
+ @score_gates.each_with_index.map do |gate, index|
94
+ scored = assignment.fetch(index)
95
+ verdict = gate_verdict(gate, scored)
96
+ { pattern: gate["pattern"], minScore: gate["min_score"], mutants: scored.size, tested: verdict[:tested],
97
+ score: verdict[:score]&.round(1), passed: verdict[:passed],
98
+ failingSubjects: failing_subjects(scored), files: gate_files(scored, report) }
99
+ end
100
+ end
101
+
82
102
  private
83
103
 
84
- # Per-path gates (#473): each gate scores the results whose subject file
85
- # (root-relative) matches its pattern, with the run's own Score formula,
86
- # and fails under its floor.
104
+ # Per-path gates (#473): each gate scores the results assigned to it by
105
+ # gate_matching, with the run's own Score formula, and fails under its
106
+ # floor. The failure line names the matched files (at most three, then
107
+ # a count) and the failing subject count.
87
108
  def score_gate_violations(report)
88
- @score_gates.filter_map do |gate|
89
- scored = score_gate_subset(gate, report)
90
- breakdown = Score.of(scored)
91
- next if breakdown.tested.zero?
109
+ assignment = gate_assignment(report)
110
+ @score_gates.each_with_index.filter_map do |gate, index|
111
+ scored = assignment.fetch(index)
112
+ verdict = gate_verdict(gate, scored)
113
+ next if verdict[:passed]
114
+
115
+ files = gate_files(scored, report)
116
+ listed = files.first(3).join(", ")
117
+ listed += ", +#{files.size - 3} more" if files.size > 3
118
+ message = "score gate #{gate['pattern']}: score #{format('%.1f', verdict[:score])} "
119
+ message += "below #{gate['min_score']} (#{scored.size} mutant(s) under it, "
120
+ message += "#{failing_subjects(scored)} failing subject(s), #{files.size} matched file(s): #{listed})"
121
+ Violation.new(cause: "scoreGate", message:)
122
+ end
123
+ end
92
124
 
93
- score = breakdown.value * 100
94
- next if score >= gate["min_score"]
125
+ # A gate that lost subjects to a more specific gate says so once per
126
+ # subject, even on a passing run. Only a gate that matched nothing at
127
+ # all is vacuous; a fully shadowed gate reads as overlapped instead.
128
+ def overlap_notes(report)
129
+ matched, assignment = gate_matching(report)
130
+ @score_gates.each_with_index.filter_map do |gate, index|
131
+ lost = (matched[index] - assignment.fetch(index)).map(&:subject).uniq.size
132
+ if lost.positive?
133
+ next "score gate #{gate['pattern']}: #{lost} subject(s) also match a more specific gate and score there"
134
+ end
95
135
 
96
- Violation.new(cause: "scoreGate",
97
- message: "score gate #{gate['pattern']}: score #{format('%.1f', score)} below " \
98
- "#{gate['min_score']} (#{scored.size} mutants under it)")
136
+ next if Score.of(assignment.fetch(index)).tested.positive?
137
+
138
+ # A gate that matched mutants but tested none of them (errors,
139
+ # timeouts, skips) passes just as vacuously as one that matched
140
+ # nothing; say so either way, unless a more specific gate shadows it.
141
+ if matched[index].empty?
142
+ "score gate #{gate['pattern']}: nothing executed under it, passing vacuously"
143
+ else
144
+ "score gate #{gate['pattern']}: no mutant(s) tested under it, passing vacuously"
145
+ end
99
146
  end
100
147
  end
101
148
 
102
- # A gate with nothing executed under it passes vacuously and says so.
103
- def vacuous_gates(report)
104
- @score_gates.filter_map do |gate|
105
- breakdown = Score.of(score_gate_subset(gate, report))
106
- "score gate #{gate['pattern']}: nothing executed under it, passing vacuously" if breakdown.tested.zero?
149
+ # One verdict per gate, shared by the console violation and the JSON
150
+ # breakdown: both judge the unrounded score, so a 79.96 against an 80
151
+ # floor fails in both places. The one-decimal score in the JSON is
152
+ # display-only. A gate with nothing tested passes vacuously, score nil.
153
+ def gate_verdict(gate, scored)
154
+ breakdown = Score.of(scored)
155
+ return { tested: 0, passed: true } if breakdown.tested.zero?
156
+
157
+ raw = breakdown.value * 100
158
+ { tested: breakdown.tested, score: raw, passed: raw >= gate["min_score"] }
159
+ end
160
+
161
+ # Each result scores under the single most specific matching gate: more
162
+ # literal (non-glob) pattern characters win, ties keep the earlier config
163
+ # entry (min_by returns the first minimum), so overlapping floors never
164
+ # double-count a mutant. Returns the per-gate matched and assigned lists.
165
+ def gate_matching(report)
166
+ matched = @score_gates.map { [] }
167
+ assignment = @score_gates.map { [] }
168
+ if @score_gates.any?
169
+ each_relative_result(report).each do |relative, result|
170
+ candidates = @score_gates.each_with_index.select { |gate, _index| gate_matches?(gate, relative) }
171
+ candidates.each { |candidate| matched[candidate[1]] << result }
172
+ winner = candidates.min_by { |gate, _index| -specificity(gate["pattern"]) }
173
+ assignment[winner[1]] << result if winner
174
+ end
107
175
  end
176
+ [matched, assignment]
108
177
  end
109
178
 
110
- def score_gate_subset(gate, report)
179
+ def gate_assignment(report)
180
+ gate_matching(report)[1]
181
+ end
182
+
183
+ # [relative path, result] pairs; the caller consumes the array, so the
184
+ # build cannot be swapped for a side-effect-only walk.
185
+ def each_relative_result(report)
111
186
  prefix = File.join(File.realpath(report.root), "")
112
- report.results.select do |result|
113
- relative = result.subject.file.delete_prefix(prefix)
114
- File.fnmatch?(gate["pattern"], relative, File::FNM_PATHNAME | File::FNM_EXTGLOB)
115
- end
187
+ report.results.map { |result| [result.subject.file.delete_prefix(prefix), result] }
188
+ end
189
+
190
+ def gate_matches?(gate, relative)
191
+ File.fnmatch?(gate["pattern"], relative, File::FNM_PATHNAME | File::FNM_EXTGLOB)
192
+ end
193
+
194
+ # Literal characters pin a gate to its files; glob metacharacters do
195
+ # not, and a {a,b} brace group counts once rather than per alternative,
196
+ # so the exact lib/saboteur/gates.rb outranks lib/saboteur/{gates,config}.rb.
197
+ def specificity(pattern)
198
+ pattern.gsub(/\{[^{}]*\}/, "x").gsub(/[*?\[\]\\]/, "").length
199
+ end
200
+
201
+ def gate_files(scored, report)
202
+ prefix = File.join(File.realpath(report.root), "")
203
+ scored.map { |result| result.subject.file.delete_prefix(prefix) }.uniq.sort
204
+ end
205
+
206
+ # A subject fails a gate when at least one of its mutants survived.
207
+ def failing_subjects(scored)
208
+ scored.select { |result| result.status == :survived }.map(&:subject).uniq.size
116
209
  end
117
210
 
118
211
  def over?(count, limit, executed)
@@ -12,7 +12,11 @@ module Saboteur
12
12
  class Install
13
13
  CONFIG = <<~RUBY
14
14
  # saboteur configuration for a Rails + RSpec app. CLI flags override this file.
15
+ # Only the decisions a first run needs are set here; every other key keeps
16
+ # its default. Who changes what, key by key:
17
+ # https://github.com/vamsipavanmahesh/saboteur/blob/main/docs/config-keys.md
15
18
  Saboteur.configure do |c|
19
+ # First-run decision: what to mutate.
16
20
  c.include = ["app/**/*.rb"]
17
21
  # Empty base classes: nothing useful to mutate.
18
22
  c.exclude = [
@@ -23,7 +27,8 @@ module Saboteur
23
27
  ]
24
28
  c.require = ["./config/environment"] # boots the app before specs load
25
29
  c.rails = true # reconnect ActiveRecord after fork, per-worker databases
26
- c.jobs = 4 # parallel workers; use 1 if your specs share external state
30
+ # First-run decision: parallelism; use 1 if your specs share external state.
31
+ c.jobs = 4
27
32
  end
28
33
  RUBY
29
34
 
@@ -62,18 +62,35 @@ module Saboteur
62
62
 
63
63
  # Runs every test, calling recorder.test_started(id, file) before and
64
64
  # recorder.test_finished(id) after each one. Used to build the coverage map.
65
- def run_all(_recorder)
65
+ # files, when given, limits the run to the tests defined in those spec
66
+ # files (one part of a partitioned build, #741); nil runs everything.
67
+ def run_all(_recorder, files: nil, seed: nil)
66
68
  raise NotImplementedError
67
69
  end
68
70
 
71
+ # How many tests run_all would run, over files when given (a part of
72
+ # a partitioned build, #741). Adapters that cannot split a build keep
73
+ # this refusal.
74
+ def example_count(files: nil)
75
+ raise NotImplementedError, "#{self.class} cannot count a part of the suite (files: #{files.inspect})"
76
+ end
77
+
69
78
  # Checks the unchanged suite in a fresh child. Existing adapters can
70
79
  # use their selected-test runner; adapters may override to retain full
71
80
  # suite ordering and report every executed test.
72
- def run_baseline(seed:)
81
+ def run_baseline(seed:, files: nil)
82
+ raise NotImplementedError, "#{self.class} cannot run a part of the suite (files: #{files.inspect})" if files
83
+
73
84
  Kernel.srand(seed)
74
85
  run(all_tests.map(&:id))
75
86
  end
76
87
 
88
+ # Whether the framework picks tests from a generated status file that no
89
+ # inputs snapshot covers.
90
+ def status_file?(_root)
91
+ false
92
+ end
93
+
77
94
  # Files whose content decides which tests exist (for the cache key).
78
95
  def test_files
79
96
  []
@@ -17,6 +17,7 @@ module Saboteur
17
17
  begin
18
18
  @runner.setup($stderr, load_output)
19
19
  rescue SystemExit => e
20
+ load_output.replay
20
21
  # A spec file calling exit at load makes RSpec print "While
21
22
  # loading <file> ..." (which names the file) and re-raise.
22
23
  # Uncaught, the parent exits with the spec's status: exit(0)
@@ -24,7 +25,15 @@ module Saboteur
24
25
  raise Error, "a spec file called exit while loading (status #{e.status})"
25
26
  end
26
27
  configuration.order = "rand:#{config.fetch('seed', 1)}"
27
- raise Error, load_failure_message(load_output.failed_files) if quit_while_loading?(world)
28
+ if quit_while_loading?(world)
29
+ # RSpec's load dump ends in the saboteur frames the load was
30
+ # triggered through and a "No examples found." from the reporter;
31
+ # both contradict the tool's own message. Print the suite's error
32
+ # without them (#608).
33
+ $stdout.write LoadErrorReport.filter(load_output.text, verbose: config.fetch("verbose", false))
34
+ raise Error, load_failure_message(load_output.failed_files, load_output.text)
35
+ end
36
+ load_output.replay
28
37
 
29
38
  refuse_unintended_focus
30
39
  end
@@ -80,7 +89,10 @@ module Saboteur
80
89
 
81
90
  # A missing rspec-core is the target's setup error, not a saboteur
82
91
  # defect (#482): refuse with the remedy instead of crashing into the
83
- # internal-error report. Only the framework's own LoadError converts -
92
+ # internal-error report. It is a usage error (#610): ConfigError maps
93
+ # to exit 2, and the remedy names docs/minitest.md rather than
94
+ # c.integration - rspec is the only integration, so pointing at it
95
+ # sent Minitest users down a dead end. Only the framework's own LoadError converts -
84
96
  # matched on the failed feature, not the message, so a missing
85
97
  # rspec/core_ext does not pose as rspec/core; a LoadError the app's
86
98
  # files raise keeps the crash route.
@@ -89,8 +101,9 @@ module Saboteur
89
101
  rescue LoadError => e
90
102
  raise unless e.path == "rspec/core"
91
103
 
92
- raise Error, "rspec is not in the bundle: the RSpec integration needs the rspec-core gem. " \
93
- "Add it to the bundle's test group (or point c.integration at the suite's framework)"
104
+ raise ConfigError, "rspec is not in the bundle: the RSpec integration needs the rspec-core gem. " \
105
+ "Add it to the bundle's test group. saboteur's built-in integration supports RSpec only - " \
106
+ "see docs/minitest.md for other suites"
94
107
  end
95
108
 
96
109
  def all_tests
@@ -99,6 +112,14 @@ module Saboteur
99
112
  .map { |example| TestCase.new(id: example.id, file: example.metadata[:rerun_file_path]) }
100
113
  end
101
114
 
115
+ # Whether RSpec picks examples from its generated status file
116
+ # (spec/examples.txt) for --only-failures or --next-failure. RSpec's own
117
+ # parser answers, so abbreviations, SPEC_OPTS, .rspec, .rspec-local,
118
+ # ~/.rspec, the XDG options file and --options FILE all count.
119
+ def status_file?(root)
120
+ from_directory(root) { ::RSpec::Core::ConfigurationOptions.new(rspec_args).options[:only_failures] ? true : false }
121
+ end
122
+
102
123
  def test_files
103
124
  world.example_groups.map { |group| File.expand_path(group.metadata[:file_path]) }.uniq
104
125
  end
@@ -118,17 +139,21 @@ module Saboteur
118
139
  failure_details: listener.failure_details)
119
140
  end
120
141
 
121
- def run_all(recorder)
122
- configuration.order = "rand:#{config.fetch('seed', 1)}"
142
+ def example_count(files: nil)
143
+ world.example_count(runnable_groups(files))
144
+ end
145
+
146
+ def run_all(recorder, files: nil, seed: nil)
147
+ configuration.order = "rand:#{seed || config.fetch('seed', 1)}"
123
148
  configuration.fail_fast = false
124
- execute(Listener.new(recorder), log_phase: true)
149
+ execute(Listener.new(recorder), log_phase: true, files: files)
125
150
  end
126
151
 
127
- def run_baseline(seed:)
152
+ def run_baseline(seed:, files: nil)
128
153
  configuration.order = "rand:#{seed}"
129
154
  configuration.fail_fast = false
130
155
  listener = Listener.new
131
- passed = execute(listener, log_phase: true)
156
+ passed = execute(listener, log_phase: true, files: files)
132
157
  RunResult.new(passed: passed && listener.failed.empty?, failed_ids: listener.failed,
133
158
  executed_ids: listener.executed, runtime: listener.runtime,
134
159
  failure_classes: listener.failure_classes, failure_messages: listener.failure_messages,
@@ -137,6 +162,19 @@ module Saboteur
137
162
 
138
163
  SHOWN_LOAD_FAILURES = 5
139
164
 
165
+ # Ruby's Coverage refuses a second start, so a suite whose
166
+ # spec_helper auto-starts SimpleCov/SingleCov collides with
167
+ # saboteur's own Coverage.start and the load quits with an opaque
168
+ # "No examples found" (#645, real run: audited @ dbf84326). Name the
169
+ # suspect and both guard styles. The gem's backtrace line is the
170
+ # fallback: the RuntimeError's text is Coverage's own, whoever
171
+ # started it second.
172
+ COVERAGE_DOUBLE_START = /coverage measurement is already setup/i
173
+ # A gem backtrace frame (path:line) or the start call itself - never a
174
+ # bare name, which a spec FILE name like simplecov_config_spec.rb
175
+ # would false-positive on (Pod 3 QA probe, #645).
176
+ COVERAGE_GEM_FRAME = %r{(?:single_cov|simplecov)[\w.\-/]*\.rb:\d+|\b(?:SimpleCov\.start|SingleCov\.setup)}i
177
+
140
178
  private
141
179
 
142
180
  # rspec-core 3.11 added rspec_is_quitting (SyntaxError and SystemExit
@@ -151,7 +189,7 @@ module Saboteur
151
189
  # printed are what a user needs. Name them and the way out: a repo
152
190
  # whose spec tree cannot load whole (an engine's dummy app, an
153
191
  # env-dependent integration spec) must narrow what loads (#378).
154
- def load_failure_message(failed_files)
192
+ def load_failure_message(failed_files, captured_text = nil)
155
193
  message = +"rspec quit while loading spec files"
156
194
  if failed_files.any?
157
195
  shown = failed_files.first(SHOWN_LOAD_FAILURES)
@@ -164,6 +202,18 @@ module Saboteur
164
202
  'c.rspec_args = ["--exclude-pattern", "spec/dummy/**/*_spec.rb"], or name the ' \
165
203
  "suite with --test. Rails engines with an embedded dummy app (spec/dummy) need " \
166
204
  'this; see README "Rails engines and dummy apps".'
205
+ hints = EnvironmentHints.all(config.root)
206
+ message << "\nSuspect: #{hints.join('; ')}." if hints.any?
207
+ if coverage_suspect?(captured_text)
208
+ message << "\nSuspect: your suite starts its own Coverage (saboteur starts one itself). " \
209
+ "Guard it - if ENV[\"COVERAGE\"], or SingleCov's conditional setup API - " \
210
+ "and see docs/recipes/README.md#simplecov."
211
+ end
212
+ message
213
+ end
214
+
215
+ def coverage_suspect?(captured_text)
216
+ captured_text&.match?(COVERAGE_DOUBLE_START) || captured_text&.match?(COVERAGE_GEM_FRAME) || false
167
217
  end
168
218
 
169
219
  def refuse_unintended_focus
@@ -224,10 +274,10 @@ module Saboteur
224
274
  # redirected to the phase's log: progress and failure detail land in
225
275
  # that log, so a failed phase can be diagnosed from it. Mutant runs
226
276
  # stay quiet.
227
- def execute(listener, log_phase: false)
277
+ def execute(listener, log_phase: false, files: nil)
228
278
  reporter = ::RSpec::Core::Reporter.new(configuration)
229
279
  reporter.register_listener(listener, *Listener::EVENTS)
230
- groups = world.ordered_example_groups.select { |group| group.descendant_filtered_examples.any? }
280
+ groups = runnable_groups(files)
231
281
  count = world.example_count(groups)
232
282
  if log_phase
233
283
  logger = PhaseLogger.new($stdout, total: count)
@@ -241,6 +291,31 @@ module Saboteur
241
291
  passed && !world.non_example_failure
242
292
  end
243
293
 
294
+ # The top-level groups with examples to run, in the world's order. With
295
+ # files, only the groups a spec file in that list defines (a part of a
296
+ # partitioned coverage build, #741). Both sides are compared as real
297
+ # paths. An empty list is refused and so is a listed file that defines
298
+ # no group: a part that silently runs nothing would leave its tests
299
+ # out of the map.
300
+ def runnable_groups(files)
301
+ groups = world.ordered_example_groups.select { |group| group.descendant_filtered_examples.any? }
302
+ return groups unless files
303
+ raise ArgumentError, "files must name at least one spec file" if files.empty?
304
+
305
+ wanted = files.to_set { |file| File.realpath(file) }
306
+ chosen = groups.select { |group| wanted.include?(real_path(group.metadata[:file_path])) }
307
+ missing = wanted - chosen.to_set { |group| real_path(group.metadata[:file_path]) }
308
+ raise ArgumentError, "no example group in: #{missing.sort.join(', ')}" unless missing.empty?
309
+
310
+ chosen
311
+ end
312
+
313
+ def real_path(path)
314
+ File.realpath(path)
315
+ rescue SystemCallError
316
+ File.expand_path(path)
317
+ end
318
+
244
319
  # Progress and failure detail for a phase log. Every 25 finished
245
320
  # examples is enough to see movement on a tail; failures print RSpec's
246
321
  # full formatted detail, and the run ends with RSpec's totals line.
@@ -293,26 +368,78 @@ module Saboteur
293
368
  @captured = +""
294
369
  end
295
370
 
371
+ # Buffer only: the caller decides after setup whether the captured
372
+ # text goes out as it is (a clean load), filtered (a load quit,
373
+ # #608) or verbatim on a SystemExit. Passing through during setup
374
+ # would print what a later filter needs to drop.
296
375
  def write(text)
297
- @captured << text
298
- super
376
+ @replayed ? __getobj__.write(text) : @captured << text
377
+ self
299
378
  end
300
379
 
301
380
  def print(*args)
302
- @captured << args.join
303
- super
381
+ write(args.join)
304
382
  end
305
383
 
306
384
  def puts(*args)
307
- @captured << args.flatten.join("\n") << "\n"
308
- super
385
+ write("#{args.flatten.join("\n")}\n")
386
+ end
387
+
388
+ def replay
389
+ __getobj__.write(@captured)
390
+ # RSpec retains this stream for later suite-hook errors.
391
+ @replayed = true
309
392
  end
310
393
 
394
+ def text = @captured
395
+
311
396
  def failed_files
312
397
  LOAD_FAILURE.flat_map { |pattern| @captured.scan(pattern) }.flatten
313
398
  end
314
399
  end
315
400
 
401
+ # The load dump RSpec prints when a spec file raises at load, trimmed
402
+ # to what the user acts on: the first error with the suite's own
403
+ # frames. The saboteur frames the load was triggered through and the
404
+ # reporter's "No examples found." (nothing ran because nothing loaded)
405
+ # read as saboteur crashing and contradict the tool's message; both
406
+ # stay under --verbose (#608).
407
+ class LoadErrorReport
408
+ HEADER = /^An error occurred while loading /
409
+ # The running tool's install root (a checkout or an unpacked gem),
410
+ # with separator. Only frames under it are saboteur's: the same
411
+ # lib/saboteur/ layout under ANOTHER root is the target's vendored
412
+ # copy or a dogfooded checkout, and those frames are the suite's
413
+ # (#608).
414
+ TOOL_ROOT = "#{File.expand_path('../../..', __dir__)}/".freeze
415
+ private_constant :HEADER, :TOOL_ROOT
416
+
417
+ def self.filter(text, verbose: false)
418
+ return text if verbose
419
+
420
+ lines = text.lines.reject { |line| line.strip == "No examples found." || saboteur_frame?(line) }
421
+ heads = lines.each_index.select { |index| lines[index].match?(HEADER) }
422
+ return lines.join if heads.size <= 1
423
+
424
+ kept = lines[0...heads[1]]
425
+ kept.pop while kept.last&.strip == ""
426
+ omitted = heads.size - 1
427
+ kept << "(#{omitted} more load error#{'s' if omitted > 1} omitted; rerun with --verbose for the full list)\n"
428
+ kept.join
429
+ end
430
+
431
+ # A backtrace frame inside the tool's install root. Relative frames
432
+ # (the suite's own "./spec/...") are never the tool's.
433
+ def self.saboteur_frame?(line)
434
+ line = line.strip
435
+ return false unless line.start_with?("# ")
436
+
437
+ path = line.sub(/\A#\s+/, "").sub(/:\d+:in .*\z/, "")
438
+ path.start_with?("/") && File.expand_path(path).start_with?(TOOL_ROOT)
439
+ end
440
+ private_class_method :saboteur_frame?
441
+ end
442
+
316
443
  class Listener
317
444
  EVENTS = %i[example_started example_finished example_failed].freeze
318
445
 
@@ -0,0 +1,81 @@
1
+ # SPDX-License-Identifier: MIT
2
+ # frozen_string_literal: true
3
+
4
+ module Saboteur
5
+ # Records which lines ran, per file, with a TracePoint on :line (#725). A
6
+ # per-test recorder can drain it between tests instead of calling
7
+ # Coverage.result, whose read walks every file of the process. Pass `files`
8
+ # (canonical paths) to keep only those; nil keeps all. Not wired into the
9
+ # recorder yet: it matches Coverage's lines mode on saboteur's own suite
10
+ # line for line, but has no branches, so the recorder's branch data still
11
+ # needs its own design.
12
+ class LineHits
13
+ def initialize(files = nil, canonical: ->(path) { path })
14
+ @wanted = files && Hash.new { |hash, path| hash[path] = wanted?(files, canonical, path) }
15
+ @hits = new_hits
16
+ @trace = TracePoint.new(:line, &method(:record))
17
+ end
18
+
19
+ def enable
20
+ @trace.enable
21
+ self
22
+ end
23
+
24
+ def disable
25
+ @trace.disable
26
+ self
27
+ end
28
+
29
+ # Stops recording without touching the TracePoint: toggling it costs
30
+ # milliseconds once an app is loaded, so a recorder pauses around its own
31
+ # work instead.
32
+ def pause
33
+ @paused = true
34
+ self
35
+ end
36
+
37
+ def resume
38
+ @paused = false
39
+ self
40
+ end
41
+
42
+ # file => Set of line numbers since the last drain. A failure inside the
43
+ # tracer is raised here: raising inside the TracePoint would unwind the
44
+ # program being traced.
45
+ def drain(clear: true)
46
+ failure = @failure
47
+ @failure = nil
48
+ raise failure if failure
49
+
50
+ hits = @hits
51
+ @hits = new_hits if clear
52
+ hits
53
+ end
54
+
55
+ private
56
+
57
+ def new_hits
58
+ Hash.new { |hash, file| hash[file] = Set.new }
59
+ end
60
+
61
+ # A path the canonicalizer refuses is not a wanted file: a NUL byte in an
62
+ # eval'd file name raises ArgumentError, and a name that is not a file
63
+ # ("(eval at ...)") raises ENOENT from a realpath canonicalizer.
64
+ def wanted?(files, canonical, path)
65
+ files.include?(canonical.call(path))
66
+ rescue ArgumentError, SystemCallError, TypeError
67
+ false
68
+ end
69
+
70
+ def record(event)
71
+ return if @paused
72
+
73
+ path = event.path
74
+ return if @wanted && !@wanted[path]
75
+
76
+ @hits[path] << event.lineno
77
+ rescue StandardError => e
78
+ @failure ||= e
79
+ end
80
+ end
81
+ end