saboteur 0.8.1 → 0.9.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (115) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +100 -0
  3. data/README.md +10 -0
  4. data/exe/saboteur +6 -2
  5. data/lib/saboteur/attribute_finder.rb +3 -0
  6. data/lib/saboteur/baseline/run_evidence.rb +2 -1
  7. data/lib/saboteur/cli/agent_provenance.rb +59 -0
  8. data/lib/saboteur/cli/dirty_tree_notice.rb +28 -0
  9. data/lib/saboteur/cli/doctor_command.rb +41 -0
  10. data/lib/saboteur/cli/error_cause.rb +40 -0
  11. data/lib/saboteur/cli/execution_options.rb +46 -0
  12. data/lib/saboteur/cli/exit_status.rb +12 -14
  13. data/lib/saboteur/cli/explicit_file_options.rb +21 -0
  14. data/lib/saboteur/cli/explicit_sources.rb +46 -0
  15. data/lib/saboteur/cli/failure_output.rb +35 -0
  16. data/lib/saboteur/cli/help_command.rb +23 -0
  17. data/lib/saboteur/cli/matrix_option.rb +17 -0
  18. data/lib/saboteur/cli/output_options.rb +27 -0
  19. data/lib/saboteur/cli/preview_command.rb +26 -0
  20. data/lib/saboteur/cli/run_budget_options.rb +3 -0
  21. data/lib/saboteur/cli/run_command.rb +33 -0
  22. data/lib/saboteur/cli/run_output.rb +38 -0
  23. data/lib/saboteur/cli/run_reporting.rb +6 -5
  24. data/lib/saboteur/cli/run_validation.rb +54 -0
  25. data/lib/saboteur/cli.rb +29 -307
  26. data/lib/saboteur/config.rb +72 -6
  27. data/lib/saboteur/constant_subject_finder.rb +3 -0
  28. data/lib/saboteur/custom_map.rb +20 -12
  29. data/lib/saboteur/deep_source.rb +49 -0
  30. data/lib/saboteur/dirty_tree_warning.rb +1 -1
  31. data/lib/saboteur/doctor.rb +6 -0
  32. data/lib/saboteur/engine/baseline_guard.rb +50 -13
  33. data/lib/saboteur/engine/baseline_parts.rb +51 -0
  34. data/lib/saboteur/engine/boot.rb +4 -0
  35. data/lib/saboteur/engine/candidate_selection.rb +15 -5
  36. data/lib/saboteur/engine/candidates.rb +4 -1
  37. data/lib/saboteur/engine/coverage_inputs_snapshot.rb +146 -0
  38. data/lib/saboteur/engine/coverage_miss_note.rb +86 -0
  39. data/lib/saboteur/engine/coverage_partial.rb +48 -0
  40. data/lib/saboteur/engine/coverage_parts.rb +50 -0
  41. data/lib/saboteur/engine/coverage_shadow.rb +92 -0
  42. data/lib/saboteur/engine/coverage_store.rb +275 -22
  43. data/lib/saboteur/engine/diagnostics.rb +23 -6
  44. data/lib/saboteur/engine/distribution.rb +2 -2
  45. data/lib/saboteur/engine/gated_run.rb +1 -0
  46. data/lib/saboteur/engine/load_coverage.rb +54 -8
  47. data/lib/saboteur/engine/partial_invalidation.rb +115 -0
  48. data/lib/saboteur/engine/partial_merge.rb +40 -0
  49. data/lib/saboteur/engine/pipelines.rb +7 -7
  50. data/lib/saboteur/engine/reference_specs.rb +71 -0
  51. data/lib/saboteur/engine/result_cache.rb +68 -5
  52. data/lib/saboteur/engine/resume_manifest.rb +110 -0
  53. data/lib/saboteur/engine/run_budget.rb +5 -5
  54. data/lib/saboteur/engine/run_fingerprint.rb +2 -1
  55. data/lib/saboteur/engine/sandbox_policy.rb +5 -2
  56. data/lib/saboteur/engine/scheduler.rb +1 -1
  57. data/lib/saboteur/engine/selection_announcement.rb +1 -0
  58. data/lib/saboteur/engine/shadow_comparison.rb +121 -0
  59. data/lib/saboteur/engine/shadow_run.rb +38 -0
  60. data/lib/saboteur/engine/subject_selection.rb +33 -37
  61. data/lib/saboteur/engine/suite_partition.rb +22 -0
  62. data/lib/saboteur/engine/test_guess.rb +82 -18
  63. data/lib/saboteur/engine/unchanged_syntax.rb +10 -2
  64. data/lib/saboteur/engine.rb +41 -9
  65. data/lib/saboteur/environment_hints.rb +55 -0
  66. data/lib/saboteur/execution/bundle_gemfile_pin.rb +72 -0
  67. data/lib/saboteur/execution/external_runner.rb +105 -1
  68. data/lib/saboteur/execution/gemfile_path_sources.rb +166 -0
  69. data/lib/saboteur/execution/sandbox.rb +30 -17
  70. data/lib/saboteur/execution/supervisor.rb +40 -7
  71. data/lib/saboteur/execution/supervisor_batch.rb +105 -0
  72. data/lib/saboteur/execution/worker_pool.rb +48 -4
  73. data/lib/saboteur/frameworks/rails.rb +2 -5
  74. data/lib/saboteur/gates.rb +121 -28
  75. data/lib/saboteur/install.rb +6 -1
  76. data/lib/saboteur/integration/base.rb +19 -2
  77. data/lib/saboteur/integration/rspec.rb +46 -7
  78. data/lib/saboteur/line_hits.rb +81 -0
  79. data/lib/saboteur/method_calls.rb +91 -0
  80. data/lib/saboteur/mutation_source/builtin.rb +32 -22
  81. data/lib/saboteur/mutation_source/condition_negation.rb +3 -12
  82. data/lib/saboteur/mutation_source/jump_value.rb +59 -0
  83. data/lib/saboteur/mutation_source/literal_replacement.rb +6 -23
  84. data/lib/saboteur/mutation_source/operator_examples.rb +5 -0
  85. data/lib/saboteur/mutation_source/operator_registry.rb +5 -0
  86. data/lib/saboteur/mutation_source/operator_visit.rb +7 -1
  87. data/lib/saboteur/mutation_source/rescue_removal.rb +69 -0
  88. data/lib/saboteur/mutation_source/unary_removal.rb +47 -0
  89. data/lib/saboteur/mutation_source/when_removal.rb +51 -0
  90. data/lib/saboteur/preview.rb +9 -1
  91. data/lib/saboteur/refinement_activations.rb +22 -10
  92. data/lib/saboteur/reporter/agent.rb +2 -2
  93. data/lib/saboteur/reporter/agent_incomplete.rb +11 -0
  94. data/lib/saboteur/reporter/agent_summary.rb +16 -10
  95. data/lib/saboteur/reporter/review_comments.rb +151 -0
  96. data/lib/saboteur/reporter/safe_out.rb +32 -0
  97. data/lib/saboteur/reporter/stryker.rb +8 -6
  98. data/lib/saboteur/reporter/survivor_classification.rb +41 -3
  99. data/lib/saboteur/reporter/survivor_hints.rb +5 -0
  100. data/lib/saboteur/reporter/terminal.rb +29 -4
  101. data/lib/saboteur/result.rb +14 -4
  102. data/lib/saboteur/selection/coverage_map.rb +136 -10
  103. data/lib/saboteur/selection/external_test_map.rb +4 -0
  104. data/lib/saboteur/selection/git_diff.rb +24 -2
  105. data/lib/saboteur/selection/layout_only_lines.rb +45 -1
  106. data/lib/saboteur/selection/recorder.rb +52 -22
  107. data/lib/saboteur/selection/trace_recorder.rb +83 -0
  108. data/lib/saboteur/selection/untracked_changes.rb +1 -2
  109. data/lib/saboteur/subject.rb +16 -0
  110. data/lib/saboteur/subject_finder.rb +91 -6
  111. data/lib/saboteur/text.rb +31 -0
  112. data/lib/saboteur/version.rb +1 -1
  113. data/lib/saboteur/working_tree.rb +7 -2
  114. data/lib/saboteur.rb +8 -0
  115. metadata +44 -2
@@ -91,15 +91,25 @@ module Saboteur
91
91
  end
92
92
 
93
93
  def distribute(items, order, &work)
94
+ @sweep_error = nil
95
+ # Explicit instead of $ERROR_INFO: the sweep error is raised only
96
+ # when the run itself completed without an error to propagate.
97
+ raised = true
94
98
  results = Array.new(items.size)
95
99
  queue = order.dup
96
100
  workers = []
97
101
  [@jobs, items.size].min.times { |number| workers << start(number, items, workers, &work) }
98
102
  workers.each { |worker| assign(worker, queue) }
99
103
  collect(workers, queue, results, items, &work) until workers.empty?
104
+ raised = false
100
105
  results
101
106
  ensure
102
107
  workers&.each { |worker| stop(worker) }
108
+ # Containment first, honesty after (#762): every worker's stop ran
109
+ # above, so a blind-table sweep no longer aborts the registry; but a
110
+ # skipped orphan sweep must not read as a clean run either. The
111
+ # run's own error wins when one is already propagating.
112
+ raise @sweep_error if @sweep_error && !raised
103
113
  end
104
114
 
105
115
  # Waits up to a second for results, then checks for workers that died
@@ -174,11 +184,18 @@ module Saboteur
174
184
 
175
185
  # Reads what a dead worker left in its pipe: a last result, or why it
176
186
  # crashed.
177
- def drain(worker, results)
187
+ # report: false while stopping - the run is unwinding, and re-invoking
188
+ # the arrival callback from cleanup would let its raise abort the
189
+ # remaining stops and leak their live entries (the CI flake).
190
+ def drain(worker, results, report: true)
178
191
  pending = worker.results.read_nonblock(1 << 20, exception: false)
179
192
  return unless pending.is_a?(String)
180
193
 
181
- pending.each_line { |line| take_and_report(worker, line, results) if line.end_with?("\n") }
194
+ pending.each_line do |line|
195
+ next unless line.end_with?("\n")
196
+
197
+ report ? take_and_report(worker, line, results) : take(worker, line, results)
198
+ end
182
199
  rescue IOError
183
200
  nil
184
201
  end
@@ -331,6 +348,33 @@ module Saboteur
331
348
  { "error" => "#{e.class}: #{e.message}" }
332
349
  end
333
350
 
351
+ # A blind process table (both backends down, as in #762) loses the
352
+ # table-driven orphan sweep, but it must not skip ending the worker
353
+ # itself - wait2 has no timeout, so kill_all was the only thing
354
+ # stopping a wedged one. Kill its group and pid directly, remember the
355
+ # error for distribute to raise once every worker is stopped, and never
356
+ # let the sweep abort the remaining stops.
357
+ def sweep(tree, worker)
358
+ tree&.kill_all
359
+ rescue SystemCallError, Fiddle::DLError => e
360
+ @sweep_error ||= e
361
+ # The group kill also ends orphaned grandchildren and is always
362
+ # safe: a process group id cannot be recycled while any member
363
+ # lives. The direct kill is only for a wedged worker: once reaped,
364
+ # its pid can already belong to an unrelated process.
365
+ kill_hard(worker.pid, direct: !reaped?(worker))
366
+ end
367
+
368
+ def kill_hard(pid, direct:)
369
+ targets = [-pid]
370
+ targets << pid if direct
371
+ targets.each do |target|
372
+ Process.kill("KILL", target)
373
+ rescue Errno::ESRCH, Errno::EPERM
374
+ nil
375
+ end
376
+ end
377
+
334
378
  # Closing the command pipe ends the worker's loop; anything it left
335
379
  # running is killed with its tree.
336
380
  def stop(worker)
@@ -338,9 +382,9 @@ module Saboteur
338
382
  tree = Supervisor.live.delete(worker.pid)
339
383
  deadline = Process.clock_gettime(Process::CLOCK_MONOTONIC) + 5
340
384
  sleep 0.01 until reaped?(worker) || Process.clock_gettime(Process::CLOCK_MONOTONIC) > deadline
341
- tree&.kill_all
385
+ sweep(tree, worker)
342
386
  worker.status ||= Process.wait2(worker.pid).last unless reaped?(worker)
343
- drain(worker, {})
387
+ drain(worker, {}, report: false)
344
388
  worker.results.close unless worker.results.closed?
345
389
  rescue Errno::ECHILD
346
390
  nil
@@ -18,9 +18,6 @@ module Saboteur
18
18
  # forking, and drops database connections in the parent so each child opens
19
19
  # its own. Prints host names only, never credentials.
20
20
  class Rails
21
- # How many blocking session names the template-copy error lists.
22
- MAX_BLOCKER_NAMES = 5
23
-
24
21
  # sqlite3-ruby's Backup#step returns raw SQLite status codes instead
25
22
  # of raising, and finish discards its own: without an explicit check
26
23
  # a partial destination would pass as a copy.
@@ -362,9 +359,9 @@ module Saboteur
362
359
  SQL
363
360
  return if blockers.empty?
364
361
 
365
- names = blockers.map { |app| app.to_s.empty? ? "unnamed" : app }.first(MAX_BLOCKER_NAMES)
362
+ names = Text.name_list(blockers.map { |app| app.to_s.empty? ? "unnamed" : app })
366
363
  raise Error, "cannot copy the test database #{database}: #{blockers.size} other session(s) connected " \
367
- "(#{names.join(', ')})#{', ...' if blockers.size > names.size}. " \
364
+ "(#{names}). " \
368
365
  "Stop other test runs, consoles or Spring, then run saboteur again."
369
366
  end
370
367
 
@@ -60,59 +60,152 @@ module Saboteur
60
60
  !@limits.fetch("max_errors").nil?
61
61
  end
62
62
 
63
- # Prints each failed gate to err and remembers the first one's cause for the JSON report.
64
- def failed?(report, err)
65
- vacuous_gates(report).each { |note| err.puts("saboteur: #{note}") }
63
+ # Prints the gate notes and each failed gate to err. The first
64
+ # violation's cause is recorded for the JSON report unless the caller
65
+ # already has a cause that must win (record_cause: false).
66
+ def failed?(report, err, record_cause: true)
67
+ overlap_notes(report).each { |note| err.puts("saboteur: #{note}") }
66
68
  found = violations(report)
67
69
  found.each { |violation| err.puts("saboteur: #{violation.message}") }
68
- @cause = found.first&.cause
69
- !@cause.nil?
70
+ @cause = found.first&.cause if record_cause
71
+ found.any?
70
72
  end
71
73
 
72
74
  def violations(report)
73
75
  SETTINGS.filter_map do |key, (status, cause, flag)|
74
76
  limit = @limits.fetch(key)
77
+ next if limit.nil?
78
+
75
79
  count = report.counts.fetch(status)
76
- next if limit.nil? || !over?(count, limit, report.executed.size)
80
+ next unless over?(count, limit, report.executed.size)
77
81
 
78
82
  Violation.new(cause: cause, message: "#{count} #{status} mutants, above #{flag} #{limit}")
79
83
  end + score_gate_violations(report)
80
84
  end
81
85
 
86
+ # A machine-readable summary per score gate for the JSON report: the
87
+ # floor, the mutants scoring under it, its score and pass/fail state,
88
+ # the failing subjects and the matched files. nil without score gates.
89
+ def score_gate_breakdowns(report)
90
+ return if @score_gates.empty?
91
+
92
+ assignment = gate_assignment(report)
93
+ @score_gates.each_with_index.map do |gate, index|
94
+ scored = assignment.fetch(index)
95
+ verdict = gate_verdict(gate, scored)
96
+ { pattern: gate["pattern"], minScore: gate["min_score"], mutants: scored.size, tested: verdict[:tested],
97
+ score: verdict[:score]&.round(1), passed: verdict[:passed],
98
+ failingSubjects: failing_subjects(scored), files: gate_files(scored, report) }
99
+ end
100
+ end
101
+
82
102
  private
83
103
 
84
- # Per-path gates (#473): each gate scores the results whose subject file
85
- # (root-relative) matches its pattern, with the run's own Score formula,
86
- # and fails under its floor.
104
+ # Per-path gates (#473): each gate scores the results assigned to it by
105
+ # gate_matching, with the run's own Score formula, and fails under its
106
+ # floor. The failure line names the matched files (at most three, then
107
+ # a count) and the failing subject count.
87
108
  def score_gate_violations(report)
88
- @score_gates.filter_map do |gate|
89
- scored = score_gate_subset(gate, report)
90
- breakdown = Score.of(scored)
91
- next if breakdown.tested.zero?
109
+ assignment = gate_assignment(report)
110
+ @score_gates.each_with_index.filter_map do |gate, index|
111
+ scored = assignment.fetch(index)
112
+ verdict = gate_verdict(gate, scored)
113
+ next if verdict[:passed]
114
+
115
+ files = gate_files(scored, report)
116
+ listed = files.first(3).join(", ")
117
+ listed += ", +#{files.size - 3} more" if files.size > 3
118
+ message = "score gate #{gate['pattern']}: score #{format('%.1f', verdict[:score])} "
119
+ message += "below #{gate['min_score']} (#{scored.size} mutant(s) under it, "
120
+ message += "#{failing_subjects(scored)} failing subject(s), #{files.size} matched file(s): #{listed})"
121
+ Violation.new(cause: "scoreGate", message:)
122
+ end
123
+ end
92
124
 
93
- score = breakdown.value * 100
94
- next if score >= gate["min_score"]
125
+ # A gate that lost subjects to a more specific gate says so once per
126
+ # subject, even on a passing run. Only a gate that matched nothing at
127
+ # all is vacuous; a fully shadowed gate reads as overlapped instead.
128
+ def overlap_notes(report)
129
+ matched, assignment = gate_matching(report)
130
+ @score_gates.each_with_index.filter_map do |gate, index|
131
+ lost = (matched[index] - assignment.fetch(index)).map(&:subject).uniq.size
132
+ if lost.positive?
133
+ next "score gate #{gate['pattern']}: #{lost} subject(s) also match a more specific gate and score there"
134
+ end
95
135
 
96
- Violation.new(cause: "scoreGate",
97
- message: "score gate #{gate['pattern']}: score #{format('%.1f', score)} below " \
98
- "#{gate['min_score']} (#{scored.size} mutants under it)")
136
+ next if Score.of(assignment.fetch(index)).tested.positive?
137
+
138
+ # A gate that matched mutants but tested none of them (errors,
139
+ # timeouts, skips) passes just as vacuously as one that matched
140
+ # nothing; say so either way, unless a more specific gate shadows it.
141
+ if matched[index].empty?
142
+ "score gate #{gate['pattern']}: nothing executed under it, passing vacuously"
143
+ else
144
+ "score gate #{gate['pattern']}: no mutant(s) tested under it, passing vacuously"
145
+ end
99
146
  end
100
147
  end
101
148
 
102
- # A gate with nothing executed under it passes vacuously and says so.
103
- def vacuous_gates(report)
104
- @score_gates.filter_map do |gate|
105
- breakdown = Score.of(score_gate_subset(gate, report))
106
- "score gate #{gate['pattern']}: nothing executed under it, passing vacuously" if breakdown.tested.zero?
149
+ # One verdict per gate, shared by the console violation and the JSON
150
+ # breakdown: both judge the unrounded score, so a 79.96 against an 80
151
+ # floor fails in both places. The one-decimal score in the JSON is
152
+ # display-only. A gate with nothing tested passes vacuously, score nil.
153
+ def gate_verdict(gate, scored)
154
+ breakdown = Score.of(scored)
155
+ return { tested: 0, passed: true } if breakdown.tested.zero?
156
+
157
+ raw = breakdown.value * 100
158
+ { tested: breakdown.tested, score: raw, passed: raw >= gate["min_score"] }
159
+ end
160
+
161
+ # Each result scores under the single most specific matching gate: more
162
+ # literal (non-glob) pattern characters win, ties keep the earlier config
163
+ # entry (min_by returns the first minimum), so overlapping floors never
164
+ # double-count a mutant. Returns the per-gate matched and assigned lists.
165
+ def gate_matching(report)
166
+ matched = @score_gates.map { [] }
167
+ assignment = @score_gates.map { [] }
168
+ if @score_gates.any?
169
+ each_relative_result(report).each do |relative, result|
170
+ candidates = @score_gates.each_with_index.select { |gate, _index| gate_matches?(gate, relative) }
171
+ candidates.each { |candidate| matched[candidate[1]] << result }
172
+ winner = candidates.min_by { |gate, _index| -specificity(gate["pattern"]) }
173
+ assignment[winner[1]] << result if winner
174
+ end
107
175
  end
176
+ [matched, assignment]
108
177
  end
109
178
 
110
- def score_gate_subset(gate, report)
179
+ def gate_assignment(report)
180
+ gate_matching(report)[1]
181
+ end
182
+
183
+ # [relative path, result] pairs; the caller consumes the array, so the
184
+ # build cannot be swapped for a side-effect-only walk.
185
+ def each_relative_result(report)
111
186
  prefix = File.join(File.realpath(report.root), "")
112
- report.results.select do |result|
113
- relative = result.subject.file.delete_prefix(prefix)
114
- File.fnmatch?(gate["pattern"], relative, File::FNM_PATHNAME | File::FNM_EXTGLOB)
115
- end
187
+ report.results.map { |result| [result.subject.file.delete_prefix(prefix), result] }
188
+ end
189
+
190
+ def gate_matches?(gate, relative)
191
+ File.fnmatch?(gate["pattern"], relative, File::FNM_PATHNAME | File::FNM_EXTGLOB)
192
+ end
193
+
194
+ # Literal characters pin a gate to its files; glob metacharacters do
195
+ # not, and a {a,b} brace group counts once rather than per alternative,
196
+ # so the exact lib/saboteur/gates.rb outranks lib/saboteur/{gates,config}.rb.
197
+ def specificity(pattern)
198
+ pattern.gsub(/\{[^{}]*\}/, "x").gsub(/[*?\[\]\\]/, "").length
199
+ end
200
+
201
+ def gate_files(scored, report)
202
+ prefix = File.join(File.realpath(report.root), "")
203
+ scored.map { |result| result.subject.file.delete_prefix(prefix) }.uniq.sort
204
+ end
205
+
206
+ # A subject fails a gate when at least one of its mutants survived.
207
+ def failing_subjects(scored)
208
+ scored.select { |result| result.status == :survived }.map(&:subject).uniq.size
116
209
  end
117
210
 
118
211
  def over?(count, limit, executed)
@@ -12,7 +12,11 @@ module Saboteur
12
12
  class Install
13
13
  CONFIG = <<~RUBY
14
14
  # saboteur configuration for a Rails + RSpec app. CLI flags override this file.
15
+ # Only the decisions a first run needs are set here; every other key keeps
16
+ # its default. Who changes what, key by key:
17
+ # https://github.com/vamsipavanmahesh/saboteur/blob/main/docs/config-keys.md
15
18
  Saboteur.configure do |c|
19
+ # First-run decision: what to mutate.
16
20
  c.include = ["app/**/*.rb"]
17
21
  # Empty base classes: nothing useful to mutate.
18
22
  c.exclude = [
@@ -23,7 +27,8 @@ module Saboteur
23
27
  ]
24
28
  c.require = ["./config/environment"] # boots the app before specs load
25
29
  c.rails = true # reconnect ActiveRecord after fork, per-worker databases
26
- c.jobs = 4 # parallel workers; use 1 if your specs share external state
30
+ # First-run decision: parallelism; use 1 if your specs share external state.
31
+ c.jobs = 4
27
32
  end
28
33
  RUBY
29
34
 
@@ -62,18 +62,35 @@ module Saboteur
62
62
 
63
63
  # Runs every test, calling recorder.test_started(id, file) before and
64
64
  # recorder.test_finished(id) after each one. Used to build the coverage map.
65
- def run_all(_recorder)
65
+ # files, when given, limits the run to the tests defined in those spec
66
+ # files (one part of a partitioned build, #741); nil runs everything.
67
+ def run_all(_recorder, files: nil, seed: nil)
66
68
  raise NotImplementedError
67
69
  end
68
70
 
71
+ # How many tests run_all would run, over files when given (a part of
72
+ # a partitioned build, #741). Adapters that cannot split a build keep
73
+ # this refusal.
74
+ def example_count(files: nil)
75
+ raise NotImplementedError, "#{self.class} cannot count a part of the suite (files: #{files.inspect})"
76
+ end
77
+
69
78
  # Checks the unchanged suite in a fresh child. Existing adapters can
70
79
  # use their selected-test runner; adapters may override to retain full
71
80
  # suite ordering and report every executed test.
72
- def run_baseline(seed:)
81
+ def run_baseline(seed:, files: nil)
82
+ raise NotImplementedError, "#{self.class} cannot run a part of the suite (files: #{files.inspect})" if files
83
+
73
84
  Kernel.srand(seed)
74
85
  run(all_tests.map(&:id))
75
86
  end
76
87
 
88
+ # Whether the framework picks tests from a generated status file that no
89
+ # inputs snapshot covers.
90
+ def status_file?(_root)
91
+ false
92
+ end
93
+
77
94
  # Files whose content decides which tests exist (for the cache key).
78
95
  def test_files
79
96
  []
@@ -112,6 +112,14 @@ module Saboteur
112
112
  .map { |example| TestCase.new(id: example.id, file: example.metadata[:rerun_file_path]) }
113
113
  end
114
114
 
115
+ # Whether RSpec picks examples from its generated status file
116
+ # (spec/examples.txt) for --only-failures or --next-failure. RSpec's own
117
+ # parser answers, so abbreviations, SPEC_OPTS, .rspec, .rspec-local,
118
+ # ~/.rspec, the XDG options file and --options FILE all count.
119
+ def status_file?(root)
120
+ from_directory(root) { ::RSpec::Core::ConfigurationOptions.new(rspec_args).options[:only_failures] ? true : false }
121
+ end
122
+
115
123
  def test_files
116
124
  world.example_groups.map { |group| File.expand_path(group.metadata[:file_path]) }.uniq
117
125
  end
@@ -131,17 +139,21 @@ module Saboteur
131
139
  failure_details: listener.failure_details)
132
140
  end
133
141
 
134
- def run_all(recorder)
135
- configuration.order = "rand:#{config.fetch('seed', 1)}"
142
+ def example_count(files: nil)
143
+ world.example_count(runnable_groups(files))
144
+ end
145
+
146
+ def run_all(recorder, files: nil, seed: nil)
147
+ configuration.order = "rand:#{seed || config.fetch('seed', 1)}"
136
148
  configuration.fail_fast = false
137
- execute(Listener.new(recorder), log_phase: true)
149
+ execute(Listener.new(recorder), log_phase: true, files: files)
138
150
  end
139
151
 
140
- def run_baseline(seed:)
152
+ def run_baseline(seed:, files: nil)
141
153
  configuration.order = "rand:#{seed}"
142
154
  configuration.fail_fast = false
143
155
  listener = Listener.new
144
- passed = execute(listener, log_phase: true)
156
+ passed = execute(listener, log_phase: true, files: files)
145
157
  RunResult.new(passed: passed && listener.failed.empty?, failed_ids: listener.failed,
146
158
  executed_ids: listener.executed, runtime: listener.runtime,
147
159
  failure_classes: listener.failure_classes, failure_messages: listener.failure_messages,
@@ -190,6 +202,8 @@ module Saboteur
190
202
  'c.rspec_args = ["--exclude-pattern", "spec/dummy/**/*_spec.rb"], or name the ' \
191
203
  "suite with --test. Rails engines with an embedded dummy app (spec/dummy) need " \
192
204
  'this; see README "Rails engines and dummy apps".'
205
+ hints = EnvironmentHints.all(config.root)
206
+ message << "\nSuspect: #{hints.join('; ')}." if hints.any?
193
207
  if coverage_suspect?(captured_text)
194
208
  message << "\nSuspect: your suite starts its own Coverage (saboteur starts one itself). " \
195
209
  "Guard it - if ENV[\"COVERAGE\"], or SingleCov's conditional setup API - " \
@@ -260,10 +274,10 @@ module Saboteur
260
274
  # redirected to the phase's log: progress and failure detail land in
261
275
  # that log, so a failed phase can be diagnosed from it. Mutant runs
262
276
  # stay quiet.
263
- def execute(listener, log_phase: false)
277
+ def execute(listener, log_phase: false, files: nil)
264
278
  reporter = ::RSpec::Core::Reporter.new(configuration)
265
279
  reporter.register_listener(listener, *Listener::EVENTS)
266
- groups = world.ordered_example_groups.select { |group| group.descendant_filtered_examples.any? }
280
+ groups = runnable_groups(files)
267
281
  count = world.example_count(groups)
268
282
  if log_phase
269
283
  logger = PhaseLogger.new($stdout, total: count)
@@ -277,6 +291,31 @@ module Saboteur
277
291
  passed && !world.non_example_failure
278
292
  end
279
293
 
294
+ # The top-level groups with examples to run, in the world's order. With
295
+ # files, only the groups a spec file in that list defines (a part of a
296
+ # partitioned coverage build, #741). Both sides are compared as real
297
+ # paths. An empty list is refused and so is a listed file that defines
298
+ # no group: a part that silently runs nothing would leave its tests
299
+ # out of the map.
300
+ def runnable_groups(files)
301
+ groups = world.ordered_example_groups.select { |group| group.descendant_filtered_examples.any? }
302
+ return groups unless files
303
+ raise ArgumentError, "files must name at least one spec file" if files.empty?
304
+
305
+ wanted = files.to_set { |file| File.realpath(file) }
306
+ chosen = groups.select { |group| wanted.include?(real_path(group.metadata[:file_path])) }
307
+ missing = wanted - chosen.to_set { |group| real_path(group.metadata[:file_path]) }
308
+ raise ArgumentError, "no example group in: #{missing.sort.join(', ')}" unless missing.empty?
309
+
310
+ chosen
311
+ end
312
+
313
+ def real_path(path)
314
+ File.realpath(path)
315
+ rescue SystemCallError
316
+ File.expand_path(path)
317
+ end
318
+
280
319
  # Progress and failure detail for a phase log. Every 25 finished
281
320
  # examples is enough to see movement on a tail; failures print RSpec's
282
321
  # full formatted detail, and the run ends with RSpec's totals line.
@@ -0,0 +1,81 @@
1
+ # SPDX-License-Identifier: MIT
2
+ # frozen_string_literal: true
3
+
4
+ module Saboteur
5
+ # Records which lines ran, per file, with a TracePoint on :line (#725). A
6
+ # per-test recorder can drain it between tests instead of calling
7
+ # Coverage.result, whose read walks every file of the process. Pass `files`
8
+ # (canonical paths) to keep only those; nil keeps all. Not wired into the
9
+ # recorder yet: it matches Coverage's lines mode on saboteur's own suite
10
+ # line for line, but has no branches, so the recorder's branch data still
11
+ # needs its own design.
12
+ class LineHits
13
+ def initialize(files = nil, canonical: ->(path) { path })
14
+ @wanted = files && Hash.new { |hash, path| hash[path] = wanted?(files, canonical, path) }
15
+ @hits = new_hits
16
+ @trace = TracePoint.new(:line, &method(:record))
17
+ end
18
+
19
+ def enable
20
+ @trace.enable
21
+ self
22
+ end
23
+
24
+ def disable
25
+ @trace.disable
26
+ self
27
+ end
28
+
29
+ # Stops recording without touching the TracePoint: toggling it costs
30
+ # milliseconds once an app is loaded, so a recorder pauses around its own
31
+ # work instead.
32
+ def pause
33
+ @paused = true
34
+ self
35
+ end
36
+
37
+ def resume
38
+ @paused = false
39
+ self
40
+ end
41
+
42
+ # file => Set of line numbers since the last drain. A failure inside the
43
+ # tracer is raised here: raising inside the TracePoint would unwind the
44
+ # program being traced.
45
+ def drain(clear: true)
46
+ failure = @failure
47
+ @failure = nil
48
+ raise failure if failure
49
+
50
+ hits = @hits
51
+ @hits = new_hits if clear
52
+ hits
53
+ end
54
+
55
+ private
56
+
57
+ def new_hits
58
+ Hash.new { |hash, file| hash[file] = Set.new }
59
+ end
60
+
61
+ # A path the canonicalizer refuses is not a wanted file: a NUL byte in an
62
+ # eval'd file name raises ArgumentError, and a name that is not a file
63
+ # ("(eval at ...)") raises ENOENT from a realpath canonicalizer.
64
+ def wanted?(files, canonical, path)
65
+ files.include?(canonical.call(path))
66
+ rescue ArgumentError, SystemCallError, TypeError
67
+ false
68
+ end
69
+
70
+ def record(event)
71
+ return if @paused
72
+
73
+ path = event.path
74
+ return if @wanted && !@wanted[path]
75
+
76
+ @hits[path] << event.lineno
77
+ rescue StandardError => e
78
+ @failure ||= e
79
+ end
80
+ end
81
+ end
@@ -0,0 +1,91 @@
1
+ # SPDX-License-Identifier: MIT
2
+ # frozen_string_literal: true
3
+
4
+ module Saboteur
5
+ # Records which Ruby methods were entered, by [first line, name] per file,
6
+ # with a TracePoint on :call. Coverage's methods report does the same job
7
+ # but its read walks every method in the process (about 8 ms per read once
8
+ # an app is loaded), and the recorder reads twice per example. :call
9
+ # events are rare next to :line events, and an endless def fires one too
10
+ # (#107). Pass `files` (canonical paths) to keep only those; nil keeps all.
11
+ class MethodCalls
12
+ # The tracer the executable arms before the app loads; the engine reads
13
+ # it once after boot and disables it.
14
+ def self.boot
15
+ @boot ||= new.enable
16
+ end
17
+
18
+ def self.booted
19
+ @boot
20
+ end
21
+
22
+ def initialize(files = nil, canonical: ->(path) { path })
23
+ @wanted = files && Hash.new { |hash, path| hash[path] = wanted?(files, canonical, path) }
24
+ @hits = new_hits
25
+ @trace = TracePoint.new(:call, &method(:record))
26
+ end
27
+
28
+ def enable
29
+ @trace.enable unless @trace.enabled?
30
+ self
31
+ end
32
+
33
+ def disable
34
+ @trace.disable if @trace.enabled?
35
+ self
36
+ end
37
+
38
+ # Stops recording without touching the TracePoint: toggling it costs
39
+ # milliseconds once an app is loaded, so the recorder pauses per read.
40
+ def pause
41
+ @paused = true
42
+ self
43
+ end
44
+
45
+ def resume
46
+ @paused = false
47
+ self
48
+ end
49
+
50
+ # file => Set of [line, name] since the last drain. A failure inside the
51
+ # tracer is raised here: raising inside the TracePoint would unwind the
52
+ # program being traced (and kill a forked child outright).
53
+ def drain(clear: true)
54
+ failure = @failure
55
+ @failure = nil
56
+ raise failure if failure
57
+
58
+ hits = @hits
59
+ @hits = new_hits if clear
60
+ hits
61
+ end
62
+
63
+ private
64
+
65
+ def new_hits
66
+ Hash.new { |hash, file| hash[file] = Set.new }
67
+ end
68
+
69
+ # A path the canonicalizer refuses (a NUL byte in an eval'd file name
70
+ # raises ArgumentError) is not a wanted file.
71
+ def wanted?(files, canonical, path)
72
+ files.include?(canonical.call(path))
73
+ rescue ArgumentError
74
+ false
75
+ end
76
+
77
+ def record(event)
78
+ # self.equal?(...) never sends a message to the traced receiver, which
79
+ # may be a BasicObject proxy that answers every message (ActiveSupport's
80
+ # deprecation proxies do).
81
+ return if @paused || equal?(event.self) || event.event != :call
82
+
83
+ path = event.path
84
+ return if @wanted && !@wanted[path]
85
+
86
+ @hits[path] << [event.lineno, event.method_id]
87
+ rescue StandardError => e
88
+ @failure ||= e
89
+ end
90
+ end
91
+ end