saboteur 0.8.0 → 0.9.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (128) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +215 -0
  3. data/README.md +44 -6
  4. data/exe/saboteur +6 -2
  5. data/lib/saboteur/attribute_finder.rb +116 -0
  6. data/lib/saboteur/baseline/run_evidence.rb +2 -1
  7. data/lib/saboteur/cli/agent_provenance.rb +59 -0
  8. data/lib/saboteur/cli/dirty_tree_notice.rb +28 -0
  9. data/lib/saboteur/cli/doctor_command.rb +41 -0
  10. data/lib/saboteur/cli/error_cause.rb +40 -0
  11. data/lib/saboteur/cli/execution_options.rb +46 -0
  12. data/lib/saboteur/cli/exit_status.rb +12 -14
  13. data/lib/saboteur/cli/explicit_file_options.rb +21 -0
  14. data/lib/saboteur/cli/explicit_sources.rb +46 -0
  15. data/lib/saboteur/cli/failure_output.rb +35 -0
  16. data/lib/saboteur/cli/help_command.rb +23 -0
  17. data/lib/saboteur/cli/matrix_option.rb +17 -0
  18. data/lib/saboteur/cli/output_options.rb +27 -0
  19. data/lib/saboteur/cli/presets.rb +41 -1
  20. data/lib/saboteur/cli/preview_command.rb +26 -0
  21. data/lib/saboteur/cli/run_budget_options.rb +6 -1
  22. data/lib/saboteur/cli/run_command.rb +33 -0
  23. data/lib/saboteur/cli/run_output.rb +38 -0
  24. data/lib/saboteur/cli/run_reporting.rb +6 -5
  25. data/lib/saboteur/cli/run_validation.rb +54 -0
  26. data/lib/saboteur/cli/source_set.rb +0 -18
  27. data/lib/saboteur/cli.rb +29 -305
  28. data/lib/saboteur/config.rb +76 -7
  29. data/lib/saboteur/config_file.rb +12 -8
  30. data/lib/saboteur/constant_subject_finder.rb +3 -0
  31. data/lib/saboteur/custom_map.rb +24 -5
  32. data/lib/saboteur/deep_source.rb +49 -0
  33. data/lib/saboteur/dirty_tree_warning.rb +6 -4
  34. data/lib/saboteur/doctor.rb +10 -0
  35. data/lib/saboteur/engine/baseline_guard.rb +118 -15
  36. data/lib/saboteur/engine/baseline_parts.rb +51 -0
  37. data/lib/saboteur/engine/boot.rb +11 -3
  38. data/lib/saboteur/engine/bytecode_equivalence.rb +17 -1
  39. data/lib/saboteur/engine/candidate_selection.rb +16 -5
  40. data/lib/saboteur/engine/candidates.rb +4 -1
  41. data/lib/saboteur/engine/coverage_inputs_snapshot.rb +146 -0
  42. data/lib/saboteur/engine/coverage_miss_note.rb +86 -0
  43. data/lib/saboteur/engine/coverage_partial.rb +48 -0
  44. data/lib/saboteur/engine/coverage_parts.rb +50 -0
  45. data/lib/saboteur/engine/coverage_shadow.rb +92 -0
  46. data/lib/saboteur/engine/coverage_store.rb +296 -21
  47. data/lib/saboteur/engine/diagnostics.rb +24 -7
  48. data/lib/saboteur/engine/distribution.rb +2 -2
  49. data/lib/saboteur/engine/gated_run.rb +4 -1
  50. data/lib/saboteur/engine/load_coverage.rb +72 -8
  51. data/lib/saboteur/engine/partial_invalidation.rb +115 -0
  52. data/lib/saboteur/engine/partial_merge.rb +40 -0
  53. data/lib/saboteur/engine/pipelines.rb +11 -3
  54. data/lib/saboteur/engine/reference_specs.rb +71 -0
  55. data/lib/saboteur/engine/result_cache.rb +68 -5
  56. data/lib/saboteur/engine/resume_manifest.rb +110 -0
  57. data/lib/saboteur/engine/run_budget.rb +10 -7
  58. data/lib/saboteur/engine/run_fingerprint.rb +2 -1
  59. data/lib/saboteur/engine/sandbox_policy.rb +26 -5
  60. data/lib/saboteur/engine/scheduler.rb +33 -25
  61. data/lib/saboteur/engine/selection_announcement.rb +9 -3
  62. data/lib/saboteur/engine/shadow_comparison.rb +121 -0
  63. data/lib/saboteur/engine/shadow_run.rb +38 -0
  64. data/lib/saboteur/engine/subject_selection.rb +144 -64
  65. data/lib/saboteur/engine/suite_partition.rb +22 -0
  66. data/lib/saboteur/engine/test_guess.rb +186 -32
  67. data/lib/saboteur/engine/unchanged_syntax.rb +10 -2
  68. data/lib/saboteur/engine.rb +44 -11
  69. data/lib/saboteur/environment_hints.rb +55 -0
  70. data/lib/saboteur/execution/bundle_gemfile_pin.rb +72 -0
  71. data/lib/saboteur/execution/external_runner.rb +123 -1
  72. data/lib/saboteur/execution/fiddle_loader.rb +34 -0
  73. data/lib/saboteur/execution/gemfile_path_sources.rb +166 -0
  74. data/lib/saboteur/execution/marker_sweep.rb +2 -1
  75. data/lib/saboteur/execution/process_table.rb +7 -3
  76. data/lib/saboteur/execution/reevaluator.rb +7 -0
  77. data/lib/saboteur/execution/sandbox.rb +39 -20
  78. data/lib/saboteur/execution/seatbelt.rb +10 -3
  79. data/lib/saboteur/execution/supervisor.rb +42 -8
  80. data/lib/saboteur/execution/supervisor_batch.rb +105 -0
  81. data/lib/saboteur/execution/test_command_shape.rb +149 -0
  82. data/lib/saboteur/execution/worker_pool.rb +48 -4
  83. data/lib/saboteur/frameworks/rails.rb +2 -5
  84. data/lib/saboteur/frameworks/sqlite_memory_database.rb +32 -2
  85. data/lib/saboteur/gates.rb +121 -28
  86. data/lib/saboteur/install.rb +6 -1
  87. data/lib/saboteur/integration/base.rb +19 -2
  88. data/lib/saboteur/integration/rspec.rb +145 -18
  89. data/lib/saboteur/line_hits.rb +81 -0
  90. data/lib/saboteur/method_calls.rb +91 -0
  91. data/lib/saboteur/mutation_source/argument_swap.rb +54 -0
  92. data/lib/saboteur/mutation_source/builtin.rb +33 -21
  93. data/lib/saboteur/mutation_source/condition_negation.rb +3 -12
  94. data/lib/saboteur/mutation_source/hazard_guard.rb +94 -13
  95. data/lib/saboteur/mutation_source/jump_value.rb +59 -0
  96. data/lib/saboteur/mutation_source/literal_replacement.rb +6 -23
  97. data/lib/saboteur/mutation_source/operator_examples.rb +6 -0
  98. data/lib/saboteur/mutation_source/operator_registry.rb +6 -0
  99. data/lib/saboteur/mutation_source/operator_visit.rb +7 -1
  100. data/lib/saboteur/mutation_source/rescue_removal.rb +69 -0
  101. data/lib/saboteur/mutation_source/unary_removal.rb +47 -0
  102. data/lib/saboteur/mutation_source/when_removal.rb +51 -0
  103. data/lib/saboteur/preview.rb +13 -0
  104. data/lib/saboteur/refinement_activations.rb +22 -10
  105. data/lib/saboteur/reporter/agent.rb +2 -2
  106. data/lib/saboteur/reporter/agent_incomplete.rb +31 -5
  107. data/lib/saboteur/reporter/agent_summary.rb +16 -10
  108. data/lib/saboteur/reporter/review_comments.rb +151 -0
  109. data/lib/saboteur/reporter/safe_out.rb +32 -0
  110. data/lib/saboteur/reporter/stryker.rb +8 -6
  111. data/lib/saboteur/reporter/survivor_classification.rb +41 -3
  112. data/lib/saboteur/reporter/survivor_hints.rb +5 -0
  113. data/lib/saboteur/reporter/terminal.rb +45 -5
  114. data/lib/saboteur/result.rb +14 -4
  115. data/lib/saboteur/selection/coverage_map.rb +150 -9
  116. data/lib/saboteur/selection/external_test_map.rb +4 -0
  117. data/lib/saboteur/selection/git_diff.rb +24 -2
  118. data/lib/saboteur/selection/layout_only_lines.rb +45 -1
  119. data/lib/saboteur/selection/recorder.rb +52 -22
  120. data/lib/saboteur/selection/trace_recorder.rb +83 -0
  121. data/lib/saboteur/selection/untracked_changes.rb +1 -2
  122. data/lib/saboteur/subject.rb +16 -0
  123. data/lib/saboteur/subject_finder.rb +91 -6
  124. data/lib/saboteur/text.rb +31 -0
  125. data/lib/saboteur/version.rb +1 -1
  126. data/lib/saboteur/working_tree.rb +7 -2
  127. data/lib/saboteur.rb +9 -0
  128. metadata +48 -16
@@ -0,0 +1,49 @@
1
+ # SPDX-License-Identifier: MIT
2
+ # frozen_string_literal: true
3
+
4
+ module Saboteur
5
+ # A file nested deeper than the walkers can safely recurse (#719).
6
+ class DeepSourceError < Error
7
+ def initialize(file, limit)
8
+ super("#{file} nests deeper than #{limit} AST levels; saboteur cannot walk it safely")
9
+ end
10
+ end
11
+
12
+ # The AST walkers (SubjectFinder, ConstantSubjectFinder, AttributeFinder
13
+ # and the operator visits) recurse one stack frame per nesting level and
14
+ # overflow the Ruby stack around a thousand nested calls, while Prism
15
+ # still parses to about five thousand (#719). Measuring depth iteratively
16
+ # before any walk turns that crash into a clean refusal: the CLI boundary
17
+ # prints one line naming the file and exits 3.
18
+ module DeepSource
19
+ # Two AST levels per nested call (`a(` adds a CallNode and an
20
+ # ArgumentsNode). The finders walk a file from its root and survive a
21
+ # thousand nested calls, so their limit sits just above that. The
22
+ # operator visits recurse deeper per level (method_swap overflows a
23
+ # bare stack around 900 nested calls, earlier under the engine), so a
24
+ # subject's body refuses earlier. Both margins stay deterministic:
25
+ # a clean refusal instead of a SystemStackError that depends on how
26
+ # deep the caller's stack already is.
27
+ MAX_AST_DEPTH = 2200
28
+ MAX_MUTATION_DEPTH = 1200
29
+
30
+ def self.check!(program, file, limit: MAX_AST_DEPTH)
31
+ raise DeepSourceError.new(file, limit) if depth(program) > limit
32
+ end
33
+
34
+ # Iterative on purpose: a recursive measure would crash on exactly the
35
+ # files it exists to catch (#719, same shape as #711).
36
+ def self.depth(node)
37
+ max = 0
38
+ work = [[node, 1]]
39
+ until work.empty?
40
+ current, level = work.pop
41
+ next if current.nil?
42
+
43
+ max = level if level > max
44
+ current.compact_child_nodes.each { |child| work << [child, level + 1] }
45
+ end
46
+ max
47
+ end
48
+ end
49
+ end
@@ -2,8 +2,9 @@
2
2
  # frozen_string_literal: true
3
3
 
4
4
  module Saboteur
5
- # The warning shown before a run on a tree with uncommitted changes
6
- # (#309, #316). Mutants run your code and can change the tree, so the
5
+ # The warning shown before a run on a tree with uncommitted
6
+ # changes (#309, #316); the headline says uncommitted or untracked, so a
7
+ # tree of only scratch files is not read as edits (#604). Mutants run your code and can change the tree, so the
7
8
  # warning stays; it splits the raw count so a tree full of notes or trial
8
9
  # setup files does not read like application edits about to be mutated.
9
10
  #
@@ -25,7 +26,7 @@ module Saboteur
25
26
  # unknown tree. The root is physical (realpath), like the paths
26
27
  # Config#included_files returns.
27
28
  def self.for(config, config_path:, gemfile: ENV.fetch("BUNDLE_GEMFILE", nil))
28
- state = WorkingTree.new(config.root, ignored: config.output_paths).state
29
+ state = WorkingTree.new(config.root, ignored: config.ignored_tree_paths(gemfile: gemfile)).state
29
30
  root = File.realpath(config.root)
30
31
  new(state, included: relative(config.included_files, root), include: config["include"],
31
32
  exclude: config["exclude"], root: root,
@@ -101,7 +102,8 @@ module Saboteur
101
102
  groups = [scope.empty? ? "none inside the mutation scope" : counted("inside the mutation scope", scope),
102
103
  counted("outside it", outside)]
103
104
  groups << counted("saboteur setup file(s)", setup) unless setup.empty?
104
- "saboteur: warning: #{@state.changed_files} uncommitted change(s) in the working tree: #{groups.join(', ')}."
105
+ "saboteur: warning: #{@state.changed_files} uncommitted or untracked change(s) in the working tree: " \
106
+ "#{groups.join(', ')}."
105
107
  end
106
108
 
107
109
  def counted(label, changes)
@@ -19,6 +19,7 @@ module Saboteur
19
19
  def call
20
20
  Engine::Boot.refuse_recursion!
21
21
  check_runtime
22
+ check_environment
22
23
  check_coverage
23
24
  check_boot_files
24
25
  check_custom_map
@@ -55,6 +56,11 @@ module Saboteur
55
56
  raise Error, "target boot failed: #{e.class}: #{e.message}; check require files, hooks and rspec_args"
56
57
  end
57
58
 
59
+ def check_environment
60
+ hints = EnvironmentHints.all(@config.root)
61
+ @out.puts "Environment: #{hints.empty? ? 'no bundler/appraisal problems detected' : hints.join('; ')}"
62
+ end
63
+
58
64
  def check_runtime
59
65
  @out.puts "Ruby: #{RUBY_DESCRIPTION}"
60
66
  unless Gem::Version.new(RUBY_VERSION) >= Gem::Version.new("3.3") && Process.respond_to?(:fork)
@@ -102,8 +108,12 @@ module Saboteur
102
108
  @out.puts %(Seatbelt (opt-in, #436): probe #{Execution::Seatbelt.probe} - capability probe only, ) \
103
109
  "not verified isolation"
104
110
  elsif RUBY_PLATFORM.match?(/darwin/i)
111
+ # Point at the way OUT of the degraded state, like the run
112
+ # warning does (#667): the opt-in Seatbelt sandbox, or silence.
105
113
  @out.puts "macOS: mutants run unsandboxed (Landlock is Linux-only) - a container contains " \
106
114
  "filesystem writes outside the project"
115
+ @out.puts %(macOS: opt into the sandbox with c.sandbox = "seatbelt" ) \
116
+ "(docs/guides/macos-seatbelt-sandbox.md), or set c.sandbox = false to silence this"
107
117
  @out.puts "Network: never restricted on any platform, and a plain container keeps outbound " \
108
118
  "access - deny or stub the network when a mutant could reach real APIs"
109
119
  end
@@ -5,6 +5,7 @@ require "json"
5
5
  require "fileutils"
6
6
  require_relative "../text"
7
7
  require_relative "../frameworks/sqlite_memory_database"
8
+ require_relative "baseline_parts"
8
9
 
9
10
  module Saboteur
10
11
  class Engine
@@ -12,6 +13,8 @@ module Saboteur
12
13
  # or mutation source is used. This catches observed instability, not all
13
14
  # possible flaky behaviour.
14
15
  class BaselineGuard
16
+ include BaselineParts
17
+
15
18
  # How many unchanged-suite checks actually completed and validated:
16
19
  # the number of order comparisons a run can claim is one less.
17
20
  attr_reader :completed_passes
@@ -24,7 +27,7 @@ module Saboteur
24
27
  # coverage build when the cache is cold, so the first timed pass can
25
28
  # price the rest (#260).
26
29
  def initialize(config:, integration:, hooks:, supervisor:, diagnostics:, full_passes: 2, between_passes: nil,
27
- timeout_cap: nil)
30
+ timeout_cap: nil, run_started: nil)
28
31
  @config = config
29
32
  @integration = integration
30
33
  @hooks = hooks
@@ -33,6 +36,7 @@ module Saboteur
33
36
  @full_passes = full_passes
34
37
  @between_passes = between_passes
35
38
  @timeout_cap = timeout_cap
39
+ @run_started = run_started
36
40
  @completed_passes = 0
37
41
  end
38
42
 
@@ -51,10 +55,7 @@ module Saboteur
51
55
  started = Process.clock_gettime(Process::CLOCK_MONOTONIC)
52
56
  data = check(number)
53
57
  announce_first_pass_cost(number, started)
54
- unless data["passed"] && data["failed"].empty?
55
- refuse(number,
56
- with_hint("the suite failed: #{Array(data['failed']).join(', ')}", data))
57
- end
58
+ refuse_suite_failure(number, data) unless data["passed"] && data["failed"].empty?
58
59
  executed = data["executed"]
59
60
  refuse(number, "executed tests do not match the loaded suite") unless
60
61
  executed.sort == expected.sort
@@ -67,6 +68,16 @@ module Saboteur
67
68
 
68
69
  private
69
70
 
71
+ def refuse_failed_suite(number, data)
72
+ outside = data["failed"].empty?
73
+ reason = if outside
74
+ "the suite errored outside any example; see the log tail"
75
+ else
76
+ "the suite failed: #{data['failed'].join(', ')}"
77
+ end
78
+ refuse(number, with_hint(reason, data), tail: outside)
79
+ end
80
+
70
81
  # A spent --max-duration budget skips the remaining passes - including
71
82
  # the first, when boot alone already spent it (#245).
72
83
  def budget_spent?(_index)
@@ -95,9 +106,10 @@ module Saboteur
95
106
 
96
107
  coverage_pending = @full_passes > @config["baseline_passes"]
97
108
  remaining = @full_passes - number - (coverage_pending ? 1 : 0)
109
+ elapsed = (Process.clock_gettime(Process::CLOCK_MONOTONIC) - started).round
110
+ warn_budget_projection(elapsed, remaining + (coverage_pending ? 1 : 0))
98
111
  return if remaining.zero? && !coverage_pending
99
112
 
100
- elapsed = (Process.clock_gettime(Process::CLOCK_MONOTONIC) - started).round
101
113
  work = []
102
114
  work << "the remaining #{remaining} full #{remaining == 1 ? 'pass' : 'passes'}" if remaining.positive?
103
115
  work << "the coverage map build" if coverage_pending
@@ -106,6 +118,28 @@ module Saboteur
106
118
  "#{work.join(' and ')} at that rate")
107
119
  end
108
120
 
121
+ # The first timed pass also prices the run against --max-duration:
122
+ # boot and this pass are spent, the remaining passes and a cold
123
+ # coverage build cost about that much again at that rate, and only
124
+ # what is left feeds mutants. Warn while a Ctrl-C is still cheap -
125
+ # the refusal only fires once the coverage build is unaffordable
126
+ # outright (#637).
127
+ def warn_budget_projection(pass_elapsed, remaining_passes)
128
+ budget = @config["max_duration"]
129
+ return unless budget && @run_started
130
+
131
+ spent = Process.clock_gettime(Process::CLOCK_MONOTONIC) - @run_started
132
+ projected = spent + (pass_elapsed * remaining_passes)
133
+ mutants_get = budget - projected
134
+ return if mutants_get >= pass_elapsed
135
+
136
+ verdict = mutants_get <= 0 ? "exceeds" : "barely fits"
137
+ @diagnostics.warn("raise --max-duration, pass fewer --baseline-passes, or narrow the scope\n " \
138
+ "projected baseline+coverage ~#{projected.round}s #{verdict} " \
139
+ "--max-duration #{budget.round}s; mutants would get " \
140
+ "~#{format('%g', [mutants_get, 0].max.round(1))}s")
141
+ end
142
+
109
143
  # An admitted pass still gets only the time left in the run budget:
110
144
  # without the cap it would overrun a spent --max-duration by its
111
145
  # whole coverage_timeout.
@@ -145,6 +179,9 @@ module Saboteur
145
179
  end
146
180
 
147
181
  def check(number)
182
+ parts = baseline_parts
183
+ return check_parts(number, parts) if parts.size > 1
184
+
148
185
  Execution::ForkReadiness.prepare_heap
149
186
  heartbeat = @diagnostics.progress_heartbeat("baseline check #{number}").watch(log_path(number))
150
187
  admitted = phase_timeout
@@ -152,6 +189,11 @@ module Saboteur
152
189
  label: "baseline child died during setup/run") do |writer|
153
190
  record(writer, number)
154
191
  end
192
+ settled(number, result, admitted)
193
+ end
194
+
195
+ def settled(number, result, admitted, part: nil)
196
+ @part = part
155
197
  refuse_budget_stop(number, admitted) if budget_stopped?(result, admitted)
156
198
  refuse(number, stopped_reason(result)) if result.stopped_by
157
199
  # Before the payload checks: a child killed by a signal (the OOM
@@ -190,15 +232,16 @@ module Saboteur
190
232
  data["executed"].is_a?(Array) && data["executed"].all?(String)
191
233
  end
192
234
 
193
- def record(writer, number)
194
- path = log_path(number)
235
+ def record(writer, number, files: nil, part: nil)
236
+ path = log_path(number, part)
195
237
  FileUtils.mkdir_p(File.dirname(path))
196
238
  $stdout.reopen(path, "w")
197
239
  $stdout.sync = true
198
240
  $stderr.reopen($stdout)
199
241
  @hooks.run(:after_fork)
200
242
  Kernel.srand(@config["seed"])
201
- result = @integration.run_baseline(seed: @config["seed"])
243
+ seed = @config["seed"]
244
+ result = files ? @integration.run_baseline(seed: seed, files: files) : @integration.run_baseline(seed: seed)
202
245
  payload = { passed: result.passed, failed: result.failed_ids, executed: result.executed_ids }
203
246
  payload[:hint] = Frameworks::SqliteMemoryDatabase.fork_loss_hint unless result.passed
204
247
  writer.write(JSON.generate(payload))
@@ -209,14 +252,74 @@ module Saboteur
209
252
  writer.close
210
253
  end
211
254
 
255
+ # A failed unchanged suite never runs mutants: warn first when the
256
+ # failure carries the Minitest signature (#626), then refuse.
257
+ def refuse_suite_failure(number, data)
258
+ warn_minitest_signature(number)
259
+ refuse_failed_suite(number, data)
260
+ end
261
+
262
+ # Minitest::Spec suites in *_spec.rb clothing load under the RSpec
263
+ # integration and then fail the baseline with NoMethodError for
264
+ # Minitest-only methods (#626): the refusal still reads "fix the
265
+ # suite", so warn what the signature actually points at. It stays a
266
+ # warning: an RSpec suite can require minitest for a helper and
267
+ # fail for its own reason, and a heuristic refusal would
268
+ # false-positive there.
269
+ def warn_minitest_signature(number)
270
+ lines = log_lines(number)
271
+ errors = lines.grep(/NoMethodError/)
272
+ return if errors.empty? || !spec_helper_requires_minitest?
273
+
274
+ symptom = (lines.find { |line| line.include?("undefined method") } || errors.first).strip
275
+ @diagnostics.warn("run saboteur on an RSpec suite: it is RSpec-only, so a Minitest suite cannot be " \
276
+ "mutation tested with it\n " \
277
+ "the baseline failure looks like a Minitest suite running under RSpec, not a " \
278
+ "broken RSpec suite: the log shows #{errors.size} " \
279
+ "NoMethodError#{'s' if errors.size > 1} (for example: #{symptom.slice(0, EXCERPT_WIDTH)}) " \
280
+ "and spec/spec_helper.rb requires minitest")
281
+ end
282
+
283
+ # The framework check behind the warning: spec_helper is the
284
+ # suite's own declaration of what it runs on. Parsed, not grepped: a
285
+ # regex also fires on commented-out or heredoc "require" lines for a
286
+ # framework the project does not use (#708).
287
+ def spec_helper_requires_minitest?
288
+ require_calls(Prism.parse_file(File.join(@config.root, "spec", "spec_helper.rb")).value)
289
+ .any? { |name| name == "minitest" || name.start_with?("minitest/") }
290
+ rescue SystemCallError
291
+ false
292
+ end
293
+
294
+ # Iterative like RefinementActivations.collect: a deeply nested
295
+ # spec_helper must never raise SystemStackError out of a warning
296
+ # heuristic (same class as #711).
297
+ def require_calls(node)
298
+ found = []
299
+ work = [node]
300
+ until work.empty?
301
+ current = work.pop
302
+ next unless current.is_a?(Prism::Node)
303
+
304
+ if current.is_a?(Prism::CallNode) && current.name == :require && current.receiver.nil?
305
+ argument = current.arguments&.arguments&.first
306
+ found << argument.unescaped if argument.is_a?(Prism::StringNode)
307
+ end
308
+ work.concat(current.compact_child_nodes)
309
+ end
310
+ found
311
+ end
312
+
212
313
  # A child that recognises its own failure shape (an empty sqlite
213
314
  # :memory: database after the fork, #387) says how to fix it.
214
315
  def with_hint(reason, data)
215
316
  data["hint"] ? "#{reason}. #{data['hint']}" : reason
216
317
  end
217
318
 
218
- def log_path(number)
219
- File.join(@config.path("cache_dir"), "baseline-#{number}.log")
319
+ # A split check keeps one log per part: baseline-1-part-2.log.
320
+ def log_path(number, part = @part)
321
+ name = part ? "baseline-#{number}-part-#{part + 1}.log" : "baseline-#{number}.log"
322
+ File.join(@config.path("cache_dir"), name)
220
323
  end
221
324
 
222
325
  # Leftover children are a pass-with-leaks verdict, not a suite
@@ -230,9 +333,9 @@ module Saboteur
230
333
  "the suite passed but left #{count} child #{count == 1 ? 'process' : 'processes'} behind: #{detail}"
231
334
  end
232
335
 
233
- def refuse(number, reason, excerpt: true)
336
+ def refuse(number, reason, excerpt: true, tail: false)
234
337
  raise Error, "baseline check #{number} failed at seed #{@config['seed']}: #{reason}. " \
235
- "#{failure_excerpt(number) if excerpt}See the full log at #{log_path(number)}. " \
338
+ "#{failure_excerpt(number, tail: tail) if excerpt}See the full log at #{log_path(number)}. " \
236
339
  "Fix the suite or its setup, then run saboteur again with the same seed. " \
237
340
  "No mutants were started."
238
341
  end
@@ -240,11 +343,11 @@ module Saboteur
240
343
  # The first lines of the failure, inline, so the cause is readable
241
344
  # without opening the log: RSpec's "Failures:" section when the log has
242
345
  # one, otherwise the last lines of the log.
243
- def failure_excerpt(number)
346
+ def failure_excerpt(number, tail: false)
244
347
  lines = log_lines(number)
245
348
  return "" if lines.empty?
246
349
 
247
- start = lines.index { |line| line.start_with?("Failures:") }
350
+ start = lines.index { |line| line.start_with?("Failures:") } unless tail
248
351
  shown = start ? lines[start, EXCERPT_LINES] : lines.last(EXCERPT_LINES)
249
352
  "The log says:\n#{shown.map { |line| " #{line.slice(0, EXCERPT_WIDTH)}" }.join("\n")}\n"
250
353
  end
@@ -0,0 +1,51 @@
1
+ # SPDX-License-Identifier: MIT
2
+ # frozen_string_literal: true
3
+
4
+ require_relative "suite_partition"
5
+
6
+ module Saboteur
7
+ class Engine
8
+ # A baseline check split like the coverage build (--coverage-parts N):
9
+ # the same disjoint spec-file parts run as children at once, each over
10
+ # its own files. Uses the guard's @config, @integration, @supervisor,
11
+ # @diagnostics, @part and settled/phase_timeout/log_path.
12
+ #
13
+ # The check's contract is unchanged. Every part must pass, and the
14
+ # examples the parts executed together must be exactly the loaded
15
+ # suite's (the guard compares them before it accepts the pass). What a
16
+ # split no longer shows is a failure that needs the whole suite in one
17
+ # process.
18
+ module BaselineParts
19
+ private
20
+
21
+ # The parts the coverage build would use. One part (or none, for an
22
+ # adapter that lists no spec files) means the ordinary single check.
23
+ def baseline_parts
24
+ Engine::SuitePartition.parts(@integration.test_files, @config["coverage_parts"])
25
+ end
26
+
27
+ def check_parts(number, parts)
28
+ Execution::ForkReadiness.prepare_heap
29
+ heartbeat = @diagnostics.progress_heartbeat("baseline check #{number} (#{parts.size} parts)")
30
+ .watch(log_path(number, 0))
31
+ admitted = phase_timeout
32
+ results = @supervisor.run_many(parts.size, timeout: admitted, on_tick: heartbeat.method(:tick),
33
+ label: "baseline child died during setup/run") do |writer, slot|
34
+ ENV["TEST_ENV_NUMBER"] = slot.zero? ? "" : (slot + 1).to_s
35
+ record(writer, number, files: parts[slot], part: slot)
36
+ end
37
+ joined(results.each_with_index.map { |result, slot| settled(number, result, admitted, part: slot) })
38
+ end
39
+
40
+ # What the guard's checks read: the first failing part's own result
41
+ # (its failures and hint; @part names its log), or a pass carrying
42
+ # every example the parts executed.
43
+ def joined(datas)
44
+ @part = datas.index { |data| !(data["passed"] && data["failed"].empty?) }
45
+ return datas[@part] if @part
46
+
47
+ { "passed" => true, "failed" => [], "executed" => datas.flat_map { |data| data["executed"] } }
48
+ end
49
+ end
50
+ end
51
+ end
@@ -66,9 +66,15 @@ module Saboteur
66
66
  @rails = rails = Frameworks::Rails.new(@config, @hooks) if @config["rails"]
67
67
  rails&.before_boot
68
68
  @hooks.run(:before_boot)
69
- add_spec_to_load_path if @config["rails"]
69
+ add_spec_to_load_path
70
+ # Helpers may configure RSpec before the integration loads specs.
71
+ Integration::RSpec.new(@config).require_framework if @config["integration"] == "rspec"
70
72
  @config["require"].each { |file| require File.expand_path(file, @root) }
71
73
  integration = Integration.fetch(@config["integration"]).new(@config)
74
+ # Running outside the bundle or beside an Appraisal gemfile set
75
+ # loads the wrong gems; say so before the spec load fails in
76
+ # confusing ways (#598, #609). Advisory only.
77
+ EnvironmentHints.all(@root).each { |hint| warn "saboteur: warning: #{hint}" }
72
78
  integration.setup
73
79
  rails&.after_boot
74
80
  @hooks.run(:after_boot)
@@ -78,8 +84,10 @@ module Saboteur
78
84
  private
79
85
 
80
86
  # rspec puts spec/ on the load path, so a boot file under spec/ (a
81
- # dummy app's rails_helper, say) can require its siblings. Match
82
- # that, or the same require list fails to load.
87
+ # dummy app's rails_helper, or a non-Rails gem's spec_helper that
88
+ # requires its siblings) can require its siblings. Match that for
89
+ # every project - a non-Rails gem does not add spec/ itself (#615) -
90
+ # or the same require list fails to load.
83
91
  def add_spec_to_load_path
84
92
  path = File.join(@root, "spec")
85
93
  $LOAD_PATH.unshift(path) unless $LOAD_PATH.include?(path)
@@ -42,11 +42,27 @@ module Saboteur
42
42
 
43
43
  def fingerprint(text, frozen)
44
44
  options = { frozen_string_literal: frozen }
45
- strip_positions(RubyVM::InstructionSequence.compile(text, nil, nil, 1, options).to_a)
45
+ iseq = warnings_silenced do
46
+ RubyVM::InstructionSequence.compile(text, "saboteur/bytecode-equivalence", nil, 1, options)
47
+ end
48
+ strip_positions(iseq.to_a)
46
49
  rescue SyntaxError, StandardError
47
50
  nil
48
51
  end
49
52
 
53
+ # Mutants compile one by one, and generated bodies (body_nil,
54
+ # statement_removal) trip Ruby's own void-context warnings, so a host
55
+ # suite running with -w gets one line per mutant flooding stderr
56
+ # (#592, #656). The check is internal: silence the compiler. A mutant
57
+ # that does not parse still fails the compile and runs.
58
+ def warnings_silenced
59
+ saved = $VERBOSE
60
+ $VERBOSE = nil
61
+ yield
62
+ ensure
63
+ $VERBOSE = saved
64
+ end
65
+
50
66
  # Structural equality that never coerces: an Integer is not a Float,
51
67
  # and +0.0 is not -0.0 (1.0 / +0.0 and 1.0 / -0.0 are different
52
68
  # infinities), which eql? alone would not tell apart.
@@ -8,11 +8,12 @@ module Saboteur
8
8
  class CandidateSelection
9
9
  attr_reader :skipped_subjects, :selected, :unscoped_digests
10
10
 
11
- def initialize(candidates:, map:, limit:, diagnostics:)
11
+ def initialize(candidates:, map:, limit:, diagnostics:, root: nil)
12
12
  @candidates = candidates
13
13
  @map = map
14
14
  @limit = limit
15
15
  @diagnostics = diagnostics
16
+ @root = root
16
17
  @skipped_subjects = {}
17
18
  @selected = {}
18
19
  @unscoped_digests = {}
@@ -29,6 +30,7 @@ module Saboteur
29
30
  return @selected[subject] = []
30
31
  end
31
32
 
33
+ warn_boot_once(subject)
32
34
  list = @candidates.for(subject)
33
35
  # The full pre-scope digests let IgnoreList#apply tell "mutation gone"
34
36
  # from "mutation scoped out of this run" (#284).
@@ -48,24 +50,33 @@ module Saboteur
48
50
 
49
51
  private
50
52
 
53
+ # A note, not a verdict (#575): every mutant still runs and scores.
54
+ def warn_boot_once(subject)
55
+ return unless @map.boot_once?(subject)
56
+
57
+ @diagnostics.warn("#{subject.id}: verify a survivor here by mutating the file on disk\n " \
58
+ "covered by every test and run at boot: mutants apply after boot, so a survivor " \
59
+ "can be a boot-once effect the suite cannot re-observe (#575)")
60
+ end
61
+
51
62
  # A scoped no-coverage skip names the specs the run loaded: coverage
52
63
  # living in a sibling spec file is otherwise invisible, and the false
53
64
  # read is "untestable" when the truth is "not loaded" (#636). Names
54
65
  # at most five, like the other selection notes.
55
66
  def skip_announcement(subject, reason)
56
- line = "#{subject.id}: skipped (#{reason}); its change is not verified by this run"
67
+ line = "#{subject.id} (#{subject.location(@root)}): skipped (#{reason}); its change is not verified by this run"
57
68
  return line unless reason == "no coverage"
58
69
 
59
70
  loaded = @map.tests.map { |id| id.split("[").first.delete_prefix("./") }.uniq.sort
60
- more = loaded.size > 5 ? ", ..." : ""
61
- names = loaded.empty? ? "none" : loaded.first(5).join(", ")
62
- "#{line} - loaded specs: #{names}#{more}; " \
71
+ names = loaded.empty? ? "none" : Text.name_list(loaded)
72
+ "#{line} - loaded specs: #{names}; " \
63
73
  "other spec files may cover it, pass --test to load them"
64
74
  end
65
75
 
66
76
  # Load-only code is not "no coverage": it ran, just never inside a
67
77
  # test, and no forked test can re-run it (see CoverageMap::RAN_AT_LOAD).
68
78
  def skip_reason_for(subject)
79
+ return Selection::CoverageMap::PRELOADED if @map.preloaded?(subject.file)
69
80
  return "no coverage" unless @map.ran_at_load?(subject.file, subject.coverage_lines)
70
81
 
71
82
  Selection::CoverageMap::RAN_AT_LOAD
@@ -109,6 +109,9 @@ module Saboteur
109
109
  end
110
110
 
111
111
  def proposals(source, subject)
112
+ # The operator visits recurse over the subject's body; refuse a
113
+ # body nested too deeply instead of crashing the run (#719).
114
+ DeepSource.check!(subject.node, subject.file, limit: DeepSource::MAX_MUTATION_DEPTH)
112
115
  proposals = source.mutations_for(subject)
113
116
  if proposals.empty?
114
117
  @diagnostics.record_generation(subject, source: source_name(source), reason: "empty_generation")
@@ -142,7 +145,7 @@ module Saboteur
142
145
 
143
146
  [mutation, text]
144
147
  rescue ArgumentError => e
145
- @diagnostics.warn("#{subject.id}: dropped #{mutation.source} mutation: #{e.message}")
148
+ @diagnostics.warn("#{subject.id}: dropped #{mutation.source} mutation\n #{e.message}")
146
149
  record_drop(subject, mutation, "invalid_edit")
147
150
  end
148
151