saboteur 0.8.1 → 0.9.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (115) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +100 -0
  3. data/README.md +10 -0
  4. data/exe/saboteur +6 -2
  5. data/lib/saboteur/attribute_finder.rb +3 -0
  6. data/lib/saboteur/baseline/run_evidence.rb +2 -1
  7. data/lib/saboteur/cli/agent_provenance.rb +59 -0
  8. data/lib/saboteur/cli/dirty_tree_notice.rb +28 -0
  9. data/lib/saboteur/cli/doctor_command.rb +41 -0
  10. data/lib/saboteur/cli/error_cause.rb +40 -0
  11. data/lib/saboteur/cli/execution_options.rb +46 -0
  12. data/lib/saboteur/cli/exit_status.rb +12 -14
  13. data/lib/saboteur/cli/explicit_file_options.rb +21 -0
  14. data/lib/saboteur/cli/explicit_sources.rb +46 -0
  15. data/lib/saboteur/cli/failure_output.rb +35 -0
  16. data/lib/saboteur/cli/help_command.rb +23 -0
  17. data/lib/saboteur/cli/matrix_option.rb +17 -0
  18. data/lib/saboteur/cli/output_options.rb +27 -0
  19. data/lib/saboteur/cli/preview_command.rb +26 -0
  20. data/lib/saboteur/cli/run_budget_options.rb +3 -0
  21. data/lib/saboteur/cli/run_command.rb +33 -0
  22. data/lib/saboteur/cli/run_output.rb +38 -0
  23. data/lib/saboteur/cli/run_reporting.rb +6 -5
  24. data/lib/saboteur/cli/run_validation.rb +54 -0
  25. data/lib/saboteur/cli.rb +29 -307
  26. data/lib/saboteur/config.rb +72 -6
  27. data/lib/saboteur/constant_subject_finder.rb +3 -0
  28. data/lib/saboteur/custom_map.rb +20 -12
  29. data/lib/saboteur/deep_source.rb +49 -0
  30. data/lib/saboteur/dirty_tree_warning.rb +1 -1
  31. data/lib/saboteur/doctor.rb +6 -0
  32. data/lib/saboteur/engine/baseline_guard.rb +50 -13
  33. data/lib/saboteur/engine/baseline_parts.rb +51 -0
  34. data/lib/saboteur/engine/boot.rb +4 -0
  35. data/lib/saboteur/engine/candidate_selection.rb +15 -5
  36. data/lib/saboteur/engine/candidates.rb +4 -1
  37. data/lib/saboteur/engine/coverage_inputs_snapshot.rb +146 -0
  38. data/lib/saboteur/engine/coverage_miss_note.rb +86 -0
  39. data/lib/saboteur/engine/coverage_partial.rb +48 -0
  40. data/lib/saboteur/engine/coverage_parts.rb +50 -0
  41. data/lib/saboteur/engine/coverage_shadow.rb +92 -0
  42. data/lib/saboteur/engine/coverage_store.rb +275 -22
  43. data/lib/saboteur/engine/diagnostics.rb +23 -6
  44. data/lib/saboteur/engine/distribution.rb +2 -2
  45. data/lib/saboteur/engine/gated_run.rb +1 -0
  46. data/lib/saboteur/engine/load_coverage.rb +54 -8
  47. data/lib/saboteur/engine/partial_invalidation.rb +115 -0
  48. data/lib/saboteur/engine/partial_merge.rb +40 -0
  49. data/lib/saboteur/engine/pipelines.rb +7 -7
  50. data/lib/saboteur/engine/reference_specs.rb +71 -0
  51. data/lib/saboteur/engine/result_cache.rb +68 -5
  52. data/lib/saboteur/engine/resume_manifest.rb +110 -0
  53. data/lib/saboteur/engine/run_budget.rb +5 -5
  54. data/lib/saboteur/engine/run_fingerprint.rb +2 -1
  55. data/lib/saboteur/engine/sandbox_policy.rb +5 -2
  56. data/lib/saboteur/engine/scheduler.rb +1 -1
  57. data/lib/saboteur/engine/selection_announcement.rb +1 -0
  58. data/lib/saboteur/engine/shadow_comparison.rb +121 -0
  59. data/lib/saboteur/engine/shadow_run.rb +38 -0
  60. data/lib/saboteur/engine/subject_selection.rb +33 -37
  61. data/lib/saboteur/engine/suite_partition.rb +22 -0
  62. data/lib/saboteur/engine/test_guess.rb +82 -18
  63. data/lib/saboteur/engine/unchanged_syntax.rb +10 -2
  64. data/lib/saboteur/engine.rb +41 -9
  65. data/lib/saboteur/environment_hints.rb +55 -0
  66. data/lib/saboteur/execution/bundle_gemfile_pin.rb +72 -0
  67. data/lib/saboteur/execution/external_runner.rb +105 -1
  68. data/lib/saboteur/execution/gemfile_path_sources.rb +166 -0
  69. data/lib/saboteur/execution/sandbox.rb +30 -17
  70. data/lib/saboteur/execution/supervisor.rb +40 -7
  71. data/lib/saboteur/execution/supervisor_batch.rb +105 -0
  72. data/lib/saboteur/execution/worker_pool.rb +48 -4
  73. data/lib/saboteur/frameworks/rails.rb +2 -5
  74. data/lib/saboteur/gates.rb +121 -28
  75. data/lib/saboteur/install.rb +6 -1
  76. data/lib/saboteur/integration/base.rb +19 -2
  77. data/lib/saboteur/integration/rspec.rb +46 -7
  78. data/lib/saboteur/line_hits.rb +81 -0
  79. data/lib/saboteur/method_calls.rb +91 -0
  80. data/lib/saboteur/mutation_source/builtin.rb +32 -22
  81. data/lib/saboteur/mutation_source/condition_negation.rb +3 -12
  82. data/lib/saboteur/mutation_source/jump_value.rb +59 -0
  83. data/lib/saboteur/mutation_source/literal_replacement.rb +6 -23
  84. data/lib/saboteur/mutation_source/operator_examples.rb +5 -0
  85. data/lib/saboteur/mutation_source/operator_registry.rb +5 -0
  86. data/lib/saboteur/mutation_source/operator_visit.rb +7 -1
  87. data/lib/saboteur/mutation_source/rescue_removal.rb +69 -0
  88. data/lib/saboteur/mutation_source/unary_removal.rb +47 -0
  89. data/lib/saboteur/mutation_source/when_removal.rb +51 -0
  90. data/lib/saboteur/preview.rb +9 -1
  91. data/lib/saboteur/refinement_activations.rb +22 -10
  92. data/lib/saboteur/reporter/agent.rb +2 -2
  93. data/lib/saboteur/reporter/agent_incomplete.rb +11 -0
  94. data/lib/saboteur/reporter/agent_summary.rb +16 -10
  95. data/lib/saboteur/reporter/review_comments.rb +151 -0
  96. data/lib/saboteur/reporter/safe_out.rb +32 -0
  97. data/lib/saboteur/reporter/stryker.rb +8 -6
  98. data/lib/saboteur/reporter/survivor_classification.rb +41 -3
  99. data/lib/saboteur/reporter/survivor_hints.rb +5 -0
  100. data/lib/saboteur/reporter/terminal.rb +29 -4
  101. data/lib/saboteur/result.rb +14 -4
  102. data/lib/saboteur/selection/coverage_map.rb +136 -10
  103. data/lib/saboteur/selection/external_test_map.rb +4 -0
  104. data/lib/saboteur/selection/git_diff.rb +24 -2
  105. data/lib/saboteur/selection/layout_only_lines.rb +45 -1
  106. data/lib/saboteur/selection/recorder.rb +52 -22
  107. data/lib/saboteur/selection/trace_recorder.rb +83 -0
  108. data/lib/saboteur/selection/untracked_changes.rb +1 -2
  109. data/lib/saboteur/subject.rb +16 -0
  110. data/lib/saboteur/subject_finder.rb +91 -6
  111. data/lib/saboteur/text.rb +31 -0
  112. data/lib/saboteur/version.rb +1 -1
  113. data/lib/saboteur/working_tree.rb +7 -2
  114. data/lib/saboteur.rb +8 -0
  115. metadata +44 -2
@@ -18,6 +18,10 @@ module Saboteur
18
18
  LEGACY_IGNORE_FILE = ".saboteur-ignore.yml"
19
19
  IGNORE_FILE = "config/saboteur_ignore.rb"
20
20
 
21
+ # Settings that say where saboteur writes, never what it mutates: left
22
+ # out of the run fingerprint, baseline evidence and score-history scope.
23
+ OUTPUT_KEYS = %w[cache_dir json_report html_report score_history resume].freeze
24
+
21
25
  DEFAULTS = {
22
26
  "integration" => "rspec",
23
27
  "require" => [],
@@ -69,7 +73,15 @@ module Saboteur
69
73
  "subjects" => [],
70
74
  "since" => nil,
71
75
  "cache" => true,
76
+ # Experimental (#46): "shadow" also builds a partial coverage map after a
77
+ # cache miss and compares it with the full map, which the run still uses.
78
+ # "use" uses the partial map when it is safe and builds the full map otherwise.
79
+ "partial_coverage" => "off",
80
+ # coverage (default) reads Coverage around every test; trace gets lines and
81
+ # methods from a TracePoint and reads branches once per spec file (#725).
82
+ "coverage_recorder" => "coverage",
72
83
  "incremental" => false,
84
+ "resume" => nil,
73
85
  "shard" => nil,
74
86
  "allow_focus" => false,
75
87
  "fail_fast" => false,
@@ -79,6 +91,8 @@ module Saboteur
79
91
  "matrix" => false,
80
92
  "seed" => 1,
81
93
  "baseline_passes" => 2,
94
+ # Parallel children for the coverage-map build (#741); 1 keeps the single child.
95
+ "coverage_parts" => 1,
82
96
  "max_duration" => nil,
83
97
  "max_duration_ok" => false,
84
98
  "tests" => [],
@@ -92,8 +106,10 @@ module Saboteur
92
106
  "verbose" => false
93
107
  }.freeze
94
108
 
95
- INTEGER_KEYS = %w[jobs max_memory_mb max_processes max_mutants_per_method seed baseline_passes].freeze
109
+ INTEGER_KEYS = %w[jobs max_memory_mb max_processes max_mutants_per_method seed baseline_passes
110
+ coverage_parts].freeze
96
111
  FLOAT_KEYS = %w[timeout_factor timeout_min baseline_timeout coverage_timeout].freeze
112
+ STRING_KEYS = %w[test_selection partial_coverage coverage_recorder].freeze
97
113
  NILABLE_FLOAT_KEYS = %w[threshold max_duration].freeze
98
114
  THRESHOLD_RANGE = (0.0..100.0)
99
115
 
@@ -215,10 +231,29 @@ module Saboteur
215
231
  # Paths of saboteur's own output, relative to the root, that do not count as source changes.
216
232
  def output_paths
217
233
  prefix = File.join(File.realpath(root), "")
218
- %w[cache_dir json_report html_report score_history].filter_map { |key| path(key) }
219
- .map { |file| File.expand_path(file) }
220
- .select { |file| file.start_with?(prefix) }
221
- .map { |file| file.delete_prefix(prefix) }
234
+ OUTPUT_KEYS.filter_map { |key| path(key) }
235
+ .select { |file| file.start_with?(prefix) }
236
+ .map { |file| file.delete_prefix(prefix) }
237
+ end
238
+
239
+ # Tool-owned paths, relative to the root, that never count as
240
+ # working-tree changes (#638): the run's output, plus an in-repo
241
+ # BUNDLE_GEMFILE sidecar and its lock - the documented trial setup
242
+ # should not read as a dirty tree. Path-exact, never a glob, so a
243
+ # real untracked Gemfile.lock still counts.
244
+ def ignored_tree_paths(gemfile: ENV.fetch("BUNDLE_GEMFILE", nil))
245
+ prefix = File.join(File.realpath(root), "")
246
+ sidecar = gemfile && File.expand_path(gemfile, prefix)
247
+ begin
248
+ sidecar &&= File.join(File.realpath(File.dirname(sidecar)), File.basename(sidecar))
249
+ rescue SystemCallError
250
+ # The sidecar's directory does not exist; keep the expanded spelling.
251
+ end
252
+ paths = output_paths
253
+ if sidecar&.start_with?(prefix) && sidecar != File.join(prefix, "Gemfile")
254
+ paths.concat([sidecar, DirtyTreeWarning.lockfile_for(sidecar)].map { |file| file.delete_prefix(prefix) })
255
+ end
256
+ paths
222
257
  end
223
258
 
224
259
  private
@@ -239,11 +274,28 @@ module Saboteur
239
274
  FLOAT_KEYS.each { |key| @values[key] = Float(@values[key]) }
240
275
  LIST_KEYS.each { |key| @values[key] = Array(@values[key]).map(&:to_s) }
241
276
  NILABLE_FLOAT_KEYS.each { |key| @values[key] = Float(@values[key]) unless @values[key].nil? }
242
- @values["test_selection"] = @values["test_selection"].to_s
277
+ STRING_KEYS.each { |key| @values[key] = @values[key].to_s }
243
278
  rescue ArgumentError, TypeError => e
244
279
  raise ConfigError, "invalid config value: #{e.message}"
245
280
  end
246
281
 
282
+ def validate_partial_coverage!
283
+ return if %w[off shadow use].include?(self["partial_coverage"])
284
+
285
+ raise ConfigError, "partial_coverage must be one of: off, shadow, use"
286
+ end
287
+
288
+ def validate_coverage_settings!
289
+ validate_partial_coverage!
290
+ validate_coverage_recorder!
291
+ end
292
+
293
+ def validate_coverage_recorder!
294
+ return if %w[coverage trace].include?(self["coverage_recorder"])
295
+
296
+ raise ConfigError, "coverage_recorder must be one of: coverage, trace"
297
+ end
298
+
247
299
  def validate_mutant_limit!
248
300
  raise ConfigError, "max_mutants_per_method must be > 0" unless self["max_mutants_per_method"].positive?
249
301
  end
@@ -252,6 +304,7 @@ module Saboteur
252
304
  def validate_baseline!
253
305
  raise ConfigError, "seed must be >= 0" if self["seed"].negative?
254
306
  raise ConfigError, "baseline_passes must be >= 1" if self["baseline_passes"] < 1
307
+ raise ConfigError, "coverage_parts must be >= 1" if self["coverage_parts"] < 1
255
308
  raise ConfigError, "max_duration must be > 0" if self["max_duration"] && !self["max_duration"].positive?
256
309
  end
257
310
 
@@ -270,6 +323,7 @@ module Saboteur
270
323
  def validate_modes!
271
324
  validate_fail_fast!
272
325
  validate_matrix!
326
+ validate_resume!
273
327
  validate_test_selection!
274
328
  validate_gates!
275
329
  validate_score_gates!
@@ -327,6 +381,17 @@ module Saboteur
327
381
  raise ConfigError, "matrix cannot be combined with incremental: reused verdicts list only the first killer"
328
382
  end
329
383
 
384
+ # --resume DIR keeps verdicts in DIR for a later run (#471); it reuses
385
+ # them the way --incremental does, so the same limits apply.
386
+ def validate_resume!
387
+ resume = self["resume"]
388
+ return if resume.nil?
389
+ raise ConfigError, "resume must be a non-blank directory path" unless resume.is_a?(String) && !resume.strip.empty?
390
+ return unless self["matrix"] || !self["test_command"].to_s.empty?
391
+
392
+ raise ConfigError, "resume cannot be combined with matrix or test_command: reused verdicts are not valid there"
393
+ end
394
+
330
395
  def validate_threshold!
331
396
  threshold = self["threshold"]
332
397
  return if threshold.nil? || THRESHOLD_RANGE.cover?(threshold)
@@ -376,6 +441,7 @@ module Saboteur
376
441
  raise ConfigError, "flaky_check must be one of: off, survivors, decisive"
377
442
  end
378
443
 
444
+ validate_coverage_settings!
379
445
  validate_mutant_limit!
380
446
  validate_threshold!
381
447
  validate_modes!
@@ -3,6 +3,7 @@
3
3
 
4
4
  require "prism"
5
5
  require_relative "subject"
6
+ require_relative "deep_source"
6
7
  require_relative "constant_subject"
7
8
 
8
9
  module Saboteur
@@ -47,6 +48,8 @@ module Saboteur
47
48
  result = Prism.parse(source)
48
49
  return Parsed.new(subjects: [], unsupported: [], errors: result.errors.map(&:message)) if result.failure?
49
50
 
51
+ DeepSource.check!(result.value, file)
52
+
50
53
  ctx = { file: file, source: source, frozen: frozen_literals?(result) }
51
54
  out = { subjects: [], unsupported: [], ambiguous: [] }
52
55
  walk(result.value, [], out, **ctx)
@@ -67,23 +67,24 @@ module Saboteur
67
67
 
68
68
  numbered = key.is_a?(Regexp) && spec.match?(/\\\d/)
69
69
  named = key.is_a?(Regexp) && spec.match?(/%[-0-9]*[<{][^>}]+[>}]|\\k</)
70
- if numbered && named
71
- raise ConfigError, "#{path}: #{key.inspect} maps to #{spec.inspect}; " \
72
- 'regex map specs support \1 backreferences only'
73
- end
74
- return if numbered # backreference filled in at match time
75
-
70
+ # Existence is deliberately not checked here: a stale or mistyped
71
+ # entry warns at guess time and the naming convention takes over,
72
+ # instead of the whole run dying at boot on one bad line (#640).
73
+ # The root check comes first: no template, backreference or named,
74
+ # may skip it (#712).
76
75
  expanded = File.expand_path(spec, root)
77
76
  unless expanded.start_with?("#{root}/")
78
77
  raise ConfigError, "#{path}: #{key.inspect} maps to #{spec.inspect}, which is outside the project root"
79
78
  end
80
- return if Dir.glob(expanded).any?
81
-
82
- if named
79
+ if numbered && named
83
80
  raise ConfigError, "#{path}: #{key.inspect} maps to #{spec.inspect}; " \
84
81
  'regex map specs support \1 backreferences only'
85
82
  end
86
- raise ConfigError, "#{path}: #{key.inspect} maps to #{spec.inspect}, which matches no file"
83
+ return if numbered # backreference filled in at match time
84
+ return unless named
85
+
86
+ raise ConfigError, "#{path}: #{key.inspect} maps to #{spec.inspect}; " \
87
+ 'regex map specs support \1 backreferences only'
87
88
  end
88
89
 
89
90
  private_class_method :validate_entry, :validate_spec
@@ -160,14 +161,14 @@ module Saboteur
160
161
  return :excluded if matches.all? { |entry, _| entry.specs.empty? }
161
162
 
162
163
  matches.reject { |entry, _| entry.specs.empty? }
163
- .flat_map { |entry, match| resolve(entry, match) }
164
+ .flat_map { |entry, match| resolve(entry, match, file) }
164
165
  end
165
166
 
166
167
  def specificity(key)
167
168
  key.count("^*?[]{}\\")
168
169
  end
169
170
 
170
- def resolve(entry, match)
171
+ def resolve(entry, match, file = nil)
171
172
  return :excluded if entry.specs.empty?
172
173
 
173
174
  specs = entry.specs
@@ -176,6 +177,13 @@ module Saboteur
176
177
  # that is never a real spec path, so drop it here rather than letting
177
178
  # a junk entry reach the run.
178
179
  specs = specs.map { |spec| spec.gsub(/\\(\d+)/) { match[::Regexp.last_match(1).to_i].to_s } }.reject(&:empty?)
180
+ # A capture can carry "../" segments the template check never saw:
181
+ # refuse a filled spec that escapes the root (#712).
182
+ escaped = specs.reject { |spec| File.expand_path(spec, @root).start_with?("#{@root}/") }
183
+ if escaped.any?
184
+ raise ConfigError,
185
+ "#{entry.key.inspect} maps #{file} to #{escaped.first.inspect}, which is outside the project root"
186
+ end
179
187
  end
180
188
  specs
181
189
  end
@@ -0,0 +1,49 @@
1
+ # SPDX-License-Identifier: MIT
2
+ # frozen_string_literal: true
3
+
4
+ module Saboteur
5
+ # A file nested deeper than the walkers can safely recurse (#719).
6
+ class DeepSourceError < Error
7
+ def initialize(file, limit)
8
+ super("#{file} nests deeper than #{limit} AST levels; saboteur cannot walk it safely")
9
+ end
10
+ end
11
+
12
+ # The AST walkers (SubjectFinder, ConstantSubjectFinder, AttributeFinder
13
+ # and the operator visits) recurse one stack frame per nesting level and
14
+ # overflow the Ruby stack around a thousand nested calls, while Prism
15
+ # still parses to about five thousand (#719). Measuring depth iteratively
16
+ # before any walk turns that crash into a clean refusal: the CLI boundary
17
+ # prints one line naming the file and exits 3.
18
+ module DeepSource
19
+ # Two AST levels per nested call (`a(` adds a CallNode and an
20
+ # ArgumentsNode). The finders walk a file from its root and survive a
21
+ # thousand nested calls, so their limit sits just above that. The
22
+ # operator visits recurse deeper per level (method_swap overflows a
23
+ # bare stack around 900 nested calls, earlier under the engine), so a
24
+ # subject's body refuses earlier. Both margins stay deterministic:
25
+ # a clean refusal instead of a SystemStackError that depends on how
26
+ # deep the caller's stack already is.
27
+ MAX_AST_DEPTH = 2200
28
+ MAX_MUTATION_DEPTH = 1200
29
+
30
+ def self.check!(program, file, limit: MAX_AST_DEPTH)
31
+ raise DeepSourceError.new(file, limit) if depth(program) > limit
32
+ end
33
+
34
+ # Iterative on purpose: a recursive measure would crash on exactly the
35
+ # files it exists to catch (#719, same shape as #711).
36
+ def self.depth(node)
37
+ max = 0
38
+ work = [[node, 1]]
39
+ until work.empty?
40
+ current, level = work.pop
41
+ next if current.nil?
42
+
43
+ max = level if level > max
44
+ current.compact_child_nodes.each { |child| work << [child, level + 1] }
45
+ end
46
+ max
47
+ end
48
+ end
49
+ end
@@ -26,7 +26,7 @@ module Saboteur
26
26
  # unknown tree. The root is physical (realpath), like the paths
27
27
  # Config#included_files returns.
28
28
  def self.for(config, config_path:, gemfile: ENV.fetch("BUNDLE_GEMFILE", nil))
29
- state = WorkingTree.new(config.root, ignored: config.output_paths).state
29
+ state = WorkingTree.new(config.root, ignored: config.ignored_tree_paths(gemfile: gemfile)).state
30
30
  root = File.realpath(config.root)
31
31
  new(state, included: relative(config.included_files, root), include: config["include"],
32
32
  exclude: config["exclude"], root: root,
@@ -19,6 +19,7 @@ module Saboteur
19
19
  def call
20
20
  Engine::Boot.refuse_recursion!
21
21
  check_runtime
22
+ check_environment
22
23
  check_coverage
23
24
  check_boot_files
24
25
  check_custom_map
@@ -55,6 +56,11 @@ module Saboteur
55
56
  raise Error, "target boot failed: #{e.class}: #{e.message}; check require files, hooks and rspec_args"
56
57
  end
57
58
 
59
+ def check_environment
60
+ hints = EnvironmentHints.all(@config.root)
61
+ @out.puts "Environment: #{hints.empty? ? 'no bundler/appraisal problems detected' : hints.join('; ')}"
62
+ end
63
+
58
64
  def check_runtime
59
65
  @out.puts "Ruby: #{RUBY_DESCRIPTION}"
60
66
  unless Gem::Version.new(RUBY_VERSION) >= Gem::Version.new("3.3") && Process.respond_to?(:fork)
@@ -5,6 +5,7 @@ require "json"
5
5
  require "fileutils"
6
6
  require_relative "../text"
7
7
  require_relative "../frameworks/sqlite_memory_database"
8
+ require_relative "baseline_parts"
8
9
 
9
10
  module Saboteur
10
11
  class Engine
@@ -12,6 +13,8 @@ module Saboteur
12
13
  # or mutation source is used. This catches observed instability, not all
13
14
  # possible flaky behaviour.
14
15
  class BaselineGuard
16
+ include BaselineParts
17
+
15
18
  # How many unchanged-suite checks actually completed and validated:
16
19
  # the number of order comparisons a run can claim is one less.
17
20
  attr_reader :completed_passes
@@ -131,10 +134,10 @@ module Saboteur
131
134
  return if mutants_get >= pass_elapsed
132
135
 
133
136
  verdict = mutants_get <= 0 ? "exceeds" : "barely fits"
134
- @diagnostics.warn("projected baseline+coverage ~#{projected.round}s #{verdict} " \
137
+ @diagnostics.warn("raise --max-duration, pass fewer --baseline-passes, or narrow the scope\n " \
138
+ "projected baseline+coverage ~#{projected.round}s #{verdict} " \
135
139
  "--max-duration #{budget.round}s; mutants would get " \
136
- "~#{format('%g', [mutants_get, 0].max.round(1))}s. Raise --max-duration, pass fewer " \
137
- "--baseline-passes, or narrow the scope")
140
+ "~#{format('%g', [mutants_get, 0].max.round(1))}s")
138
141
  end
139
142
 
140
143
  # An admitted pass still gets only the time left in the run budget:
@@ -176,6 +179,9 @@ module Saboteur
176
179
  end
177
180
 
178
181
  def check(number)
182
+ parts = baseline_parts
183
+ return check_parts(number, parts) if parts.size > 1
184
+
179
185
  Execution::ForkReadiness.prepare_heap
180
186
  heartbeat = @diagnostics.progress_heartbeat("baseline check #{number}").watch(log_path(number))
181
187
  admitted = phase_timeout
@@ -183,6 +189,11 @@ module Saboteur
183
189
  label: "baseline child died during setup/run") do |writer|
184
190
  record(writer, number)
185
191
  end
192
+ settled(number, result, admitted)
193
+ end
194
+
195
+ def settled(number, result, admitted, part: nil)
196
+ @part = part
186
197
  refuse_budget_stop(number, admitted) if budget_stopped?(result, admitted)
187
198
  refuse(number, stopped_reason(result)) if result.stopped_by
188
199
  # Before the payload checks: a child killed by a signal (the OOM
@@ -221,15 +232,16 @@ module Saboteur
221
232
  data["executed"].is_a?(Array) && data["executed"].all?(String)
222
233
  end
223
234
 
224
- def record(writer, number)
225
- path = log_path(number)
235
+ def record(writer, number, files: nil, part: nil)
236
+ path = log_path(number, part)
226
237
  FileUtils.mkdir_p(File.dirname(path))
227
238
  $stdout.reopen(path, "w")
228
239
  $stdout.sync = true
229
240
  $stderr.reopen($stdout)
230
241
  @hooks.run(:after_fork)
231
242
  Kernel.srand(@config["seed"])
232
- result = @integration.run_baseline(seed: @config["seed"])
243
+ seed = @config["seed"]
244
+ result = files ? @integration.run_baseline(seed: seed, files: files) : @integration.run_baseline(seed: seed)
233
245
  payload = { passed: result.passed, failed: result.failed_ids, executed: result.executed_ids }
234
246
  payload[:hint] = Frameworks::SqliteMemoryDatabase.fork_loss_hint unless result.passed
235
247
  writer.write(JSON.generate(payload))
@@ -260,29 +272,54 @@ module Saboteur
260
272
  return if errors.empty? || !spec_helper_requires_minitest?
261
273
 
262
274
  symptom = (lines.find { |line| line.include?("undefined method") } || errors.first).strip
263
- @diagnostics.warn("the baseline failure looks like a Minitest suite running under RSpec, not a " \
275
+ @diagnostics.warn("run saboteur on an RSpec suite: it is RSpec-only, so a Minitest suite cannot be " \
276
+ "mutation tested with it\n " \
277
+ "the baseline failure looks like a Minitest suite running under RSpec, not a " \
264
278
  "broken RSpec suite: the log shows #{errors.size} " \
265
279
  "NoMethodError#{'s' if errors.size > 1} (for example: #{symptom.slice(0, EXCERPT_WIDTH)}) " \
266
- "and spec/spec_helper.rb requires minitest. saboteur is RSpec-only, so a " \
267
- "Minitest suite cannot be mutation tested with it.")
280
+ "and spec/spec_helper.rb requires minitest")
268
281
  end
269
282
 
270
283
  # The framework check behind the warning: spec_helper is the
271
- # suite's own declaration of what it runs on.
284
+ # suite's own declaration of what it runs on. Parsed, not grepped: a
285
+ # regex also fires on commented-out or heredoc "require" lines for a
286
+ # framework the project does not use (#708).
272
287
  def spec_helper_requires_minitest?
273
- File.read(File.join(@config.root, "spec", "spec_helper.rb")).match?(/^\s*require\s+["']minitest/)
288
+ require_calls(Prism.parse_file(File.join(@config.root, "spec", "spec_helper.rb")).value)
289
+ .any? { |name| name == "minitest" || name.start_with?("minitest/") }
274
290
  rescue SystemCallError
275
291
  false
276
292
  end
277
293
 
294
+ # Iterative like RefinementActivations.collect: a deeply nested
295
+ # spec_helper must never raise SystemStackError out of a warning
296
+ # heuristic (same class as #711).
297
+ def require_calls(node)
298
+ found = []
299
+ work = [node]
300
+ until work.empty?
301
+ current = work.pop
302
+ next unless current.is_a?(Prism::Node)
303
+
304
+ if current.is_a?(Prism::CallNode) && current.name == :require && current.receiver.nil?
305
+ argument = current.arguments&.arguments&.first
306
+ found << argument.unescaped if argument.is_a?(Prism::StringNode)
307
+ end
308
+ work.concat(current.compact_child_nodes)
309
+ end
310
+ found
311
+ end
312
+
278
313
  # A child that recognises its own failure shape (an empty sqlite
279
314
  # :memory: database after the fork, #387) says how to fix it.
280
315
  def with_hint(reason, data)
281
316
  data["hint"] ? "#{reason}. #{data['hint']}" : reason
282
317
  end
283
318
 
284
- def log_path(number)
285
- File.join(@config.path("cache_dir"), "baseline-#{number}.log")
319
+ # A split check keeps one log per part: baseline-1-part-2.log.
320
+ def log_path(number, part = @part)
321
+ name = part ? "baseline-#{number}-part-#{part + 1}.log" : "baseline-#{number}.log"
322
+ File.join(@config.path("cache_dir"), name)
286
323
  end
287
324
 
288
325
  # Leftover children are a pass-with-leaks verdict, not a suite
@@ -0,0 +1,51 @@
1
+ # SPDX-License-Identifier: MIT
2
+ # frozen_string_literal: true
3
+
4
+ require_relative "suite_partition"
5
+
6
+ module Saboteur
7
+ class Engine
8
+ # A baseline check split like the coverage build (--coverage-parts N):
9
+ # the same disjoint spec-file parts run as children at once, each over
10
+ # its own files. Uses the guard's @config, @integration, @supervisor,
11
+ # @diagnostics, @part and settled/phase_timeout/log_path.
12
+ #
13
+ # The check's contract is unchanged. Every part must pass, and the
14
+ # examples the parts executed together must be exactly the loaded
15
+ # suite's (the guard compares them before it accepts the pass). What a
16
+ # split no longer shows is a failure that needs the whole suite in one
17
+ # process.
18
+ module BaselineParts
19
+ private
20
+
21
+ # The parts the coverage build would use. One part (or none, for an
22
+ # adapter that lists no spec files) means the ordinary single check.
23
+ def baseline_parts
24
+ Engine::SuitePartition.parts(@integration.test_files, @config["coverage_parts"])
25
+ end
26
+
27
+ def check_parts(number, parts)
28
+ Execution::ForkReadiness.prepare_heap
29
+ heartbeat = @diagnostics.progress_heartbeat("baseline check #{number} (#{parts.size} parts)")
30
+ .watch(log_path(number, 0))
31
+ admitted = phase_timeout
32
+ results = @supervisor.run_many(parts.size, timeout: admitted, on_tick: heartbeat.method(:tick),
33
+ label: "baseline child died during setup/run") do |writer, slot|
34
+ ENV["TEST_ENV_NUMBER"] = slot.zero? ? "" : (slot + 1).to_s
35
+ record(writer, number, files: parts[slot], part: slot)
36
+ end
37
+ joined(results.each_with_index.map { |result, slot| settled(number, result, admitted, part: slot) })
38
+ end
39
+
40
+ # What the guard's checks read: the first failing part's own result
41
+ # (its failures and hint; @part names its log), or a pass carrying
42
+ # every example the parts executed.
43
+ def joined(datas)
44
+ @part = datas.index { |data| !(data["passed"] && data["failed"].empty?) }
45
+ return datas[@part] if @part
46
+
47
+ { "passed" => true, "failed" => [], "executed" => datas.flat_map { |data| data["executed"] } }
48
+ end
49
+ end
50
+ end
51
+ end
@@ -71,6 +71,10 @@ module Saboteur
71
71
  Integration::RSpec.new(@config).require_framework if @config["integration"] == "rspec"
72
72
  @config["require"].each { |file| require File.expand_path(file, @root) }
73
73
  integration = Integration.fetch(@config["integration"]).new(@config)
74
+ # Running outside the bundle or beside an Appraisal gemfile set
75
+ # loads the wrong gems; say so before the spec load fails in
76
+ # confusing ways (#598, #609). Advisory only.
77
+ EnvironmentHints.all(@root).each { |hint| warn "saboteur: warning: #{hint}" }
74
78
  integration.setup
75
79
  rails&.after_boot
76
80
  @hooks.run(:after_boot)
@@ -8,11 +8,12 @@ module Saboteur
8
8
  class CandidateSelection
9
9
  attr_reader :skipped_subjects, :selected, :unscoped_digests
10
10
 
11
- def initialize(candidates:, map:, limit:, diagnostics:)
11
+ def initialize(candidates:, map:, limit:, diagnostics:, root: nil)
12
12
  @candidates = candidates
13
13
  @map = map
14
14
  @limit = limit
15
15
  @diagnostics = diagnostics
16
+ @root = root
16
17
  @skipped_subjects = {}
17
18
  @selected = {}
18
19
  @unscoped_digests = {}
@@ -29,6 +30,7 @@ module Saboteur
29
30
  return @selected[subject] = []
30
31
  end
31
32
 
33
+ warn_boot_once(subject)
32
34
  list = @candidates.for(subject)
33
35
  # The full pre-scope digests let IgnoreList#apply tell "mutation gone"
34
36
  # from "mutation scoped out of this run" (#284).
@@ -48,18 +50,26 @@ module Saboteur
48
50
 
49
51
  private
50
52
 
53
+ # A note, not a verdict (#575): every mutant still runs and scores.
54
+ def warn_boot_once(subject)
55
+ return unless @map.boot_once?(subject)
56
+
57
+ @diagnostics.warn("#{subject.id}: verify a survivor here by mutating the file on disk\n " \
58
+ "covered by every test and run at boot: mutants apply after boot, so a survivor " \
59
+ "can be a boot-once effect the suite cannot re-observe (#575)")
60
+ end
61
+
51
62
  # A scoped no-coverage skip names the specs the run loaded: coverage
52
63
  # living in a sibling spec file is otherwise invisible, and the false
53
64
  # read is "untestable" when the truth is "not loaded" (#636). Names
54
65
  # at most five, like the other selection notes.
55
66
  def skip_announcement(subject, reason)
56
- line = "#{subject.id}: skipped (#{reason}); its change is not verified by this run"
67
+ line = "#{subject.id} (#{subject.location(@root)}): skipped (#{reason}); its change is not verified by this run"
57
68
  return line unless reason == "no coverage"
58
69
 
59
70
  loaded = @map.tests.map { |id| id.split("[").first.delete_prefix("./") }.uniq.sort
60
- more = loaded.size > 5 ? ", ..." : ""
61
- names = loaded.empty? ? "none" : loaded.first(5).join(", ")
62
- "#{line} - loaded specs: #{names}#{more}; " \
71
+ names = loaded.empty? ? "none" : Text.name_list(loaded)
72
+ "#{line} - loaded specs: #{names}; " \
63
73
  "other spec files may cover it, pass --test to load them"
64
74
  end
65
75
 
@@ -109,6 +109,9 @@ module Saboteur
109
109
  end
110
110
 
111
111
  def proposals(source, subject)
112
+ # The operator visits recurse over the subject's body; refuse a
113
+ # body nested too deeply instead of crashing the run (#719).
114
+ DeepSource.check!(subject.node, subject.file, limit: DeepSource::MAX_MUTATION_DEPTH)
112
115
  proposals = source.mutations_for(subject)
113
116
  if proposals.empty?
114
117
  @diagnostics.record_generation(subject, source: source_name(source), reason: "empty_generation")
@@ -142,7 +145,7 @@ module Saboteur
142
145
 
143
146
  [mutation, text]
144
147
  rescue ArgumentError => e
145
- @diagnostics.warn("#{subject.id}: dropped #{mutation.source} mutation: #{e.message}")
148
+ @diagnostics.warn("#{subject.id}: dropped #{mutation.source} mutation\n #{e.message}")
146
149
  record_drop(subject, mutation, "invalid_edit")
147
150
  end
148
151