saboteur 0.8.1 → 0.9.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +100 -0
- data/README.md +10 -0
- data/exe/saboteur +6 -2
- data/lib/saboteur/attribute_finder.rb +3 -0
- data/lib/saboteur/baseline/run_evidence.rb +2 -1
- data/lib/saboteur/cli/agent_provenance.rb +59 -0
- data/lib/saboteur/cli/dirty_tree_notice.rb +28 -0
- data/lib/saboteur/cli/doctor_command.rb +41 -0
- data/lib/saboteur/cli/error_cause.rb +40 -0
- data/lib/saboteur/cli/execution_options.rb +46 -0
- data/lib/saboteur/cli/exit_status.rb +12 -14
- data/lib/saboteur/cli/explicit_file_options.rb +21 -0
- data/lib/saboteur/cli/explicit_sources.rb +46 -0
- data/lib/saboteur/cli/failure_output.rb +35 -0
- data/lib/saboteur/cli/help_command.rb +23 -0
- data/lib/saboteur/cli/matrix_option.rb +17 -0
- data/lib/saboteur/cli/output_options.rb +27 -0
- data/lib/saboteur/cli/preview_command.rb +26 -0
- data/lib/saboteur/cli/run_budget_options.rb +3 -0
- data/lib/saboteur/cli/run_command.rb +33 -0
- data/lib/saboteur/cli/run_output.rb +38 -0
- data/lib/saboteur/cli/run_reporting.rb +6 -5
- data/lib/saboteur/cli/run_validation.rb +54 -0
- data/lib/saboteur/cli.rb +29 -307
- data/lib/saboteur/config.rb +72 -6
- data/lib/saboteur/constant_subject_finder.rb +3 -0
- data/lib/saboteur/custom_map.rb +20 -12
- data/lib/saboteur/deep_source.rb +49 -0
- data/lib/saboteur/dirty_tree_warning.rb +1 -1
- data/lib/saboteur/doctor.rb +6 -0
- data/lib/saboteur/engine/baseline_guard.rb +50 -13
- data/lib/saboteur/engine/baseline_parts.rb +51 -0
- data/lib/saboteur/engine/boot.rb +4 -0
- data/lib/saboteur/engine/candidate_selection.rb +15 -5
- data/lib/saboteur/engine/candidates.rb +4 -1
- data/lib/saboteur/engine/coverage_inputs_snapshot.rb +146 -0
- data/lib/saboteur/engine/coverage_miss_note.rb +86 -0
- data/lib/saboteur/engine/coverage_partial.rb +48 -0
- data/lib/saboteur/engine/coverage_parts.rb +50 -0
- data/lib/saboteur/engine/coverage_shadow.rb +92 -0
- data/lib/saboteur/engine/coverage_store.rb +275 -22
- data/lib/saboteur/engine/diagnostics.rb +23 -6
- data/lib/saboteur/engine/distribution.rb +2 -2
- data/lib/saboteur/engine/gated_run.rb +1 -0
- data/lib/saboteur/engine/load_coverage.rb +54 -8
- data/lib/saboteur/engine/partial_invalidation.rb +115 -0
- data/lib/saboteur/engine/partial_merge.rb +40 -0
- data/lib/saboteur/engine/pipelines.rb +7 -7
- data/lib/saboteur/engine/reference_specs.rb +71 -0
- data/lib/saboteur/engine/result_cache.rb +68 -5
- data/lib/saboteur/engine/resume_manifest.rb +110 -0
- data/lib/saboteur/engine/run_budget.rb +5 -5
- data/lib/saboteur/engine/run_fingerprint.rb +2 -1
- data/lib/saboteur/engine/sandbox_policy.rb +5 -2
- data/lib/saboteur/engine/scheduler.rb +1 -1
- data/lib/saboteur/engine/selection_announcement.rb +1 -0
- data/lib/saboteur/engine/shadow_comparison.rb +121 -0
- data/lib/saboteur/engine/shadow_run.rb +38 -0
- data/lib/saboteur/engine/subject_selection.rb +33 -37
- data/lib/saboteur/engine/suite_partition.rb +22 -0
- data/lib/saboteur/engine/test_guess.rb +82 -18
- data/lib/saboteur/engine/unchanged_syntax.rb +10 -2
- data/lib/saboteur/engine.rb +41 -9
- data/lib/saboteur/environment_hints.rb +55 -0
- data/lib/saboteur/execution/bundle_gemfile_pin.rb +72 -0
- data/lib/saboteur/execution/external_runner.rb +105 -1
- data/lib/saboteur/execution/gemfile_path_sources.rb +166 -0
- data/lib/saboteur/execution/sandbox.rb +30 -17
- data/lib/saboteur/execution/supervisor.rb +40 -7
- data/lib/saboteur/execution/supervisor_batch.rb +105 -0
- data/lib/saboteur/execution/worker_pool.rb +48 -4
- data/lib/saboteur/frameworks/rails.rb +2 -5
- data/lib/saboteur/gates.rb +121 -28
- data/lib/saboteur/install.rb +6 -1
- data/lib/saboteur/integration/base.rb +19 -2
- data/lib/saboteur/integration/rspec.rb +46 -7
- data/lib/saboteur/line_hits.rb +81 -0
- data/lib/saboteur/method_calls.rb +91 -0
- data/lib/saboteur/mutation_source/builtin.rb +32 -22
- data/lib/saboteur/mutation_source/condition_negation.rb +3 -12
- data/lib/saboteur/mutation_source/jump_value.rb +59 -0
- data/lib/saboteur/mutation_source/literal_replacement.rb +6 -23
- data/lib/saboteur/mutation_source/operator_examples.rb +5 -0
- data/lib/saboteur/mutation_source/operator_registry.rb +5 -0
- data/lib/saboteur/mutation_source/operator_visit.rb +7 -1
- data/lib/saboteur/mutation_source/rescue_removal.rb +69 -0
- data/lib/saboteur/mutation_source/unary_removal.rb +47 -0
- data/lib/saboteur/mutation_source/when_removal.rb +51 -0
- data/lib/saboteur/preview.rb +9 -1
- data/lib/saboteur/refinement_activations.rb +22 -10
- data/lib/saboteur/reporter/agent.rb +2 -2
- data/lib/saboteur/reporter/agent_incomplete.rb +11 -0
- data/lib/saboteur/reporter/agent_summary.rb +16 -10
- data/lib/saboteur/reporter/review_comments.rb +151 -0
- data/lib/saboteur/reporter/safe_out.rb +32 -0
- data/lib/saboteur/reporter/stryker.rb +8 -6
- data/lib/saboteur/reporter/survivor_classification.rb +41 -3
- data/lib/saboteur/reporter/survivor_hints.rb +5 -0
- data/lib/saboteur/reporter/terminal.rb +29 -4
- data/lib/saboteur/result.rb +14 -4
- data/lib/saboteur/selection/coverage_map.rb +136 -10
- data/lib/saboteur/selection/external_test_map.rb +4 -0
- data/lib/saboteur/selection/git_diff.rb +24 -2
- data/lib/saboteur/selection/layout_only_lines.rb +45 -1
- data/lib/saboteur/selection/recorder.rb +52 -22
- data/lib/saboteur/selection/trace_recorder.rb +83 -0
- data/lib/saboteur/selection/untracked_changes.rb +1 -2
- data/lib/saboteur/subject.rb +16 -0
- data/lib/saboteur/subject_finder.rb +91 -6
- data/lib/saboteur/text.rb +31 -0
- data/lib/saboteur/version.rb +1 -1
- data/lib/saboteur/working_tree.rb +7 -2
- data/lib/saboteur.rb +8 -0
- metadata +44 -2
data/lib/saboteur/config.rb
CHANGED
|
@@ -18,6 +18,10 @@ module Saboteur
|
|
|
18
18
|
LEGACY_IGNORE_FILE = ".saboteur-ignore.yml"
|
|
19
19
|
IGNORE_FILE = "config/saboteur_ignore.rb"
|
|
20
20
|
|
|
21
|
+
# Settings that say where saboteur writes, never what it mutates: left
|
|
22
|
+
# out of the run fingerprint, baseline evidence and score-history scope.
|
|
23
|
+
OUTPUT_KEYS = %w[cache_dir json_report html_report score_history resume].freeze
|
|
24
|
+
|
|
21
25
|
DEFAULTS = {
|
|
22
26
|
"integration" => "rspec",
|
|
23
27
|
"require" => [],
|
|
@@ -69,7 +73,15 @@ module Saboteur
|
|
|
69
73
|
"subjects" => [],
|
|
70
74
|
"since" => nil,
|
|
71
75
|
"cache" => true,
|
|
76
|
+
# Experimental (#46): "shadow" also builds a partial coverage map after a
|
|
77
|
+
# cache miss and compares it with the full map, which the run still uses.
|
|
78
|
+
# "use" uses the partial map when it is safe and builds the full map otherwise.
|
|
79
|
+
"partial_coverage" => "off",
|
|
80
|
+
# coverage (default) reads Coverage around every test; trace gets lines and
|
|
81
|
+
# methods from a TracePoint and reads branches once per spec file (#725).
|
|
82
|
+
"coverage_recorder" => "coverage",
|
|
72
83
|
"incremental" => false,
|
|
84
|
+
"resume" => nil,
|
|
73
85
|
"shard" => nil,
|
|
74
86
|
"allow_focus" => false,
|
|
75
87
|
"fail_fast" => false,
|
|
@@ -79,6 +91,8 @@ module Saboteur
|
|
|
79
91
|
"matrix" => false,
|
|
80
92
|
"seed" => 1,
|
|
81
93
|
"baseline_passes" => 2,
|
|
94
|
+
# Parallel children for the coverage-map build (#741); 1 keeps the single child.
|
|
95
|
+
"coverage_parts" => 1,
|
|
82
96
|
"max_duration" => nil,
|
|
83
97
|
"max_duration_ok" => false,
|
|
84
98
|
"tests" => [],
|
|
@@ -92,8 +106,10 @@ module Saboteur
|
|
|
92
106
|
"verbose" => false
|
|
93
107
|
}.freeze
|
|
94
108
|
|
|
95
|
-
INTEGER_KEYS = %w[jobs max_memory_mb max_processes max_mutants_per_method seed baseline_passes
|
|
109
|
+
INTEGER_KEYS = %w[jobs max_memory_mb max_processes max_mutants_per_method seed baseline_passes
|
|
110
|
+
coverage_parts].freeze
|
|
96
111
|
FLOAT_KEYS = %w[timeout_factor timeout_min baseline_timeout coverage_timeout].freeze
|
|
112
|
+
STRING_KEYS = %w[test_selection partial_coverage coverage_recorder].freeze
|
|
97
113
|
NILABLE_FLOAT_KEYS = %w[threshold max_duration].freeze
|
|
98
114
|
THRESHOLD_RANGE = (0.0..100.0)
|
|
99
115
|
|
|
@@ -215,10 +231,29 @@ module Saboteur
|
|
|
215
231
|
# Paths of saboteur's own output, relative to the root, that do not count as source changes.
|
|
216
232
|
def output_paths
|
|
217
233
|
prefix = File.join(File.realpath(root), "")
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
234
|
+
OUTPUT_KEYS.filter_map { |key| path(key) }
|
|
235
|
+
.select { |file| file.start_with?(prefix) }
|
|
236
|
+
.map { |file| file.delete_prefix(prefix) }
|
|
237
|
+
end
|
|
238
|
+
|
|
239
|
+
# Tool-owned paths, relative to the root, that never count as
|
|
240
|
+
# working-tree changes (#638): the run's output, plus an in-repo
|
|
241
|
+
# BUNDLE_GEMFILE sidecar and its lock - the documented trial setup
|
|
242
|
+
# should not read as a dirty tree. Path-exact, never a glob, so a
|
|
243
|
+
# real untracked Gemfile.lock still counts.
|
|
244
|
+
def ignored_tree_paths(gemfile: ENV.fetch("BUNDLE_GEMFILE", nil))
|
|
245
|
+
prefix = File.join(File.realpath(root), "")
|
|
246
|
+
sidecar = gemfile && File.expand_path(gemfile, prefix)
|
|
247
|
+
begin
|
|
248
|
+
sidecar &&= File.join(File.realpath(File.dirname(sidecar)), File.basename(sidecar))
|
|
249
|
+
rescue SystemCallError
|
|
250
|
+
# The sidecar's directory does not exist; keep the expanded spelling.
|
|
251
|
+
end
|
|
252
|
+
paths = output_paths
|
|
253
|
+
if sidecar&.start_with?(prefix) && sidecar != File.join(prefix, "Gemfile")
|
|
254
|
+
paths.concat([sidecar, DirtyTreeWarning.lockfile_for(sidecar)].map { |file| file.delete_prefix(prefix) })
|
|
255
|
+
end
|
|
256
|
+
paths
|
|
222
257
|
end
|
|
223
258
|
|
|
224
259
|
private
|
|
@@ -239,11 +274,28 @@ module Saboteur
|
|
|
239
274
|
FLOAT_KEYS.each { |key| @values[key] = Float(@values[key]) }
|
|
240
275
|
LIST_KEYS.each { |key| @values[key] = Array(@values[key]).map(&:to_s) }
|
|
241
276
|
NILABLE_FLOAT_KEYS.each { |key| @values[key] = Float(@values[key]) unless @values[key].nil? }
|
|
242
|
-
@values[
|
|
277
|
+
STRING_KEYS.each { |key| @values[key] = @values[key].to_s }
|
|
243
278
|
rescue ArgumentError, TypeError => e
|
|
244
279
|
raise ConfigError, "invalid config value: #{e.message}"
|
|
245
280
|
end
|
|
246
281
|
|
|
282
|
+
def validate_partial_coverage!
|
|
283
|
+
return if %w[off shadow use].include?(self["partial_coverage"])
|
|
284
|
+
|
|
285
|
+
raise ConfigError, "partial_coverage must be one of: off, shadow, use"
|
|
286
|
+
end
|
|
287
|
+
|
|
288
|
+
def validate_coverage_settings!
|
|
289
|
+
validate_partial_coverage!
|
|
290
|
+
validate_coverage_recorder!
|
|
291
|
+
end
|
|
292
|
+
|
|
293
|
+
def validate_coverage_recorder!
|
|
294
|
+
return if %w[coverage trace].include?(self["coverage_recorder"])
|
|
295
|
+
|
|
296
|
+
raise ConfigError, "coverage_recorder must be one of: coverage, trace"
|
|
297
|
+
end
|
|
298
|
+
|
|
247
299
|
def validate_mutant_limit!
|
|
248
300
|
raise ConfigError, "max_mutants_per_method must be > 0" unless self["max_mutants_per_method"].positive?
|
|
249
301
|
end
|
|
@@ -252,6 +304,7 @@ module Saboteur
|
|
|
252
304
|
def validate_baseline!
|
|
253
305
|
raise ConfigError, "seed must be >= 0" if self["seed"].negative?
|
|
254
306
|
raise ConfigError, "baseline_passes must be >= 1" if self["baseline_passes"] < 1
|
|
307
|
+
raise ConfigError, "coverage_parts must be >= 1" if self["coverage_parts"] < 1
|
|
255
308
|
raise ConfigError, "max_duration must be > 0" if self["max_duration"] && !self["max_duration"].positive?
|
|
256
309
|
end
|
|
257
310
|
|
|
@@ -270,6 +323,7 @@ module Saboteur
|
|
|
270
323
|
def validate_modes!
|
|
271
324
|
validate_fail_fast!
|
|
272
325
|
validate_matrix!
|
|
326
|
+
validate_resume!
|
|
273
327
|
validate_test_selection!
|
|
274
328
|
validate_gates!
|
|
275
329
|
validate_score_gates!
|
|
@@ -327,6 +381,17 @@ module Saboteur
|
|
|
327
381
|
raise ConfigError, "matrix cannot be combined with incremental: reused verdicts list only the first killer"
|
|
328
382
|
end
|
|
329
383
|
|
|
384
|
+
# --resume DIR keeps verdicts in DIR for a later run (#471); it reuses
|
|
385
|
+
# them the way --incremental does, so the same limits apply.
|
|
386
|
+
def validate_resume!
|
|
387
|
+
resume = self["resume"]
|
|
388
|
+
return if resume.nil?
|
|
389
|
+
raise ConfigError, "resume must be a non-blank directory path" unless resume.is_a?(String) && !resume.strip.empty?
|
|
390
|
+
return unless self["matrix"] || !self["test_command"].to_s.empty?
|
|
391
|
+
|
|
392
|
+
raise ConfigError, "resume cannot be combined with matrix or test_command: reused verdicts are not valid there"
|
|
393
|
+
end
|
|
394
|
+
|
|
330
395
|
def validate_threshold!
|
|
331
396
|
threshold = self["threshold"]
|
|
332
397
|
return if threshold.nil? || THRESHOLD_RANGE.cover?(threshold)
|
|
@@ -376,6 +441,7 @@ module Saboteur
|
|
|
376
441
|
raise ConfigError, "flaky_check must be one of: off, survivors, decisive"
|
|
377
442
|
end
|
|
378
443
|
|
|
444
|
+
validate_coverage_settings!
|
|
379
445
|
validate_mutant_limit!
|
|
380
446
|
validate_threshold!
|
|
381
447
|
validate_modes!
|
|
@@ -3,6 +3,7 @@
|
|
|
3
3
|
|
|
4
4
|
require "prism"
|
|
5
5
|
require_relative "subject"
|
|
6
|
+
require_relative "deep_source"
|
|
6
7
|
require_relative "constant_subject"
|
|
7
8
|
|
|
8
9
|
module Saboteur
|
|
@@ -47,6 +48,8 @@ module Saboteur
|
|
|
47
48
|
result = Prism.parse(source)
|
|
48
49
|
return Parsed.new(subjects: [], unsupported: [], errors: result.errors.map(&:message)) if result.failure?
|
|
49
50
|
|
|
51
|
+
DeepSource.check!(result.value, file)
|
|
52
|
+
|
|
50
53
|
ctx = { file: file, source: source, frozen: frozen_literals?(result) }
|
|
51
54
|
out = { subjects: [], unsupported: [], ambiguous: [] }
|
|
52
55
|
walk(result.value, [], out, **ctx)
|
data/lib/saboteur/custom_map.rb
CHANGED
|
@@ -67,23 +67,24 @@ module Saboteur
|
|
|
67
67
|
|
|
68
68
|
numbered = key.is_a?(Regexp) && spec.match?(/\\\d/)
|
|
69
69
|
named = key.is_a?(Regexp) && spec.match?(/%[-0-9]*[<{][^>}]+[>}]|\\k</)
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
70
|
+
# Existence is deliberately not checked here: a stale or mistyped
|
|
71
|
+
# entry warns at guess time and the naming convention takes over,
|
|
72
|
+
# instead of the whole run dying at boot on one bad line (#640).
|
|
73
|
+
# The root check comes first: no template, backreference or named,
|
|
74
|
+
# may skip it (#712).
|
|
76
75
|
expanded = File.expand_path(spec, root)
|
|
77
76
|
unless expanded.start_with?("#{root}/")
|
|
78
77
|
raise ConfigError, "#{path}: #{key.inspect} maps to #{spec.inspect}, which is outside the project root"
|
|
79
78
|
end
|
|
80
|
-
|
|
81
|
-
|
|
82
|
-
if named
|
|
79
|
+
if numbered && named
|
|
83
80
|
raise ConfigError, "#{path}: #{key.inspect} maps to #{spec.inspect}; " \
|
|
84
81
|
'regex map specs support \1 backreferences only'
|
|
85
82
|
end
|
|
86
|
-
|
|
83
|
+
return if numbered # backreference filled in at match time
|
|
84
|
+
return unless named
|
|
85
|
+
|
|
86
|
+
raise ConfigError, "#{path}: #{key.inspect} maps to #{spec.inspect}; " \
|
|
87
|
+
'regex map specs support \1 backreferences only'
|
|
87
88
|
end
|
|
88
89
|
|
|
89
90
|
private_class_method :validate_entry, :validate_spec
|
|
@@ -160,14 +161,14 @@ module Saboteur
|
|
|
160
161
|
return :excluded if matches.all? { |entry, _| entry.specs.empty? }
|
|
161
162
|
|
|
162
163
|
matches.reject { |entry, _| entry.specs.empty? }
|
|
163
|
-
.flat_map { |entry, match| resolve(entry, match) }
|
|
164
|
+
.flat_map { |entry, match| resolve(entry, match, file) }
|
|
164
165
|
end
|
|
165
166
|
|
|
166
167
|
def specificity(key)
|
|
167
168
|
key.count("^*?[]{}\\")
|
|
168
169
|
end
|
|
169
170
|
|
|
170
|
-
def resolve(entry, match)
|
|
171
|
+
def resolve(entry, match, file = nil)
|
|
171
172
|
return :excluded if entry.specs.empty?
|
|
172
173
|
|
|
173
174
|
specs = entry.specs
|
|
@@ -176,6 +177,13 @@ module Saboteur
|
|
|
176
177
|
# that is never a real spec path, so drop it here rather than letting
|
|
177
178
|
# a junk entry reach the run.
|
|
178
179
|
specs = specs.map { |spec| spec.gsub(/\\(\d+)/) { match[::Regexp.last_match(1).to_i].to_s } }.reject(&:empty?)
|
|
180
|
+
# A capture can carry "../" segments the template check never saw:
|
|
181
|
+
# refuse a filled spec that escapes the root (#712).
|
|
182
|
+
escaped = specs.reject { |spec| File.expand_path(spec, @root).start_with?("#{@root}/") }
|
|
183
|
+
if escaped.any?
|
|
184
|
+
raise ConfigError,
|
|
185
|
+
"#{entry.key.inspect} maps #{file} to #{escaped.first.inspect}, which is outside the project root"
|
|
186
|
+
end
|
|
179
187
|
end
|
|
180
188
|
specs
|
|
181
189
|
end
|
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
# SPDX-License-Identifier: MIT
|
|
2
|
+
# frozen_string_literal: true
|
|
3
|
+
|
|
4
|
+
module Saboteur
|
|
5
|
+
# A file nested deeper than the walkers can safely recurse (#719).
|
|
6
|
+
class DeepSourceError < Error
|
|
7
|
+
def initialize(file, limit)
|
|
8
|
+
super("#{file} nests deeper than #{limit} AST levels; saboteur cannot walk it safely")
|
|
9
|
+
end
|
|
10
|
+
end
|
|
11
|
+
|
|
12
|
+
# The AST walkers (SubjectFinder, ConstantSubjectFinder, AttributeFinder
|
|
13
|
+
# and the operator visits) recurse one stack frame per nesting level and
|
|
14
|
+
# overflow the Ruby stack around a thousand nested calls, while Prism
|
|
15
|
+
# still parses to about five thousand (#719). Measuring depth iteratively
|
|
16
|
+
# before any walk turns that crash into a clean refusal: the CLI boundary
|
|
17
|
+
# prints one line naming the file and exits 3.
|
|
18
|
+
module DeepSource
|
|
19
|
+
# Two AST levels per nested call (`a(` adds a CallNode and an
|
|
20
|
+
# ArgumentsNode). The finders walk a file from its root and survive a
|
|
21
|
+
# thousand nested calls, so their limit sits just above that. The
|
|
22
|
+
# operator visits recurse deeper per level (method_swap overflows a
|
|
23
|
+
# bare stack around 900 nested calls, earlier under the engine), so a
|
|
24
|
+
# subject's body refuses earlier. Both margins stay deterministic:
|
|
25
|
+
# a clean refusal instead of a SystemStackError that depends on how
|
|
26
|
+
# deep the caller's stack already is.
|
|
27
|
+
MAX_AST_DEPTH = 2200
|
|
28
|
+
MAX_MUTATION_DEPTH = 1200
|
|
29
|
+
|
|
30
|
+
def self.check!(program, file, limit: MAX_AST_DEPTH)
|
|
31
|
+
raise DeepSourceError.new(file, limit) if depth(program) > limit
|
|
32
|
+
end
|
|
33
|
+
|
|
34
|
+
# Iterative on purpose: a recursive measure would crash on exactly the
|
|
35
|
+
# files it exists to catch (#719, same shape as #711).
|
|
36
|
+
def self.depth(node)
|
|
37
|
+
max = 0
|
|
38
|
+
work = [[node, 1]]
|
|
39
|
+
until work.empty?
|
|
40
|
+
current, level = work.pop
|
|
41
|
+
next if current.nil?
|
|
42
|
+
|
|
43
|
+
max = level if level > max
|
|
44
|
+
current.compact_child_nodes.each { |child| work << [child, level + 1] }
|
|
45
|
+
end
|
|
46
|
+
max
|
|
47
|
+
end
|
|
48
|
+
end
|
|
49
|
+
end
|
|
@@ -26,7 +26,7 @@ module Saboteur
|
|
|
26
26
|
# unknown tree. The root is physical (realpath), like the paths
|
|
27
27
|
# Config#included_files returns.
|
|
28
28
|
def self.for(config, config_path:, gemfile: ENV.fetch("BUNDLE_GEMFILE", nil))
|
|
29
|
-
state = WorkingTree.new(config.root, ignored: config.
|
|
29
|
+
state = WorkingTree.new(config.root, ignored: config.ignored_tree_paths(gemfile: gemfile)).state
|
|
30
30
|
root = File.realpath(config.root)
|
|
31
31
|
new(state, included: relative(config.included_files, root), include: config["include"],
|
|
32
32
|
exclude: config["exclude"], root: root,
|
data/lib/saboteur/doctor.rb
CHANGED
|
@@ -19,6 +19,7 @@ module Saboteur
|
|
|
19
19
|
def call
|
|
20
20
|
Engine::Boot.refuse_recursion!
|
|
21
21
|
check_runtime
|
|
22
|
+
check_environment
|
|
22
23
|
check_coverage
|
|
23
24
|
check_boot_files
|
|
24
25
|
check_custom_map
|
|
@@ -55,6 +56,11 @@ module Saboteur
|
|
|
55
56
|
raise Error, "target boot failed: #{e.class}: #{e.message}; check require files, hooks and rspec_args"
|
|
56
57
|
end
|
|
57
58
|
|
|
59
|
+
def check_environment
|
|
60
|
+
hints = EnvironmentHints.all(@config.root)
|
|
61
|
+
@out.puts "Environment: #{hints.empty? ? 'no bundler/appraisal problems detected' : hints.join('; ')}"
|
|
62
|
+
end
|
|
63
|
+
|
|
58
64
|
def check_runtime
|
|
59
65
|
@out.puts "Ruby: #{RUBY_DESCRIPTION}"
|
|
60
66
|
unless Gem::Version.new(RUBY_VERSION) >= Gem::Version.new("3.3") && Process.respond_to?(:fork)
|
|
@@ -5,6 +5,7 @@ require "json"
|
|
|
5
5
|
require "fileutils"
|
|
6
6
|
require_relative "../text"
|
|
7
7
|
require_relative "../frameworks/sqlite_memory_database"
|
|
8
|
+
require_relative "baseline_parts"
|
|
8
9
|
|
|
9
10
|
module Saboteur
|
|
10
11
|
class Engine
|
|
@@ -12,6 +13,8 @@ module Saboteur
|
|
|
12
13
|
# or mutation source is used. This catches observed instability, not all
|
|
13
14
|
# possible flaky behaviour.
|
|
14
15
|
class BaselineGuard
|
|
16
|
+
include BaselineParts
|
|
17
|
+
|
|
15
18
|
# How many unchanged-suite checks actually completed and validated:
|
|
16
19
|
# the number of order comparisons a run can claim is one less.
|
|
17
20
|
attr_reader :completed_passes
|
|
@@ -131,10 +134,10 @@ module Saboteur
|
|
|
131
134
|
return if mutants_get >= pass_elapsed
|
|
132
135
|
|
|
133
136
|
verdict = mutants_get <= 0 ? "exceeds" : "barely fits"
|
|
134
|
-
@diagnostics.warn("
|
|
137
|
+
@diagnostics.warn("raise --max-duration, pass fewer --baseline-passes, or narrow the scope\n " \
|
|
138
|
+
"projected baseline+coverage ~#{projected.round}s #{verdict} " \
|
|
135
139
|
"--max-duration #{budget.round}s; mutants would get " \
|
|
136
|
-
"~#{format('%g', [mutants_get, 0].max.round(1))}s
|
|
137
|
-
"--baseline-passes, or narrow the scope")
|
|
140
|
+
"~#{format('%g', [mutants_get, 0].max.round(1))}s")
|
|
138
141
|
end
|
|
139
142
|
|
|
140
143
|
# An admitted pass still gets only the time left in the run budget:
|
|
@@ -176,6 +179,9 @@ module Saboteur
|
|
|
176
179
|
end
|
|
177
180
|
|
|
178
181
|
def check(number)
|
|
182
|
+
parts = baseline_parts
|
|
183
|
+
return check_parts(number, parts) if parts.size > 1
|
|
184
|
+
|
|
179
185
|
Execution::ForkReadiness.prepare_heap
|
|
180
186
|
heartbeat = @diagnostics.progress_heartbeat("baseline check #{number}").watch(log_path(number))
|
|
181
187
|
admitted = phase_timeout
|
|
@@ -183,6 +189,11 @@ module Saboteur
|
|
|
183
189
|
label: "baseline child died during setup/run") do |writer|
|
|
184
190
|
record(writer, number)
|
|
185
191
|
end
|
|
192
|
+
settled(number, result, admitted)
|
|
193
|
+
end
|
|
194
|
+
|
|
195
|
+
def settled(number, result, admitted, part: nil)
|
|
196
|
+
@part = part
|
|
186
197
|
refuse_budget_stop(number, admitted) if budget_stopped?(result, admitted)
|
|
187
198
|
refuse(number, stopped_reason(result)) if result.stopped_by
|
|
188
199
|
# Before the payload checks: a child killed by a signal (the OOM
|
|
@@ -221,15 +232,16 @@ module Saboteur
|
|
|
221
232
|
data["executed"].is_a?(Array) && data["executed"].all?(String)
|
|
222
233
|
end
|
|
223
234
|
|
|
224
|
-
def record(writer, number)
|
|
225
|
-
path = log_path(number)
|
|
235
|
+
def record(writer, number, files: nil, part: nil)
|
|
236
|
+
path = log_path(number, part)
|
|
226
237
|
FileUtils.mkdir_p(File.dirname(path))
|
|
227
238
|
$stdout.reopen(path, "w")
|
|
228
239
|
$stdout.sync = true
|
|
229
240
|
$stderr.reopen($stdout)
|
|
230
241
|
@hooks.run(:after_fork)
|
|
231
242
|
Kernel.srand(@config["seed"])
|
|
232
|
-
|
|
243
|
+
seed = @config["seed"]
|
|
244
|
+
result = files ? @integration.run_baseline(seed: seed, files: files) : @integration.run_baseline(seed: seed)
|
|
233
245
|
payload = { passed: result.passed, failed: result.failed_ids, executed: result.executed_ids }
|
|
234
246
|
payload[:hint] = Frameworks::SqliteMemoryDatabase.fork_loss_hint unless result.passed
|
|
235
247
|
writer.write(JSON.generate(payload))
|
|
@@ -260,29 +272,54 @@ module Saboteur
|
|
|
260
272
|
return if errors.empty? || !spec_helper_requires_minitest?
|
|
261
273
|
|
|
262
274
|
symptom = (lines.find { |line| line.include?("undefined method") } || errors.first).strip
|
|
263
|
-
@diagnostics.warn("
|
|
275
|
+
@diagnostics.warn("run saboteur on an RSpec suite: it is RSpec-only, so a Minitest suite cannot be " \
|
|
276
|
+
"mutation tested with it\n " \
|
|
277
|
+
"the baseline failure looks like a Minitest suite running under RSpec, not a " \
|
|
264
278
|
"broken RSpec suite: the log shows #{errors.size} " \
|
|
265
279
|
"NoMethodError#{'s' if errors.size > 1} (for example: #{symptom.slice(0, EXCERPT_WIDTH)}) " \
|
|
266
|
-
"and spec/spec_helper.rb requires minitest
|
|
267
|
-
"Minitest suite cannot be mutation tested with it.")
|
|
280
|
+
"and spec/spec_helper.rb requires minitest")
|
|
268
281
|
end
|
|
269
282
|
|
|
270
283
|
# The framework check behind the warning: spec_helper is the
|
|
271
|
-
# suite's own declaration of what it runs on.
|
|
284
|
+
# suite's own declaration of what it runs on. Parsed, not grepped: a
|
|
285
|
+
# regex also fires on commented-out or heredoc "require" lines for a
|
|
286
|
+
# framework the project does not use (#708).
|
|
272
287
|
def spec_helper_requires_minitest?
|
|
273
|
-
|
|
288
|
+
require_calls(Prism.parse_file(File.join(@config.root, "spec", "spec_helper.rb")).value)
|
|
289
|
+
.any? { |name| name == "minitest" || name.start_with?("minitest/") }
|
|
274
290
|
rescue SystemCallError
|
|
275
291
|
false
|
|
276
292
|
end
|
|
277
293
|
|
|
294
|
+
# Iterative like RefinementActivations.collect: a deeply nested
|
|
295
|
+
# spec_helper must never raise SystemStackError out of a warning
|
|
296
|
+
# heuristic (same class as #711).
|
|
297
|
+
def require_calls(node)
|
|
298
|
+
found = []
|
|
299
|
+
work = [node]
|
|
300
|
+
until work.empty?
|
|
301
|
+
current = work.pop
|
|
302
|
+
next unless current.is_a?(Prism::Node)
|
|
303
|
+
|
|
304
|
+
if current.is_a?(Prism::CallNode) && current.name == :require && current.receiver.nil?
|
|
305
|
+
argument = current.arguments&.arguments&.first
|
|
306
|
+
found << argument.unescaped if argument.is_a?(Prism::StringNode)
|
|
307
|
+
end
|
|
308
|
+
work.concat(current.compact_child_nodes)
|
|
309
|
+
end
|
|
310
|
+
found
|
|
311
|
+
end
|
|
312
|
+
|
|
278
313
|
# A child that recognises its own failure shape (an empty sqlite
|
|
279
314
|
# :memory: database after the fork, #387) says how to fix it.
|
|
280
315
|
def with_hint(reason, data)
|
|
281
316
|
data["hint"] ? "#{reason}. #{data['hint']}" : reason
|
|
282
317
|
end
|
|
283
318
|
|
|
284
|
-
|
|
285
|
-
|
|
319
|
+
# A split check keeps one log per part: baseline-1-part-2.log.
|
|
320
|
+
def log_path(number, part = @part)
|
|
321
|
+
name = part ? "baseline-#{number}-part-#{part + 1}.log" : "baseline-#{number}.log"
|
|
322
|
+
File.join(@config.path("cache_dir"), name)
|
|
286
323
|
end
|
|
287
324
|
|
|
288
325
|
# Leftover children are a pass-with-leaks verdict, not a suite
|
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
# SPDX-License-Identifier: MIT
|
|
2
|
+
# frozen_string_literal: true
|
|
3
|
+
|
|
4
|
+
require_relative "suite_partition"
|
|
5
|
+
|
|
6
|
+
module Saboteur
|
|
7
|
+
class Engine
|
|
8
|
+
# A baseline check split like the coverage build (--coverage-parts N):
|
|
9
|
+
# the same disjoint spec-file parts run as children at once, each over
|
|
10
|
+
# its own files. Uses the guard's @config, @integration, @supervisor,
|
|
11
|
+
# @diagnostics, @part and settled/phase_timeout/log_path.
|
|
12
|
+
#
|
|
13
|
+
# The check's contract is unchanged. Every part must pass, and the
|
|
14
|
+
# examples the parts executed together must be exactly the loaded
|
|
15
|
+
# suite's (the guard compares them before it accepts the pass). What a
|
|
16
|
+
# split no longer shows is a failure that needs the whole suite in one
|
|
17
|
+
# process.
|
|
18
|
+
module BaselineParts
|
|
19
|
+
private
|
|
20
|
+
|
|
21
|
+
# The parts the coverage build would use. One part (or none, for an
|
|
22
|
+
# adapter that lists no spec files) means the ordinary single check.
|
|
23
|
+
def baseline_parts
|
|
24
|
+
Engine::SuitePartition.parts(@integration.test_files, @config["coverage_parts"])
|
|
25
|
+
end
|
|
26
|
+
|
|
27
|
+
def check_parts(number, parts)
|
|
28
|
+
Execution::ForkReadiness.prepare_heap
|
|
29
|
+
heartbeat = @diagnostics.progress_heartbeat("baseline check #{number} (#{parts.size} parts)")
|
|
30
|
+
.watch(log_path(number, 0))
|
|
31
|
+
admitted = phase_timeout
|
|
32
|
+
results = @supervisor.run_many(parts.size, timeout: admitted, on_tick: heartbeat.method(:tick),
|
|
33
|
+
label: "baseline child died during setup/run") do |writer, slot|
|
|
34
|
+
ENV["TEST_ENV_NUMBER"] = slot.zero? ? "" : (slot + 1).to_s
|
|
35
|
+
record(writer, number, files: parts[slot], part: slot)
|
|
36
|
+
end
|
|
37
|
+
joined(results.each_with_index.map { |result, slot| settled(number, result, admitted, part: slot) })
|
|
38
|
+
end
|
|
39
|
+
|
|
40
|
+
# What the guard's checks read: the first failing part's own result
|
|
41
|
+
# (its failures and hint; @part names its log), or a pass carrying
|
|
42
|
+
# every example the parts executed.
|
|
43
|
+
def joined(datas)
|
|
44
|
+
@part = datas.index { |data| !(data["passed"] && data["failed"].empty?) }
|
|
45
|
+
return datas[@part] if @part
|
|
46
|
+
|
|
47
|
+
{ "passed" => true, "failed" => [], "executed" => datas.flat_map { |data| data["executed"] } }
|
|
48
|
+
end
|
|
49
|
+
end
|
|
50
|
+
end
|
|
51
|
+
end
|
data/lib/saboteur/engine/boot.rb
CHANGED
|
@@ -71,6 +71,10 @@ module Saboteur
|
|
|
71
71
|
Integration::RSpec.new(@config).require_framework if @config["integration"] == "rspec"
|
|
72
72
|
@config["require"].each { |file| require File.expand_path(file, @root) }
|
|
73
73
|
integration = Integration.fetch(@config["integration"]).new(@config)
|
|
74
|
+
# Running outside the bundle or beside an Appraisal gemfile set
|
|
75
|
+
# loads the wrong gems; say so before the spec load fails in
|
|
76
|
+
# confusing ways (#598, #609). Advisory only.
|
|
77
|
+
EnvironmentHints.all(@root).each { |hint| warn "saboteur: warning: #{hint}" }
|
|
74
78
|
integration.setup
|
|
75
79
|
rails&.after_boot
|
|
76
80
|
@hooks.run(:after_boot)
|
|
@@ -8,11 +8,12 @@ module Saboteur
|
|
|
8
8
|
class CandidateSelection
|
|
9
9
|
attr_reader :skipped_subjects, :selected, :unscoped_digests
|
|
10
10
|
|
|
11
|
-
def initialize(candidates:, map:, limit:, diagnostics:)
|
|
11
|
+
def initialize(candidates:, map:, limit:, diagnostics:, root: nil)
|
|
12
12
|
@candidates = candidates
|
|
13
13
|
@map = map
|
|
14
14
|
@limit = limit
|
|
15
15
|
@diagnostics = diagnostics
|
|
16
|
+
@root = root
|
|
16
17
|
@skipped_subjects = {}
|
|
17
18
|
@selected = {}
|
|
18
19
|
@unscoped_digests = {}
|
|
@@ -29,6 +30,7 @@ module Saboteur
|
|
|
29
30
|
return @selected[subject] = []
|
|
30
31
|
end
|
|
31
32
|
|
|
33
|
+
warn_boot_once(subject)
|
|
32
34
|
list = @candidates.for(subject)
|
|
33
35
|
# The full pre-scope digests let IgnoreList#apply tell "mutation gone"
|
|
34
36
|
# from "mutation scoped out of this run" (#284).
|
|
@@ -48,18 +50,26 @@ module Saboteur
|
|
|
48
50
|
|
|
49
51
|
private
|
|
50
52
|
|
|
53
|
+
# A note, not a verdict (#575): every mutant still runs and scores.
|
|
54
|
+
def warn_boot_once(subject)
|
|
55
|
+
return unless @map.boot_once?(subject)
|
|
56
|
+
|
|
57
|
+
@diagnostics.warn("#{subject.id}: verify a survivor here by mutating the file on disk\n " \
|
|
58
|
+
"covered by every test and run at boot: mutants apply after boot, so a survivor " \
|
|
59
|
+
"can be a boot-once effect the suite cannot re-observe (#575)")
|
|
60
|
+
end
|
|
61
|
+
|
|
51
62
|
# A scoped no-coverage skip names the specs the run loaded: coverage
|
|
52
63
|
# living in a sibling spec file is otherwise invisible, and the false
|
|
53
64
|
# read is "untestable" when the truth is "not loaded" (#636). Names
|
|
54
65
|
# at most five, like the other selection notes.
|
|
55
66
|
def skip_announcement(subject, reason)
|
|
56
|
-
line = "#{subject.id}: skipped (#{reason}); its change is not verified by this run"
|
|
67
|
+
line = "#{subject.id} (#{subject.location(@root)}): skipped (#{reason}); its change is not verified by this run"
|
|
57
68
|
return line unless reason == "no coverage"
|
|
58
69
|
|
|
59
70
|
loaded = @map.tests.map { |id| id.split("[").first.delete_prefix("./") }.uniq.sort
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
"#{line} - loaded specs: #{names}#{more}; " \
|
|
71
|
+
names = loaded.empty? ? "none" : Text.name_list(loaded)
|
|
72
|
+
"#{line} - loaded specs: #{names}; " \
|
|
63
73
|
"other spec files may cover it, pass --test to load them"
|
|
64
74
|
end
|
|
65
75
|
|
|
@@ -109,6 +109,9 @@ module Saboteur
|
|
|
109
109
|
end
|
|
110
110
|
|
|
111
111
|
def proposals(source, subject)
|
|
112
|
+
# The operator visits recurse over the subject's body; refuse a
|
|
113
|
+
# body nested too deeply instead of crashing the run (#719).
|
|
114
|
+
DeepSource.check!(subject.node, subject.file, limit: DeepSource::MAX_MUTATION_DEPTH)
|
|
112
115
|
proposals = source.mutations_for(subject)
|
|
113
116
|
if proposals.empty?
|
|
114
117
|
@diagnostics.record_generation(subject, source: source_name(source), reason: "empty_generation")
|
|
@@ -142,7 +145,7 @@ module Saboteur
|
|
|
142
145
|
|
|
143
146
|
[mutation, text]
|
|
144
147
|
rescue ArgumentError => e
|
|
145
|
-
@diagnostics.warn("#{subject.id}: dropped #{mutation.source} mutation
|
|
148
|
+
@diagnostics.warn("#{subject.id}: dropped #{mutation.source} mutation\n #{e.message}")
|
|
146
149
|
record_drop(subject, mutation, "invalid_edit")
|
|
147
150
|
end
|
|
148
151
|
|