saboteur 0.8.0 → 0.9.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +215 -0
- data/README.md +44 -6
- data/exe/saboteur +6 -2
- data/lib/saboteur/attribute_finder.rb +116 -0
- data/lib/saboteur/baseline/run_evidence.rb +2 -1
- data/lib/saboteur/cli/agent_provenance.rb +59 -0
- data/lib/saboteur/cli/dirty_tree_notice.rb +28 -0
- data/lib/saboteur/cli/doctor_command.rb +41 -0
- data/lib/saboteur/cli/error_cause.rb +40 -0
- data/lib/saboteur/cli/execution_options.rb +46 -0
- data/lib/saboteur/cli/exit_status.rb +12 -14
- data/lib/saboteur/cli/explicit_file_options.rb +21 -0
- data/lib/saboteur/cli/explicit_sources.rb +46 -0
- data/lib/saboteur/cli/failure_output.rb +35 -0
- data/lib/saboteur/cli/help_command.rb +23 -0
- data/lib/saboteur/cli/matrix_option.rb +17 -0
- data/lib/saboteur/cli/output_options.rb +27 -0
- data/lib/saboteur/cli/presets.rb +41 -1
- data/lib/saboteur/cli/preview_command.rb +26 -0
- data/lib/saboteur/cli/run_budget_options.rb +6 -1
- data/lib/saboteur/cli/run_command.rb +33 -0
- data/lib/saboteur/cli/run_output.rb +38 -0
- data/lib/saboteur/cli/run_reporting.rb +6 -5
- data/lib/saboteur/cli/run_validation.rb +54 -0
- data/lib/saboteur/cli/source_set.rb +0 -18
- data/lib/saboteur/cli.rb +29 -305
- data/lib/saboteur/config.rb +76 -7
- data/lib/saboteur/config_file.rb +12 -8
- data/lib/saboteur/constant_subject_finder.rb +3 -0
- data/lib/saboteur/custom_map.rb +24 -5
- data/lib/saboteur/deep_source.rb +49 -0
- data/lib/saboteur/dirty_tree_warning.rb +6 -4
- data/lib/saboteur/doctor.rb +10 -0
- data/lib/saboteur/engine/baseline_guard.rb +118 -15
- data/lib/saboteur/engine/baseline_parts.rb +51 -0
- data/lib/saboteur/engine/boot.rb +11 -3
- data/lib/saboteur/engine/bytecode_equivalence.rb +17 -1
- data/lib/saboteur/engine/candidate_selection.rb +16 -5
- data/lib/saboteur/engine/candidates.rb +4 -1
- data/lib/saboteur/engine/coverage_inputs_snapshot.rb +146 -0
- data/lib/saboteur/engine/coverage_miss_note.rb +86 -0
- data/lib/saboteur/engine/coverage_partial.rb +48 -0
- data/lib/saboteur/engine/coverage_parts.rb +50 -0
- data/lib/saboteur/engine/coverage_shadow.rb +92 -0
- data/lib/saboteur/engine/coverage_store.rb +296 -21
- data/lib/saboteur/engine/diagnostics.rb +24 -7
- data/lib/saboteur/engine/distribution.rb +2 -2
- data/lib/saboteur/engine/gated_run.rb +4 -1
- data/lib/saboteur/engine/load_coverage.rb +72 -8
- data/lib/saboteur/engine/partial_invalidation.rb +115 -0
- data/lib/saboteur/engine/partial_merge.rb +40 -0
- data/lib/saboteur/engine/pipelines.rb +11 -3
- data/lib/saboteur/engine/reference_specs.rb +71 -0
- data/lib/saboteur/engine/result_cache.rb +68 -5
- data/lib/saboteur/engine/resume_manifest.rb +110 -0
- data/lib/saboteur/engine/run_budget.rb +10 -7
- data/lib/saboteur/engine/run_fingerprint.rb +2 -1
- data/lib/saboteur/engine/sandbox_policy.rb +26 -5
- data/lib/saboteur/engine/scheduler.rb +33 -25
- data/lib/saboteur/engine/selection_announcement.rb +9 -3
- data/lib/saboteur/engine/shadow_comparison.rb +121 -0
- data/lib/saboteur/engine/shadow_run.rb +38 -0
- data/lib/saboteur/engine/subject_selection.rb +144 -64
- data/lib/saboteur/engine/suite_partition.rb +22 -0
- data/lib/saboteur/engine/test_guess.rb +186 -32
- data/lib/saboteur/engine/unchanged_syntax.rb +10 -2
- data/lib/saboteur/engine.rb +44 -11
- data/lib/saboteur/environment_hints.rb +55 -0
- data/lib/saboteur/execution/bundle_gemfile_pin.rb +72 -0
- data/lib/saboteur/execution/external_runner.rb +123 -1
- data/lib/saboteur/execution/fiddle_loader.rb +34 -0
- data/lib/saboteur/execution/gemfile_path_sources.rb +166 -0
- data/lib/saboteur/execution/marker_sweep.rb +2 -1
- data/lib/saboteur/execution/process_table.rb +7 -3
- data/lib/saboteur/execution/reevaluator.rb +7 -0
- data/lib/saboteur/execution/sandbox.rb +39 -20
- data/lib/saboteur/execution/seatbelt.rb +10 -3
- data/lib/saboteur/execution/supervisor.rb +42 -8
- data/lib/saboteur/execution/supervisor_batch.rb +105 -0
- data/lib/saboteur/execution/test_command_shape.rb +149 -0
- data/lib/saboteur/execution/worker_pool.rb +48 -4
- data/lib/saboteur/frameworks/rails.rb +2 -5
- data/lib/saboteur/frameworks/sqlite_memory_database.rb +32 -2
- data/lib/saboteur/gates.rb +121 -28
- data/lib/saboteur/install.rb +6 -1
- data/lib/saboteur/integration/base.rb +19 -2
- data/lib/saboteur/integration/rspec.rb +145 -18
- data/lib/saboteur/line_hits.rb +81 -0
- data/lib/saboteur/method_calls.rb +91 -0
- data/lib/saboteur/mutation_source/argument_swap.rb +54 -0
- data/lib/saboteur/mutation_source/builtin.rb +33 -21
- data/lib/saboteur/mutation_source/condition_negation.rb +3 -12
- data/lib/saboteur/mutation_source/hazard_guard.rb +94 -13
- data/lib/saboteur/mutation_source/jump_value.rb +59 -0
- data/lib/saboteur/mutation_source/literal_replacement.rb +6 -23
- data/lib/saboteur/mutation_source/operator_examples.rb +6 -0
- data/lib/saboteur/mutation_source/operator_registry.rb +6 -0
- data/lib/saboteur/mutation_source/operator_visit.rb +7 -1
- data/lib/saboteur/mutation_source/rescue_removal.rb +69 -0
- data/lib/saboteur/mutation_source/unary_removal.rb +47 -0
- data/lib/saboteur/mutation_source/when_removal.rb +51 -0
- data/lib/saboteur/preview.rb +13 -0
- data/lib/saboteur/refinement_activations.rb +22 -10
- data/lib/saboteur/reporter/agent.rb +2 -2
- data/lib/saboteur/reporter/agent_incomplete.rb +31 -5
- data/lib/saboteur/reporter/agent_summary.rb +16 -10
- data/lib/saboteur/reporter/review_comments.rb +151 -0
- data/lib/saboteur/reporter/safe_out.rb +32 -0
- data/lib/saboteur/reporter/stryker.rb +8 -6
- data/lib/saboteur/reporter/survivor_classification.rb +41 -3
- data/lib/saboteur/reporter/survivor_hints.rb +5 -0
- data/lib/saboteur/reporter/terminal.rb +45 -5
- data/lib/saboteur/result.rb +14 -4
- data/lib/saboteur/selection/coverage_map.rb +150 -9
- data/lib/saboteur/selection/external_test_map.rb +4 -0
- data/lib/saboteur/selection/git_diff.rb +24 -2
- data/lib/saboteur/selection/layout_only_lines.rb +45 -1
- data/lib/saboteur/selection/recorder.rb +52 -22
- data/lib/saboteur/selection/trace_recorder.rb +83 -0
- data/lib/saboteur/selection/untracked_changes.rb +1 -2
- data/lib/saboteur/subject.rb +16 -0
- data/lib/saboteur/subject_finder.rb +91 -6
- data/lib/saboteur/text.rb +31 -0
- data/lib/saboteur/version.rb +1 -1
- data/lib/saboteur/working_tree.rb +7 -2
- data/lib/saboteur.rb +9 -0
- metadata +48 -16
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
# SPDX-License-Identifier: MIT
|
|
2
|
+
# frozen_string_literal: true
|
|
3
|
+
|
|
4
|
+
module Saboteur
|
|
5
|
+
# A file nested deeper than the walkers can safely recurse (#719).
|
|
6
|
+
class DeepSourceError < Error
|
|
7
|
+
def initialize(file, limit)
|
|
8
|
+
super("#{file} nests deeper than #{limit} AST levels; saboteur cannot walk it safely")
|
|
9
|
+
end
|
|
10
|
+
end
|
|
11
|
+
|
|
12
|
+
# The AST walkers (SubjectFinder, ConstantSubjectFinder, AttributeFinder
|
|
13
|
+
# and the operator visits) recurse one stack frame per nesting level and
|
|
14
|
+
# overflow the Ruby stack around a thousand nested calls, while Prism
|
|
15
|
+
# still parses to about five thousand (#719). Measuring depth iteratively
|
|
16
|
+
# before any walk turns that crash into a clean refusal: the CLI boundary
|
|
17
|
+
# prints one line naming the file and exits 3.
|
|
18
|
+
module DeepSource
|
|
19
|
+
# Two AST levels per nested call (`a(` adds a CallNode and an
|
|
20
|
+
# ArgumentsNode). The finders walk a file from its root and survive a
|
|
21
|
+
# thousand nested calls, so their limit sits just above that. The
|
|
22
|
+
# operator visits recurse deeper per level (method_swap overflows a
|
|
23
|
+
# bare stack around 900 nested calls, earlier under the engine), so a
|
|
24
|
+
# subject's body refuses earlier. Both margins stay deterministic:
|
|
25
|
+
# a clean refusal instead of a SystemStackError that depends on how
|
|
26
|
+
# deep the caller's stack already is.
|
|
27
|
+
MAX_AST_DEPTH = 2200
|
|
28
|
+
MAX_MUTATION_DEPTH = 1200
|
|
29
|
+
|
|
30
|
+
def self.check!(program, file, limit: MAX_AST_DEPTH)
|
|
31
|
+
raise DeepSourceError.new(file, limit) if depth(program) > limit
|
|
32
|
+
end
|
|
33
|
+
|
|
34
|
+
# Iterative on purpose: a recursive measure would crash on exactly the
|
|
35
|
+
# files it exists to catch (#719, same shape as #711).
|
|
36
|
+
def self.depth(node)
|
|
37
|
+
max = 0
|
|
38
|
+
work = [[node, 1]]
|
|
39
|
+
until work.empty?
|
|
40
|
+
current, level = work.pop
|
|
41
|
+
next if current.nil?
|
|
42
|
+
|
|
43
|
+
max = level if level > max
|
|
44
|
+
current.compact_child_nodes.each { |child| work << [child, level + 1] }
|
|
45
|
+
end
|
|
46
|
+
max
|
|
47
|
+
end
|
|
48
|
+
end
|
|
49
|
+
end
|
|
@@ -2,8 +2,9 @@
|
|
|
2
2
|
# frozen_string_literal: true
|
|
3
3
|
|
|
4
4
|
module Saboteur
|
|
5
|
-
# The warning shown before a run on a tree with uncommitted
|
|
6
|
-
# (#309, #316)
|
|
5
|
+
# The warning shown before a run on a tree with uncommitted
|
|
6
|
+
# changes (#309, #316); the headline says uncommitted or untracked, so a
|
|
7
|
+
# tree of only scratch files is not read as edits (#604). Mutants run your code and can change the tree, so the
|
|
7
8
|
# warning stays; it splits the raw count so a tree full of notes or trial
|
|
8
9
|
# setup files does not read like application edits about to be mutated.
|
|
9
10
|
#
|
|
@@ -25,7 +26,7 @@ module Saboteur
|
|
|
25
26
|
# unknown tree. The root is physical (realpath), like the paths
|
|
26
27
|
# Config#included_files returns.
|
|
27
28
|
def self.for(config, config_path:, gemfile: ENV.fetch("BUNDLE_GEMFILE", nil))
|
|
28
|
-
state = WorkingTree.new(config.root, ignored: config.
|
|
29
|
+
state = WorkingTree.new(config.root, ignored: config.ignored_tree_paths(gemfile: gemfile)).state
|
|
29
30
|
root = File.realpath(config.root)
|
|
30
31
|
new(state, included: relative(config.included_files, root), include: config["include"],
|
|
31
32
|
exclude: config["exclude"], root: root,
|
|
@@ -101,7 +102,8 @@ module Saboteur
|
|
|
101
102
|
groups = [scope.empty? ? "none inside the mutation scope" : counted("inside the mutation scope", scope),
|
|
102
103
|
counted("outside it", outside)]
|
|
103
104
|
groups << counted("saboteur setup file(s)", setup) unless setup.empty?
|
|
104
|
-
"saboteur: warning: #{@state.changed_files} uncommitted change(s) in the working tree:
|
|
105
|
+
"saboteur: warning: #{@state.changed_files} uncommitted or untracked change(s) in the working tree: " \
|
|
106
|
+
"#{groups.join(', ')}."
|
|
105
107
|
end
|
|
106
108
|
|
|
107
109
|
def counted(label, changes)
|
data/lib/saboteur/doctor.rb
CHANGED
|
@@ -19,6 +19,7 @@ module Saboteur
|
|
|
19
19
|
def call
|
|
20
20
|
Engine::Boot.refuse_recursion!
|
|
21
21
|
check_runtime
|
|
22
|
+
check_environment
|
|
22
23
|
check_coverage
|
|
23
24
|
check_boot_files
|
|
24
25
|
check_custom_map
|
|
@@ -55,6 +56,11 @@ module Saboteur
|
|
|
55
56
|
raise Error, "target boot failed: #{e.class}: #{e.message}; check require files, hooks and rspec_args"
|
|
56
57
|
end
|
|
57
58
|
|
|
59
|
+
def check_environment
|
|
60
|
+
hints = EnvironmentHints.all(@config.root)
|
|
61
|
+
@out.puts "Environment: #{hints.empty? ? 'no bundler/appraisal problems detected' : hints.join('; ')}"
|
|
62
|
+
end
|
|
63
|
+
|
|
58
64
|
def check_runtime
|
|
59
65
|
@out.puts "Ruby: #{RUBY_DESCRIPTION}"
|
|
60
66
|
unless Gem::Version.new(RUBY_VERSION) >= Gem::Version.new("3.3") && Process.respond_to?(:fork)
|
|
@@ -102,8 +108,12 @@ module Saboteur
|
|
|
102
108
|
@out.puts %(Seatbelt (opt-in, #436): probe #{Execution::Seatbelt.probe} - capability probe only, ) \
|
|
103
109
|
"not verified isolation"
|
|
104
110
|
elsif RUBY_PLATFORM.match?(/darwin/i)
|
|
111
|
+
# Point at the way OUT of the degraded state, like the run
|
|
112
|
+
# warning does (#667): the opt-in Seatbelt sandbox, or silence.
|
|
105
113
|
@out.puts "macOS: mutants run unsandboxed (Landlock is Linux-only) - a container contains " \
|
|
106
114
|
"filesystem writes outside the project"
|
|
115
|
+
@out.puts %(macOS: opt into the sandbox with c.sandbox = "seatbelt" ) \
|
|
116
|
+
"(docs/guides/macos-seatbelt-sandbox.md), or set c.sandbox = false to silence this"
|
|
107
117
|
@out.puts "Network: never restricted on any platform, and a plain container keeps outbound " \
|
|
108
118
|
"access - deny or stub the network when a mutant could reach real APIs"
|
|
109
119
|
end
|
|
@@ -5,6 +5,7 @@ require "json"
|
|
|
5
5
|
require "fileutils"
|
|
6
6
|
require_relative "../text"
|
|
7
7
|
require_relative "../frameworks/sqlite_memory_database"
|
|
8
|
+
require_relative "baseline_parts"
|
|
8
9
|
|
|
9
10
|
module Saboteur
|
|
10
11
|
class Engine
|
|
@@ -12,6 +13,8 @@ module Saboteur
|
|
|
12
13
|
# or mutation source is used. This catches observed instability, not all
|
|
13
14
|
# possible flaky behaviour.
|
|
14
15
|
class BaselineGuard
|
|
16
|
+
include BaselineParts
|
|
17
|
+
|
|
15
18
|
# How many unchanged-suite checks actually completed and validated:
|
|
16
19
|
# the number of order comparisons a run can claim is one less.
|
|
17
20
|
attr_reader :completed_passes
|
|
@@ -24,7 +27,7 @@ module Saboteur
|
|
|
24
27
|
# coverage build when the cache is cold, so the first timed pass can
|
|
25
28
|
# price the rest (#260).
|
|
26
29
|
def initialize(config:, integration:, hooks:, supervisor:, diagnostics:, full_passes: 2, between_passes: nil,
|
|
27
|
-
timeout_cap: nil)
|
|
30
|
+
timeout_cap: nil, run_started: nil)
|
|
28
31
|
@config = config
|
|
29
32
|
@integration = integration
|
|
30
33
|
@hooks = hooks
|
|
@@ -33,6 +36,7 @@ module Saboteur
|
|
|
33
36
|
@full_passes = full_passes
|
|
34
37
|
@between_passes = between_passes
|
|
35
38
|
@timeout_cap = timeout_cap
|
|
39
|
+
@run_started = run_started
|
|
36
40
|
@completed_passes = 0
|
|
37
41
|
end
|
|
38
42
|
|
|
@@ -51,10 +55,7 @@ module Saboteur
|
|
|
51
55
|
started = Process.clock_gettime(Process::CLOCK_MONOTONIC)
|
|
52
56
|
data = check(number)
|
|
53
57
|
announce_first_pass_cost(number, started)
|
|
54
|
-
unless data["passed"] && data["failed"].empty?
|
|
55
|
-
refuse(number,
|
|
56
|
-
with_hint("the suite failed: #{Array(data['failed']).join(', ')}", data))
|
|
57
|
-
end
|
|
58
|
+
refuse_suite_failure(number, data) unless data["passed"] && data["failed"].empty?
|
|
58
59
|
executed = data["executed"]
|
|
59
60
|
refuse(number, "executed tests do not match the loaded suite") unless
|
|
60
61
|
executed.sort == expected.sort
|
|
@@ -67,6 +68,16 @@ module Saboteur
|
|
|
67
68
|
|
|
68
69
|
private
|
|
69
70
|
|
|
71
|
+
def refuse_failed_suite(number, data)
|
|
72
|
+
outside = data["failed"].empty?
|
|
73
|
+
reason = if outside
|
|
74
|
+
"the suite errored outside any example; see the log tail"
|
|
75
|
+
else
|
|
76
|
+
"the suite failed: #{data['failed'].join(', ')}"
|
|
77
|
+
end
|
|
78
|
+
refuse(number, with_hint(reason, data), tail: outside)
|
|
79
|
+
end
|
|
80
|
+
|
|
70
81
|
# A spent --max-duration budget skips the remaining passes - including
|
|
71
82
|
# the first, when boot alone already spent it (#245).
|
|
72
83
|
def budget_spent?(_index)
|
|
@@ -95,9 +106,10 @@ module Saboteur
|
|
|
95
106
|
|
|
96
107
|
coverage_pending = @full_passes > @config["baseline_passes"]
|
|
97
108
|
remaining = @full_passes - number - (coverage_pending ? 1 : 0)
|
|
109
|
+
elapsed = (Process.clock_gettime(Process::CLOCK_MONOTONIC) - started).round
|
|
110
|
+
warn_budget_projection(elapsed, remaining + (coverage_pending ? 1 : 0))
|
|
98
111
|
return if remaining.zero? && !coverage_pending
|
|
99
112
|
|
|
100
|
-
elapsed = (Process.clock_gettime(Process::CLOCK_MONOTONIC) - started).round
|
|
101
113
|
work = []
|
|
102
114
|
work << "the remaining #{remaining} full #{remaining == 1 ? 'pass' : 'passes'}" if remaining.positive?
|
|
103
115
|
work << "the coverage map build" if coverage_pending
|
|
@@ -106,6 +118,28 @@ module Saboteur
|
|
|
106
118
|
"#{work.join(' and ')} at that rate")
|
|
107
119
|
end
|
|
108
120
|
|
|
121
|
+
# The first timed pass also prices the run against --max-duration:
|
|
122
|
+
# boot and this pass are spent, the remaining passes and a cold
|
|
123
|
+
# coverage build cost about that much again at that rate, and only
|
|
124
|
+
# what is left feeds mutants. Warn while a Ctrl-C is still cheap -
|
|
125
|
+
# the refusal only fires once the coverage build is unaffordable
|
|
126
|
+
# outright (#637).
|
|
127
|
+
def warn_budget_projection(pass_elapsed, remaining_passes)
|
|
128
|
+
budget = @config["max_duration"]
|
|
129
|
+
return unless budget && @run_started
|
|
130
|
+
|
|
131
|
+
spent = Process.clock_gettime(Process::CLOCK_MONOTONIC) - @run_started
|
|
132
|
+
projected = spent + (pass_elapsed * remaining_passes)
|
|
133
|
+
mutants_get = budget - projected
|
|
134
|
+
return if mutants_get >= pass_elapsed
|
|
135
|
+
|
|
136
|
+
verdict = mutants_get <= 0 ? "exceeds" : "barely fits"
|
|
137
|
+
@diagnostics.warn("raise --max-duration, pass fewer --baseline-passes, or narrow the scope\n " \
|
|
138
|
+
"projected baseline+coverage ~#{projected.round}s #{verdict} " \
|
|
139
|
+
"--max-duration #{budget.round}s; mutants would get " \
|
|
140
|
+
"~#{format('%g', [mutants_get, 0].max.round(1))}s")
|
|
141
|
+
end
|
|
142
|
+
|
|
109
143
|
# An admitted pass still gets only the time left in the run budget:
|
|
110
144
|
# without the cap it would overrun a spent --max-duration by its
|
|
111
145
|
# whole coverage_timeout.
|
|
@@ -145,6 +179,9 @@ module Saboteur
|
|
|
145
179
|
end
|
|
146
180
|
|
|
147
181
|
def check(number)
|
|
182
|
+
parts = baseline_parts
|
|
183
|
+
return check_parts(number, parts) if parts.size > 1
|
|
184
|
+
|
|
148
185
|
Execution::ForkReadiness.prepare_heap
|
|
149
186
|
heartbeat = @diagnostics.progress_heartbeat("baseline check #{number}").watch(log_path(number))
|
|
150
187
|
admitted = phase_timeout
|
|
@@ -152,6 +189,11 @@ module Saboteur
|
|
|
152
189
|
label: "baseline child died during setup/run") do |writer|
|
|
153
190
|
record(writer, number)
|
|
154
191
|
end
|
|
192
|
+
settled(number, result, admitted)
|
|
193
|
+
end
|
|
194
|
+
|
|
195
|
+
def settled(number, result, admitted, part: nil)
|
|
196
|
+
@part = part
|
|
155
197
|
refuse_budget_stop(number, admitted) if budget_stopped?(result, admitted)
|
|
156
198
|
refuse(number, stopped_reason(result)) if result.stopped_by
|
|
157
199
|
# Before the payload checks: a child killed by a signal (the OOM
|
|
@@ -190,15 +232,16 @@ module Saboteur
|
|
|
190
232
|
data["executed"].is_a?(Array) && data["executed"].all?(String)
|
|
191
233
|
end
|
|
192
234
|
|
|
193
|
-
def record(writer, number)
|
|
194
|
-
path = log_path(number)
|
|
235
|
+
def record(writer, number, files: nil, part: nil)
|
|
236
|
+
path = log_path(number, part)
|
|
195
237
|
FileUtils.mkdir_p(File.dirname(path))
|
|
196
238
|
$stdout.reopen(path, "w")
|
|
197
239
|
$stdout.sync = true
|
|
198
240
|
$stderr.reopen($stdout)
|
|
199
241
|
@hooks.run(:after_fork)
|
|
200
242
|
Kernel.srand(@config["seed"])
|
|
201
|
-
|
|
243
|
+
seed = @config["seed"]
|
|
244
|
+
result = files ? @integration.run_baseline(seed: seed, files: files) : @integration.run_baseline(seed: seed)
|
|
202
245
|
payload = { passed: result.passed, failed: result.failed_ids, executed: result.executed_ids }
|
|
203
246
|
payload[:hint] = Frameworks::SqliteMemoryDatabase.fork_loss_hint unless result.passed
|
|
204
247
|
writer.write(JSON.generate(payload))
|
|
@@ -209,14 +252,74 @@ module Saboteur
|
|
|
209
252
|
writer.close
|
|
210
253
|
end
|
|
211
254
|
|
|
255
|
+
# A failed unchanged suite never runs mutants: warn first when the
|
|
256
|
+
# failure carries the Minitest signature (#626), then refuse.
|
|
257
|
+
def refuse_suite_failure(number, data)
|
|
258
|
+
warn_minitest_signature(number)
|
|
259
|
+
refuse_failed_suite(number, data)
|
|
260
|
+
end
|
|
261
|
+
|
|
262
|
+
# Minitest::Spec suites in *_spec.rb clothing load under the RSpec
|
|
263
|
+
# integration and then fail the baseline with NoMethodError for
|
|
264
|
+
# Minitest-only methods (#626): the refusal still reads "fix the
|
|
265
|
+
# suite", so warn what the signature actually points at. It stays a
|
|
266
|
+
# warning: an RSpec suite can require minitest for a helper and
|
|
267
|
+
# fail for its own reason, and a heuristic refusal would
|
|
268
|
+
# false-positive there.
|
|
269
|
+
def warn_minitest_signature(number)
|
|
270
|
+
lines = log_lines(number)
|
|
271
|
+
errors = lines.grep(/NoMethodError/)
|
|
272
|
+
return if errors.empty? || !spec_helper_requires_minitest?
|
|
273
|
+
|
|
274
|
+
symptom = (lines.find { |line| line.include?("undefined method") } || errors.first).strip
|
|
275
|
+
@diagnostics.warn("run saboteur on an RSpec suite: it is RSpec-only, so a Minitest suite cannot be " \
|
|
276
|
+
"mutation tested with it\n " \
|
|
277
|
+
"the baseline failure looks like a Minitest suite running under RSpec, not a " \
|
|
278
|
+
"broken RSpec suite: the log shows #{errors.size} " \
|
|
279
|
+
"NoMethodError#{'s' if errors.size > 1} (for example: #{symptom.slice(0, EXCERPT_WIDTH)}) " \
|
|
280
|
+
"and spec/spec_helper.rb requires minitest")
|
|
281
|
+
end
|
|
282
|
+
|
|
283
|
+
# The framework check behind the warning: spec_helper is the
|
|
284
|
+
# suite's own declaration of what it runs on. Parsed, not grepped: a
|
|
285
|
+
# regex also fires on commented-out or heredoc "require" lines for a
|
|
286
|
+
# framework the project does not use (#708).
|
|
287
|
+
def spec_helper_requires_minitest?
|
|
288
|
+
require_calls(Prism.parse_file(File.join(@config.root, "spec", "spec_helper.rb")).value)
|
|
289
|
+
.any? { |name| name == "minitest" || name.start_with?("minitest/") }
|
|
290
|
+
rescue SystemCallError
|
|
291
|
+
false
|
|
292
|
+
end
|
|
293
|
+
|
|
294
|
+
# Iterative like RefinementActivations.collect: a deeply nested
|
|
295
|
+
# spec_helper must never raise SystemStackError out of a warning
|
|
296
|
+
# heuristic (same class as #711).
|
|
297
|
+
def require_calls(node)
|
|
298
|
+
found = []
|
|
299
|
+
work = [node]
|
|
300
|
+
until work.empty?
|
|
301
|
+
current = work.pop
|
|
302
|
+
next unless current.is_a?(Prism::Node)
|
|
303
|
+
|
|
304
|
+
if current.is_a?(Prism::CallNode) && current.name == :require && current.receiver.nil?
|
|
305
|
+
argument = current.arguments&.arguments&.first
|
|
306
|
+
found << argument.unescaped if argument.is_a?(Prism::StringNode)
|
|
307
|
+
end
|
|
308
|
+
work.concat(current.compact_child_nodes)
|
|
309
|
+
end
|
|
310
|
+
found
|
|
311
|
+
end
|
|
312
|
+
|
|
212
313
|
# A child that recognises its own failure shape (an empty sqlite
|
|
213
314
|
# :memory: database after the fork, #387) says how to fix it.
|
|
214
315
|
def with_hint(reason, data)
|
|
215
316
|
data["hint"] ? "#{reason}. #{data['hint']}" : reason
|
|
216
317
|
end
|
|
217
318
|
|
|
218
|
-
|
|
219
|
-
|
|
319
|
+
# A split check keeps one log per part: baseline-1-part-2.log.
|
|
320
|
+
def log_path(number, part = @part)
|
|
321
|
+
name = part ? "baseline-#{number}-part-#{part + 1}.log" : "baseline-#{number}.log"
|
|
322
|
+
File.join(@config.path("cache_dir"), name)
|
|
220
323
|
end
|
|
221
324
|
|
|
222
325
|
# Leftover children are a pass-with-leaks verdict, not a suite
|
|
@@ -230,9 +333,9 @@ module Saboteur
|
|
|
230
333
|
"the suite passed but left #{count} child #{count == 1 ? 'process' : 'processes'} behind: #{detail}"
|
|
231
334
|
end
|
|
232
335
|
|
|
233
|
-
def refuse(number, reason, excerpt: true)
|
|
336
|
+
def refuse(number, reason, excerpt: true, tail: false)
|
|
234
337
|
raise Error, "baseline check #{number} failed at seed #{@config['seed']}: #{reason}. " \
|
|
235
|
-
"#{failure_excerpt(number) if excerpt}See the full log at #{log_path(number)}. " \
|
|
338
|
+
"#{failure_excerpt(number, tail: tail) if excerpt}See the full log at #{log_path(number)}. " \
|
|
236
339
|
"Fix the suite or its setup, then run saboteur again with the same seed. " \
|
|
237
340
|
"No mutants were started."
|
|
238
341
|
end
|
|
@@ -240,11 +343,11 @@ module Saboteur
|
|
|
240
343
|
# The first lines of the failure, inline, so the cause is readable
|
|
241
344
|
# without opening the log: RSpec's "Failures:" section when the log has
|
|
242
345
|
# one, otherwise the last lines of the log.
|
|
243
|
-
def failure_excerpt(number)
|
|
346
|
+
def failure_excerpt(number, tail: false)
|
|
244
347
|
lines = log_lines(number)
|
|
245
348
|
return "" if lines.empty?
|
|
246
349
|
|
|
247
|
-
start = lines.index { |line| line.start_with?("Failures:") }
|
|
350
|
+
start = lines.index { |line| line.start_with?("Failures:") } unless tail
|
|
248
351
|
shown = start ? lines[start, EXCERPT_LINES] : lines.last(EXCERPT_LINES)
|
|
249
352
|
"The log says:\n#{shown.map { |line| " #{line.slice(0, EXCERPT_WIDTH)}" }.join("\n")}\n"
|
|
250
353
|
end
|
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
# SPDX-License-Identifier: MIT
|
|
2
|
+
# frozen_string_literal: true
|
|
3
|
+
|
|
4
|
+
require_relative "suite_partition"
|
|
5
|
+
|
|
6
|
+
module Saboteur
|
|
7
|
+
class Engine
|
|
8
|
+
# A baseline check split like the coverage build (--coverage-parts N):
|
|
9
|
+
# the same disjoint spec-file parts run as children at once, each over
|
|
10
|
+
# its own files. Uses the guard's @config, @integration, @supervisor,
|
|
11
|
+
# @diagnostics, @part and settled/phase_timeout/log_path.
|
|
12
|
+
#
|
|
13
|
+
# The check's contract is unchanged. Every part must pass, and the
|
|
14
|
+
# examples the parts executed together must be exactly the loaded
|
|
15
|
+
# suite's (the guard compares them before it accepts the pass). What a
|
|
16
|
+
# split no longer shows is a failure that needs the whole suite in one
|
|
17
|
+
# process.
|
|
18
|
+
module BaselineParts
|
|
19
|
+
private
|
|
20
|
+
|
|
21
|
+
# The parts the coverage build would use. One part (or none, for an
|
|
22
|
+
# adapter that lists no spec files) means the ordinary single check.
|
|
23
|
+
def baseline_parts
|
|
24
|
+
Engine::SuitePartition.parts(@integration.test_files, @config["coverage_parts"])
|
|
25
|
+
end
|
|
26
|
+
|
|
27
|
+
def check_parts(number, parts)
|
|
28
|
+
Execution::ForkReadiness.prepare_heap
|
|
29
|
+
heartbeat = @diagnostics.progress_heartbeat("baseline check #{number} (#{parts.size} parts)")
|
|
30
|
+
.watch(log_path(number, 0))
|
|
31
|
+
admitted = phase_timeout
|
|
32
|
+
results = @supervisor.run_many(parts.size, timeout: admitted, on_tick: heartbeat.method(:tick),
|
|
33
|
+
label: "baseline child died during setup/run") do |writer, slot|
|
|
34
|
+
ENV["TEST_ENV_NUMBER"] = slot.zero? ? "" : (slot + 1).to_s
|
|
35
|
+
record(writer, number, files: parts[slot], part: slot)
|
|
36
|
+
end
|
|
37
|
+
joined(results.each_with_index.map { |result, slot| settled(number, result, admitted, part: slot) })
|
|
38
|
+
end
|
|
39
|
+
|
|
40
|
+
# What the guard's checks read: the first failing part's own result
|
|
41
|
+
# (its failures and hint; @part names its log), or a pass carrying
|
|
42
|
+
# every example the parts executed.
|
|
43
|
+
def joined(datas)
|
|
44
|
+
@part = datas.index { |data| !(data["passed"] && data["failed"].empty?) }
|
|
45
|
+
return datas[@part] if @part
|
|
46
|
+
|
|
47
|
+
{ "passed" => true, "failed" => [], "executed" => datas.flat_map { |data| data["executed"] } }
|
|
48
|
+
end
|
|
49
|
+
end
|
|
50
|
+
end
|
|
51
|
+
end
|
data/lib/saboteur/engine/boot.rb
CHANGED
|
@@ -66,9 +66,15 @@ module Saboteur
|
|
|
66
66
|
@rails = rails = Frameworks::Rails.new(@config, @hooks) if @config["rails"]
|
|
67
67
|
rails&.before_boot
|
|
68
68
|
@hooks.run(:before_boot)
|
|
69
|
-
add_spec_to_load_path
|
|
69
|
+
add_spec_to_load_path
|
|
70
|
+
# Helpers may configure RSpec before the integration loads specs.
|
|
71
|
+
Integration::RSpec.new(@config).require_framework if @config["integration"] == "rspec"
|
|
70
72
|
@config["require"].each { |file| require File.expand_path(file, @root) }
|
|
71
73
|
integration = Integration.fetch(@config["integration"]).new(@config)
|
|
74
|
+
# Running outside the bundle or beside an Appraisal gemfile set
|
|
75
|
+
# loads the wrong gems; say so before the spec load fails in
|
|
76
|
+
# confusing ways (#598, #609). Advisory only.
|
|
77
|
+
EnvironmentHints.all(@root).each { |hint| warn "saboteur: warning: #{hint}" }
|
|
72
78
|
integration.setup
|
|
73
79
|
rails&.after_boot
|
|
74
80
|
@hooks.run(:after_boot)
|
|
@@ -78,8 +84,10 @@ module Saboteur
|
|
|
78
84
|
private
|
|
79
85
|
|
|
80
86
|
# rspec puts spec/ on the load path, so a boot file under spec/ (a
|
|
81
|
-
# dummy app's rails_helper,
|
|
82
|
-
#
|
|
87
|
+
# dummy app's rails_helper, or a non-Rails gem's spec_helper that
|
|
88
|
+
# requires its siblings) can require its siblings. Match that for
|
|
89
|
+
# every project - a non-Rails gem does not add spec/ itself (#615) -
|
|
90
|
+
# or the same require list fails to load.
|
|
83
91
|
def add_spec_to_load_path
|
|
84
92
|
path = File.join(@root, "spec")
|
|
85
93
|
$LOAD_PATH.unshift(path) unless $LOAD_PATH.include?(path)
|
|
@@ -42,11 +42,27 @@ module Saboteur
|
|
|
42
42
|
|
|
43
43
|
def fingerprint(text, frozen)
|
|
44
44
|
options = { frozen_string_literal: frozen }
|
|
45
|
-
|
|
45
|
+
iseq = warnings_silenced do
|
|
46
|
+
RubyVM::InstructionSequence.compile(text, "saboteur/bytecode-equivalence", nil, 1, options)
|
|
47
|
+
end
|
|
48
|
+
strip_positions(iseq.to_a)
|
|
46
49
|
rescue SyntaxError, StandardError
|
|
47
50
|
nil
|
|
48
51
|
end
|
|
49
52
|
|
|
53
|
+
# Mutants compile one by one, and generated bodies (body_nil,
|
|
54
|
+
# statement_removal) trip Ruby's own void-context warnings, so a host
|
|
55
|
+
# suite running with -w gets one line per mutant flooding stderr
|
|
56
|
+
# (#592, #656). The check is internal: silence the compiler. A mutant
|
|
57
|
+
# that does not parse still fails the compile and runs.
|
|
58
|
+
def warnings_silenced
|
|
59
|
+
saved = $VERBOSE
|
|
60
|
+
$VERBOSE = nil
|
|
61
|
+
yield
|
|
62
|
+
ensure
|
|
63
|
+
$VERBOSE = saved
|
|
64
|
+
end
|
|
65
|
+
|
|
50
66
|
# Structural equality that never coerces: an Integer is not a Float,
|
|
51
67
|
# and +0.0 is not -0.0 (1.0 / +0.0 and 1.0 / -0.0 are different
|
|
52
68
|
# infinities), which eql? alone would not tell apart.
|
|
@@ -8,11 +8,12 @@ module Saboteur
|
|
|
8
8
|
class CandidateSelection
|
|
9
9
|
attr_reader :skipped_subjects, :selected, :unscoped_digests
|
|
10
10
|
|
|
11
|
-
def initialize(candidates:, map:, limit:, diagnostics:)
|
|
11
|
+
def initialize(candidates:, map:, limit:, diagnostics:, root: nil)
|
|
12
12
|
@candidates = candidates
|
|
13
13
|
@map = map
|
|
14
14
|
@limit = limit
|
|
15
15
|
@diagnostics = diagnostics
|
|
16
|
+
@root = root
|
|
16
17
|
@skipped_subjects = {}
|
|
17
18
|
@selected = {}
|
|
18
19
|
@unscoped_digests = {}
|
|
@@ -29,6 +30,7 @@ module Saboteur
|
|
|
29
30
|
return @selected[subject] = []
|
|
30
31
|
end
|
|
31
32
|
|
|
33
|
+
warn_boot_once(subject)
|
|
32
34
|
list = @candidates.for(subject)
|
|
33
35
|
# The full pre-scope digests let IgnoreList#apply tell "mutation gone"
|
|
34
36
|
# from "mutation scoped out of this run" (#284).
|
|
@@ -48,24 +50,33 @@ module Saboteur
|
|
|
48
50
|
|
|
49
51
|
private
|
|
50
52
|
|
|
53
|
+
# A note, not a verdict (#575): every mutant still runs and scores.
|
|
54
|
+
def warn_boot_once(subject)
|
|
55
|
+
return unless @map.boot_once?(subject)
|
|
56
|
+
|
|
57
|
+
@diagnostics.warn("#{subject.id}: verify a survivor here by mutating the file on disk\n " \
|
|
58
|
+
"covered by every test and run at boot: mutants apply after boot, so a survivor " \
|
|
59
|
+
"can be a boot-once effect the suite cannot re-observe (#575)")
|
|
60
|
+
end
|
|
61
|
+
|
|
51
62
|
# A scoped no-coverage skip names the specs the run loaded: coverage
|
|
52
63
|
# living in a sibling spec file is otherwise invisible, and the false
|
|
53
64
|
# read is "untestable" when the truth is "not loaded" (#636). Names
|
|
54
65
|
# at most five, like the other selection notes.
|
|
55
66
|
def skip_announcement(subject, reason)
|
|
56
|
-
line = "#{subject.id}: skipped (#{reason}); its change is not verified by this run"
|
|
67
|
+
line = "#{subject.id} (#{subject.location(@root)}): skipped (#{reason}); its change is not verified by this run"
|
|
57
68
|
return line unless reason == "no coverage"
|
|
58
69
|
|
|
59
70
|
loaded = @map.tests.map { |id| id.split("[").first.delete_prefix("./") }.uniq.sort
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
"#{line} - loaded specs: #{names}#{more}; " \
|
|
71
|
+
names = loaded.empty? ? "none" : Text.name_list(loaded)
|
|
72
|
+
"#{line} - loaded specs: #{names}; " \
|
|
63
73
|
"other spec files may cover it, pass --test to load them"
|
|
64
74
|
end
|
|
65
75
|
|
|
66
76
|
# Load-only code is not "no coverage": it ran, just never inside a
|
|
67
77
|
# test, and no forked test can re-run it (see CoverageMap::RAN_AT_LOAD).
|
|
68
78
|
def skip_reason_for(subject)
|
|
79
|
+
return Selection::CoverageMap::PRELOADED if @map.preloaded?(subject.file)
|
|
69
80
|
return "no coverage" unless @map.ran_at_load?(subject.file, subject.coverage_lines)
|
|
70
81
|
|
|
71
82
|
Selection::CoverageMap::RAN_AT_LOAD
|
|
@@ -109,6 +109,9 @@ module Saboteur
|
|
|
109
109
|
end
|
|
110
110
|
|
|
111
111
|
def proposals(source, subject)
|
|
112
|
+
# The operator visits recurse over the subject's body; refuse a
|
|
113
|
+
# body nested too deeply instead of crashing the run (#719).
|
|
114
|
+
DeepSource.check!(subject.node, subject.file, limit: DeepSource::MAX_MUTATION_DEPTH)
|
|
112
115
|
proposals = source.mutations_for(subject)
|
|
113
116
|
if proposals.empty?
|
|
114
117
|
@diagnostics.record_generation(subject, source: source_name(source), reason: "empty_generation")
|
|
@@ -142,7 +145,7 @@ module Saboteur
|
|
|
142
145
|
|
|
143
146
|
[mutation, text]
|
|
144
147
|
rescue ArgumentError => e
|
|
145
|
-
@diagnostics.warn("#{subject.id}: dropped #{mutation.source} mutation
|
|
148
|
+
@diagnostics.warn("#{subject.id}: dropped #{mutation.source} mutation\n #{e.message}")
|
|
146
149
|
record_drop(subject, mutation, "invalid_edit")
|
|
147
150
|
end
|
|
148
151
|
|