saboteur 0.7.0 → 0.8.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +772 -0
- data/README.md +221 -1072
- data/lib/saboteur/attribute_finder.rb +113 -0
- data/lib/saboteur/cli/exit_status.rb +32 -1
- data/lib/saboteur/cli/help_options.rb +2 -1
- data/lib/saboteur/cli/history_command.rb +135 -0
- data/lib/saboteur/cli/presets.rb +47 -3
- data/lib/saboteur/cli/run_budget_options.rb +3 -1
- data/lib/saboteur/cli/selection_options.rb +31 -0
- data/lib/saboteur/cli/shard_options.rb +17 -0
- data/lib/saboteur/cli/source_set.rb +35 -0
- data/lib/saboteur/cli/survivors_command.rb +157 -0
- data/lib/saboteur/cli.rb +74 -55
- data/lib/saboteur/config.rb +102 -6
- data/lib/saboteur/config_file.rb +19 -7
- data/lib/saboteur/constant_reads.rb +182 -0
- data/lib/saboteur/constant_subject.rb +70 -0
- data/lib/saboteur/constant_subject_finder.rb +281 -0
- data/lib/saboteur/coverage_reads.rb +29 -0
- data/lib/saboteur/custom_map.rb +12 -1
- data/lib/saboteur/dirty_tree_warning.rb +130 -0
- data/lib/saboteur/doctor.rb +13 -0
- data/lib/saboteur/engine/baseline_guard.rb +135 -21
- data/lib/saboteur/engine/boot.rb +11 -3
- data/lib/saboteur/engine/bytecode_equivalence.rb +17 -1
- data/lib/saboteur/engine/candidate_selection.rb +20 -0
- data/lib/saboteur/engine/candidates.rb +43 -3
- data/lib/saboteur/engine/coverage_store.rb +58 -14
- data/lib/saboteur/engine/defined_order_prefix.rb +43 -0
- data/lib/saboteur/engine/diagnostics.rb +19 -1
- data/lib/saboteur/engine/distribution.rb +10 -0
- data/lib/saboteur/engine/early_stop_recording.rb +3 -2
- data/lib/saboteur/engine/gated_run.rb +37 -0
- data/lib/saboteur/engine/load_coverage.rb +63 -0
- data/lib/saboteur/engine/out_of_scope_guard.rb +21 -0
- data/lib/saboteur/engine/pipelines.rb +115 -0
- data/lib/saboteur/engine/result_cache.rb +65 -8
- data/lib/saboteur/engine/run_budget.rb +32 -2
- data/lib/saboteur/engine/sandbox_policy.rb +59 -3
- data/lib/saboteur/engine/scheduler.rb +131 -17
- data/lib/saboteur/engine/selection_announcement.rb +59 -0
- data/lib/saboteur/engine/shard.rb +69 -0
- data/lib/saboteur/engine/subject_selection.rb +380 -21
- data/lib/saboteur/engine/test_guess.rb +407 -27
- data/lib/saboteur/engine/unchanged_syntax.rb +17 -2
- data/lib/saboteur/engine.rb +103 -129
- data/lib/saboteur/execution/external_runner.rb +489 -0
- data/lib/saboteur/execution/fiddle_loader.rb +34 -0
- data/lib/saboteur/execution/marker_sweep.rb +80 -0
- data/lib/saboteur/execution/peak_memory.rb +28 -0
- data/lib/saboteur/execution/process_table.rb +19 -3
- data/lib/saboteur/execution/process_tree.rb +65 -5
- data/lib/saboteur/execution/reevaluator.rb +75 -12
- data/lib/saboteur/execution/runner.rb +46 -7
- data/lib/saboteur/execution/sandbox.rb +25 -6
- data/lib/saboteur/execution/seatbelt.rb +233 -0
- data/lib/saboteur/execution/supervisor.rb +29 -11
- data/lib/saboteur/execution/test_command_shape.rb +149 -0
- data/lib/saboteur/execution/worker_pool.rb +23 -1
- data/lib/saboteur/frameworks/sqlite_memory_database.rb +79 -0
- data/lib/saboteur/gates.rb +43 -2
- data/lib/saboteur/help.rb +60 -9
- data/lib/saboteur/ignore_list.rb +29 -6
- data/lib/saboteur/integration/rspec.rb +223 -5
- data/lib/saboteur/mutation_source/argument_removal.rb +1 -1
- data/lib/saboteur/mutation_source/argument_swap.rb +54 -0
- data/lib/saboteur/mutation_source/builtin.rb +28 -5
- data/lib/saboteur/mutation_source/chain_removal.rb +1 -1
- data/lib/saboteur/mutation_source/condition_negation.rb +1 -1
- data/lib/saboteur/mutation_source/constant_literals.rb +66 -0
- data/lib/saboteur/mutation_source/element_removal.rb +73 -0
- data/lib/saboteur/mutation_source/endpoint_replacement.rb +1 -1
- data/lib/saboteur/mutation_source/entry_edits.rb +44 -0
- data/lib/saboteur/mutation_source/filter_replacement.rb +1 -1
- data/lib/saboteur/mutation_source/hazard_guard.rb +189 -0
- data/lib/saboteur/mutation_source/literal_replacement.rb +1 -1
- data/lib/saboteur/mutation_source/method_swap.rb +148 -0
- data/lib/saboteur/mutation_source/operator_assignment.rb +1 -1
- data/lib/saboteur/mutation_source/operator_examples.rb +4 -1
- data/lib/saboteur/mutation_source/operator_registry.rb +5 -2
- data/lib/saboteur/mutation_source/operator_visit.rb +1 -1
- data/lib/saboteur/mutation_source/pair_removal.rb +13 -31
- data/lib/saboteur/mutation_source/range_replacement.rb +1 -1
- data/lib/saboteur/mutation_source/regex_replacement.rb +93 -0
- data/lib/saboteur/preview.rb +125 -12
- data/lib/saboteur/refinement_activations.rb +60 -0
- data/lib/saboteur/reporter/agent.rb +9 -1
- data/lib/saboteur/reporter/agent_commands.rb +10 -3
- data/lib/saboteur/reporter/agent_incomplete.rb +31 -5
- data/lib/saboteur/reporter/agent_summary.rb +4 -3
- data/lib/saboteur/reporter/findings.rb +5 -2
- data/lib/saboteur/reporter/history_policy.rb +11 -0
- data/lib/saboteur/reporter/history_reader.rb +47 -0
- data/lib/saboteur/reporter/html.rb +26 -4
- data/lib/saboteur/reporter/mutation_text.rb +6 -0
- data/lib/saboteur/reporter/stryker.rb +27 -3
- data/lib/saboteur/reporter/survivor_classification.rb +101 -0
- data/lib/saboteur/reporter/survivor_list.rb +15 -9
- data/lib/saboteur/reporter/terminal.rb +56 -8
- data/lib/saboteur/result.rb +19 -3
- data/lib/saboteur/selection/coverage_map.rb +120 -10
- data/lib/saboteur/selection/dependency_shield.rb +80 -0
- data/lib/saboteur/selection/external_test_map.rb +27 -0
- data/lib/saboteur/selection/git_diff.rb +93 -11
- data/lib/saboteur/selection/layout_only_lines.rb +80 -0
- data/lib/saboteur/selection/recorder.rb +22 -4
- data/lib/saboteur/selection/source_scope.rb +22 -0
- data/lib/saboteur/subject.rb +42 -7
- data/lib/saboteur/subject_finder.rb +173 -29
- data/lib/saboteur/version.rb +1 -1
- data/lib/saboteur/working_tree.rb +38 -15
- data/lib/saboteur.rb +14 -0
- metadata +39 -16
data/lib/saboteur/ignore_list.rb
CHANGED
|
@@ -21,7 +21,15 @@ module Saboteur
|
|
|
21
21
|
#
|
|
22
22
|
# reason is mandatory; a file with a missing or blank reason does not load.
|
|
23
23
|
class IgnoreList
|
|
24
|
-
|
|
24
|
+
# Triage states beyond plain ignoring (#155): accepted_equivalent and
|
|
25
|
+
# wont_fix suppress the survivor's exit failure (it still counts in the
|
|
26
|
+
# score), todo records the decision to fix and lets the survivor count.
|
|
27
|
+
STATES = %w[accepted_equivalent wont_fix todo].freeze
|
|
28
|
+
|
|
29
|
+
Entry = Data.define(:subject, :mutation, :reason, :state) do
|
|
30
|
+
# Entries built without a state predate triage states: they accept.
|
|
31
|
+
def initialize(subject:, mutation:, reason:, state: "accepted_equivalent") = super
|
|
32
|
+
end
|
|
25
33
|
|
|
26
34
|
# Collects the entries a saboteur_ignore.rb file declares.
|
|
27
35
|
class Collector
|
|
@@ -31,16 +39,23 @@ module Saboteur
|
|
|
31
39
|
@entries = []
|
|
32
40
|
end
|
|
33
41
|
|
|
34
|
-
def mutation(digest = nil, subject: nil, reason: nil)
|
|
42
|
+
def mutation(digest = nil, subject: nil, reason: nil, state: nil)
|
|
35
43
|
line = caller_locations(1, 1).first.lineno
|
|
36
44
|
values = { "subject" => subject, "mutation" => digest, "reason" => reason }
|
|
37
45
|
missing = values.select { |_, value| value.to_s.strip.empty? }.keys
|
|
38
46
|
raise ConfigError, "#{line}: entry is missing #{missing.join(', ')}" if missing.any?
|
|
39
47
|
|
|
48
|
+
state = state.to_s.strip
|
|
49
|
+
state = "accepted_equivalent" if state.empty?
|
|
50
|
+
unless STATES.include?(state)
|
|
51
|
+
raise ConfigError, "#{line}: unknown state #{state.inspect} - use one of #{STATES.join(', ')}"
|
|
52
|
+
end
|
|
53
|
+
|
|
40
54
|
@entries << Entry.new(
|
|
41
55
|
subject: subject.to_s.strip,
|
|
42
56
|
mutation: digest.to_s.strip,
|
|
43
|
-
reason: reason.to_s.strip
|
|
57
|
+
reason: reason.to_s.strip,
|
|
58
|
+
state: state
|
|
44
59
|
)
|
|
45
60
|
end
|
|
46
61
|
end
|
|
@@ -96,9 +111,13 @@ module Saboteur
|
|
|
96
111
|
@index = entries.to_h { |entry| [[entry.subject, entry.mutation], entry] }
|
|
97
112
|
end
|
|
98
113
|
|
|
99
|
-
# True when this result is a survivor the list accepts
|
|
114
|
+
# True when this result is a survivor the list accepts: a todo entry
|
|
115
|
+
# records the triage but keeps the survivor counting as unignored.
|
|
100
116
|
def accepts?(result)
|
|
101
|
-
|
|
117
|
+
return false unless result.survived?
|
|
118
|
+
|
|
119
|
+
entry = @index[[result.subject.id, result.digest]]
|
|
120
|
+
!entry.nil? && entry.state != "todo"
|
|
102
121
|
end
|
|
103
122
|
|
|
104
123
|
# Marks matching survivors as ignored. Returns warnings for stale entries:
|
|
@@ -114,7 +133,11 @@ module Saboteur
|
|
|
114
133
|
next unless entry
|
|
115
134
|
|
|
116
135
|
seen[key] = true
|
|
117
|
-
|
|
136
|
+
next unless result.survived?
|
|
137
|
+
|
|
138
|
+
result.triage_state = entry.state
|
|
139
|
+
result.triage_reason = entry.reason
|
|
140
|
+
result.ignored = entry.reason if entry.state != "todo"
|
|
118
141
|
end
|
|
119
142
|
stale_warnings(results, seen, unscoped_digests)
|
|
120
143
|
end
|
|
@@ -1,6 +1,8 @@
|
|
|
1
1
|
# SPDX-License-Identifier: MIT
|
|
2
2
|
# frozen_string_literal: true
|
|
3
3
|
|
|
4
|
+
require "delegate"
|
|
5
|
+
|
|
4
6
|
module Saboteur
|
|
5
7
|
module Integration
|
|
6
8
|
# Loads spec files once in the parent. In a child, narrows the already
|
|
@@ -8,20 +10,44 @@ module Saboteur
|
|
|
8
10
|
# reporter, so nothing is printed and no status file is written.
|
|
9
11
|
class RSpec < Base
|
|
10
12
|
def setup
|
|
11
|
-
|
|
13
|
+
require_framework
|
|
12
14
|
::RSpec::Core::Runner.disable_autorun!
|
|
13
15
|
@runner = ::RSpec::Core::Runner.new(configuration_options)
|
|
14
|
-
|
|
16
|
+
load_output = SetupOutputCapture.new($stdout)
|
|
17
|
+
begin
|
|
18
|
+
@runner.setup($stderr, load_output)
|
|
19
|
+
rescue SystemExit => e
|
|
20
|
+
load_output.replay
|
|
21
|
+
# A spec file calling exit at load makes RSpec print "While
|
|
22
|
+
# loading <file> ..." (which names the file) and re-raise.
|
|
23
|
+
# Uncaught, the parent exits with the spec's status: exit(0)
|
|
24
|
+
# would end the run silently green (#411).
|
|
25
|
+
raise Error, "a spec file called exit while loading (status #{e.status})"
|
|
26
|
+
end
|
|
15
27
|
configuration.order = "rand:#{config.fetch('seed', 1)}"
|
|
16
|
-
|
|
28
|
+
if quit_while_loading?(world)
|
|
29
|
+
# RSpec's load dump ends in the saboteur frames the load was
|
|
30
|
+
# triggered through and a "No examples found." from the reporter;
|
|
31
|
+
# both contradict the tool's own message. Print the suite's error
|
|
32
|
+
# without them (#608).
|
|
33
|
+
$stdout.write LoadErrorReport.filter(load_output.text, verbose: config.fetch("verbose", false))
|
|
34
|
+
raise Error, load_failure_message(load_output.failed_files, load_output.text)
|
|
35
|
+
end
|
|
36
|
+
load_output.replay
|
|
17
37
|
|
|
18
38
|
refuse_unintended_focus
|
|
19
39
|
end
|
|
20
40
|
|
|
21
|
-
# RSpec only adds its default
|
|
41
|
+
# RSpec only adds its default path when $0 is `rspec`, so saboteur adds
|
|
42
|
+
# it when RSpec's own parse of the args, .rspec and SPEC_OPTS names no
|
|
43
|
+
# file or directory to run. Parsing, not a look at the tokens, keeps an
|
|
44
|
+
# option value such as `--require spec_helper` from passing as a path,
|
|
45
|
+
# and a --default-path from .rspec wins over "spec".
|
|
22
46
|
def rspec_args
|
|
47
|
+
require_framework
|
|
23
48
|
args = Array(config.fetch("rspec_args", []))
|
|
24
|
-
|
|
49
|
+
options = ::RSpec::Core::ConfigurationOptions.new(args).options
|
|
50
|
+
Array(options[:files_or_directories_to_run]).empty? ? args + [options[:default_path] || "spec"] : args
|
|
25
51
|
end
|
|
26
52
|
|
|
27
53
|
# RSpec's own parser reads rspec_args and .rspec, so option values such
|
|
@@ -34,6 +60,52 @@ module Saboteur
|
|
|
34
60
|
end
|
|
35
61
|
end
|
|
36
62
|
|
|
63
|
+
# The spec files the configured suite runs, as UTF-8 paths relative to
|
|
64
|
+
# root (a file outside it stays absolute), by
|
|
65
|
+
# RSpec's own rules: .rspec, rspec_args, the default path, --pattern
|
|
66
|
+
# and --exclude-pattern. RSpec applies the patterns only to the
|
|
67
|
+
# directories it scans, never to a file named explicitly, so a file
|
|
68
|
+
# saboteur names as a test must come from this list (#307). Nothing is
|
|
69
|
+
# loaded and nothing global changes: only the file-selection options
|
|
70
|
+
# reach a fresh Configuration, so --require, -I and formatters are
|
|
71
|
+
# ignored. A pattern set in Ruby (RSpec.configure in a spec helper) is
|
|
72
|
+
# not seen. RSpec resolves patterns against the working directory, so
|
|
73
|
+
# this reads from the root.
|
|
74
|
+
def configured_spec_files(root)
|
|
75
|
+
require_framework
|
|
76
|
+
from_directory(root) do
|
|
77
|
+
options = ::RSpec::Core::ConfigurationOptions.new(rspec_args).options
|
|
78
|
+
configuration = ::RSpec::Core::Configuration.new
|
|
79
|
+
%i[default_path pattern exclude_pattern].each do |name|
|
|
80
|
+
configuration.public_send(:"#{name}=", options[name]) if options[name]
|
|
81
|
+
end
|
|
82
|
+
configuration.files_or_directories_to_run = options[:files_or_directories_to_run]
|
|
83
|
+
# RSpec expands from the working directory, which can spell the
|
|
84
|
+
# root differently from its realpath (/var and /private/var).
|
|
85
|
+
prefix = Text.utf8_identity(File.join(Dir.pwd, ""))
|
|
86
|
+
configuration.files_to_run.map { |file| Text.utf8_identity(file).delete_prefix(prefix) }
|
|
87
|
+
end
|
|
88
|
+
end
|
|
89
|
+
|
|
90
|
+
# A missing rspec-core is the target's setup error, not a saboteur
|
|
91
|
+
# defect (#482): refuse with the remedy instead of crashing into the
|
|
92
|
+
# internal-error report. It is a usage error (#610): ConfigError maps
|
|
93
|
+
# to exit 2, and the remedy names docs/minitest.md rather than
|
|
94
|
+
# c.integration - rspec is the only integration, so pointing at it
|
|
95
|
+
# sent Minitest users down a dead end. Only the framework's own LoadError converts -
|
|
96
|
+
# matched on the failed feature, not the message, so a missing
|
|
97
|
+
# rspec/core_ext does not pose as rspec/core; a LoadError the app's
|
|
98
|
+
# files raise keeps the crash route.
|
|
99
|
+
def require_framework
|
|
100
|
+
require "rspec/core"
|
|
101
|
+
rescue LoadError => e
|
|
102
|
+
raise unless e.path == "rspec/core"
|
|
103
|
+
|
|
104
|
+
raise ConfigError, "rspec is not in the bundle: the RSpec integration needs the rspec-core gem. " \
|
|
105
|
+
"Add it to the bundle's test group. saboteur's built-in integration supports RSpec only - " \
|
|
106
|
+
"see docs/minitest.md for other suites"
|
|
107
|
+
end
|
|
108
|
+
|
|
37
109
|
def all_tests
|
|
38
110
|
groups = world.ordered_example_groups.flat_map(&:descendants)
|
|
39
111
|
groups.flat_map { |group| world.filtered_examples[group] }
|
|
@@ -76,8 +148,60 @@ module Saboteur
|
|
|
76
148
|
failure_details: listener.failure_details)
|
|
77
149
|
end
|
|
78
150
|
|
|
151
|
+
SHOWN_LOAD_FAILURES = 5
|
|
152
|
+
|
|
153
|
+
# Ruby's Coverage refuses a second start, so a suite whose
|
|
154
|
+
# spec_helper auto-starts SimpleCov/SingleCov collides with
|
|
155
|
+
# saboteur's own Coverage.start and the load quits with an opaque
|
|
156
|
+
# "No examples found" (#645, real run: audited @ dbf84326). Name the
|
|
157
|
+
# suspect and both guard styles. The gem's backtrace line is the
|
|
158
|
+
# fallback: the RuntimeError's text is Coverage's own, whoever
|
|
159
|
+
# started it second.
|
|
160
|
+
COVERAGE_DOUBLE_START = /coverage measurement is already setup/i
|
|
161
|
+
# A gem backtrace frame (path:line) or the start call itself - never a
|
|
162
|
+
# bare name, which a spec FILE name like simplecov_config_spec.rb
|
|
163
|
+
# would false-positive on (Pod 3 QA probe, #645).
|
|
164
|
+
COVERAGE_GEM_FRAME = %r{(?:single_cov|simplecov)[\w.\-/]*\.rb:\d+|\b(?:SimpleCov\.start|SingleCov\.setup)}i
|
|
165
|
+
|
|
79
166
|
private
|
|
80
167
|
|
|
168
|
+
# rspec-core 3.11 added rspec_is_quitting (SyntaxError and SystemExit
|
|
169
|
+
# at load quit through it, not through wants_to_quit). The gemspec
|
|
170
|
+
# does not pin rspec, so a target app can carry an older rspec-core;
|
|
171
|
+
# ask before reading the newer flag (#378).
|
|
172
|
+
def quit_while_loading?(world)
|
|
173
|
+
world.wants_to_quit || (world.respond_to?(:rspec_is_quitting) && world.rspec_is_quitting)
|
|
174
|
+
end
|
|
175
|
+
|
|
176
|
+
# The quit reason RSpec keeps is only a flag; the failing files it
|
|
177
|
+
# printed are what a user needs. Name them and the way out: a repo
|
|
178
|
+
# whose spec tree cannot load whole (an engine's dummy app, an
|
|
179
|
+
# env-dependent integration spec) must narrow what loads (#378).
|
|
180
|
+
def load_failure_message(failed_files, captured_text = nil)
|
|
181
|
+
message = +"rspec quit while loading spec files"
|
|
182
|
+
if failed_files.any?
|
|
183
|
+
shown = failed_files.first(SHOWN_LOAD_FAILURES)
|
|
184
|
+
rest = failed_files.size - shown.size
|
|
185
|
+
message << ": #{failed_files.size} failed to load:"
|
|
186
|
+
shown.each { |file| message << "\n #{file}" }
|
|
187
|
+
message << "\n ... and #{rest} more" if rest.positive?
|
|
188
|
+
end
|
|
189
|
+
message << "\nExclude the broken subtrees in config/saboteur.rb, e.g. " \
|
|
190
|
+
'c.rspec_args = ["--exclude-pattern", "spec/dummy/**/*_spec.rb"], or name the ' \
|
|
191
|
+
"suite with --test. Rails engines with an embedded dummy app (spec/dummy) need " \
|
|
192
|
+
'this; see README "Rails engines and dummy apps".'
|
|
193
|
+
if coverage_suspect?(captured_text)
|
|
194
|
+
message << "\nSuspect: your suite starts its own Coverage (saboteur starts one itself). " \
|
|
195
|
+
"Guard it - if ENV[\"COVERAGE\"], or SingleCov's conditional setup API - " \
|
|
196
|
+
"and see docs/recipes/README.md#simplecov."
|
|
197
|
+
end
|
|
198
|
+
message
|
|
199
|
+
end
|
|
200
|
+
|
|
201
|
+
def coverage_suspect?(captured_text)
|
|
202
|
+
captured_text&.match?(COVERAGE_DOUBLE_START) || captured_text&.match?(COVERAGE_GEM_FRAME) || false
|
|
203
|
+
end
|
|
204
|
+
|
|
81
205
|
def refuse_unintended_focus
|
|
82
206
|
return if config["allow_focus"]
|
|
83
207
|
|
|
@@ -96,6 +220,12 @@ module Saboteur
|
|
|
96
220
|
options.options[:full_description] ? keys + [:full_description] : keys
|
|
97
221
|
end
|
|
98
222
|
|
|
223
|
+
# A real run already starts in the root; only a caller elsewhere (a
|
|
224
|
+
# spec) changes directory, and only for the block.
|
|
225
|
+
def from_directory(root, &)
|
|
226
|
+
File.realpath(Dir.pwd) == File.realpath(root) ? yield : Dir.chdir(root, &)
|
|
227
|
+
end
|
|
228
|
+
|
|
99
229
|
def world
|
|
100
230
|
::RSpec.world
|
|
101
231
|
end
|
|
@@ -183,6 +313,94 @@ module Saboteur
|
|
|
183
313
|
end
|
|
184
314
|
end
|
|
185
315
|
|
|
316
|
+
# RSpec prints each load failure ("An error occurred while loading
|
|
317
|
+
# ...") to the output stream during setup and only marks
|
|
318
|
+
# world.wants_to_quit. Capture the stream while passing everything
|
|
319
|
+
# through, so the tool error can name the files (#378).
|
|
320
|
+
class SetupOutputCapture < SimpleDelegator
|
|
321
|
+
LOAD_FAILURE = [
|
|
322
|
+
/^An error occurred while loading (\S+?)\.?(?: - Did you mean\?)?$/,
|
|
323
|
+
# SyntaxError and SystemExit use their own wording (#378).
|
|
324
|
+
/^While loading (\S+?) an? `(?:raise SyntaxError|exit)`/
|
|
325
|
+
].freeze
|
|
326
|
+
|
|
327
|
+
def initialize(io)
|
|
328
|
+
super
|
|
329
|
+
@captured = +""
|
|
330
|
+
end
|
|
331
|
+
|
|
332
|
+
# Buffer only: the caller decides after setup whether the captured
|
|
333
|
+
# text goes out as it is (a clean load), filtered (a load quit,
|
|
334
|
+
# #608) or verbatim on a SystemExit. Passing through during setup
|
|
335
|
+
# would print what a later filter needs to drop.
|
|
336
|
+
def write(text)
|
|
337
|
+
@replayed ? __getobj__.write(text) : @captured << text
|
|
338
|
+
self
|
|
339
|
+
end
|
|
340
|
+
|
|
341
|
+
def print(*args)
|
|
342
|
+
write(args.join)
|
|
343
|
+
end
|
|
344
|
+
|
|
345
|
+
def puts(*args)
|
|
346
|
+
write("#{args.flatten.join("\n")}\n")
|
|
347
|
+
end
|
|
348
|
+
|
|
349
|
+
def replay
|
|
350
|
+
__getobj__.write(@captured)
|
|
351
|
+
# RSpec retains this stream for later suite-hook errors.
|
|
352
|
+
@replayed = true
|
|
353
|
+
end
|
|
354
|
+
|
|
355
|
+
def text = @captured
|
|
356
|
+
|
|
357
|
+
def failed_files
|
|
358
|
+
LOAD_FAILURE.flat_map { |pattern| @captured.scan(pattern) }.flatten
|
|
359
|
+
end
|
|
360
|
+
end
|
|
361
|
+
|
|
362
|
+
# The load dump RSpec prints when a spec file raises at load, trimmed
|
|
363
|
+
# to what the user acts on: the first error with the suite's own
|
|
364
|
+
# frames. The saboteur frames the load was triggered through and the
|
|
365
|
+
# reporter's "No examples found." (nothing ran because nothing loaded)
|
|
366
|
+
# read as saboteur crashing and contradict the tool's message; both
|
|
367
|
+
# stay under --verbose (#608).
|
|
368
|
+
class LoadErrorReport
|
|
369
|
+
HEADER = /^An error occurred while loading /
|
|
370
|
+
# The running tool's install root (a checkout or an unpacked gem),
|
|
371
|
+
# with separator. Only frames under it are saboteur's: the same
|
|
372
|
+
# lib/saboteur/ layout under ANOTHER root is the target's vendored
|
|
373
|
+
# copy or a dogfooded checkout, and those frames are the suite's
|
|
374
|
+
# (#608).
|
|
375
|
+
TOOL_ROOT = "#{File.expand_path('../../..', __dir__)}/".freeze
|
|
376
|
+
private_constant :HEADER, :TOOL_ROOT
|
|
377
|
+
|
|
378
|
+
def self.filter(text, verbose: false)
|
|
379
|
+
return text if verbose
|
|
380
|
+
|
|
381
|
+
lines = text.lines.reject { |line| line.strip == "No examples found." || saboteur_frame?(line) }
|
|
382
|
+
heads = lines.each_index.select { |index| lines[index].match?(HEADER) }
|
|
383
|
+
return lines.join if heads.size <= 1
|
|
384
|
+
|
|
385
|
+
kept = lines[0...heads[1]]
|
|
386
|
+
kept.pop while kept.last&.strip == ""
|
|
387
|
+
omitted = heads.size - 1
|
|
388
|
+
kept << "(#{omitted} more load error#{'s' if omitted > 1} omitted; rerun with --verbose for the full list)\n"
|
|
389
|
+
kept.join
|
|
390
|
+
end
|
|
391
|
+
|
|
392
|
+
# A backtrace frame inside the tool's install root. Relative frames
|
|
393
|
+
# (the suite's own "./spec/...") are never the tool's.
|
|
394
|
+
def self.saboteur_frame?(line)
|
|
395
|
+
line = line.strip
|
|
396
|
+
return false unless line.start_with?("# ")
|
|
397
|
+
|
|
398
|
+
path = line.sub(/\A#\s+/, "").sub(/:\d+:in .*\z/, "")
|
|
399
|
+
path.start_with?("/") && File.expand_path(path).start_with?(TOOL_ROOT)
|
|
400
|
+
end
|
|
401
|
+
private_class_method :saboteur_frame?
|
|
402
|
+
end
|
|
403
|
+
|
|
186
404
|
class Listener
|
|
187
405
|
EVENTS = %i[example_started example_finished example_failed].freeze
|
|
188
406
|
|
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
# SPDX-License-Identifier: MIT
|
|
2
|
+
# frozen_string_literal: true
|
|
3
|
+
|
|
4
|
+
require_relative "base"
|
|
5
|
+
require_relative "operator_visit"
|
|
6
|
+
|
|
7
|
+
module Saboteur
|
|
8
|
+
module MutationSource
|
|
9
|
+
# Swap two explicit positional arguments without moving their separators.
|
|
10
|
+
# Order-insensitive calls (includes, attr_reader, validates) can be equivalent; arid patterns can exclude them.
|
|
11
|
+
class ArgumentSwap < Base
|
|
12
|
+
include OperatorVisit
|
|
13
|
+
|
|
14
|
+
EXCLUDED = [Prism::KeywordHashNode, Prism::SplatNode, Prism::ForwardingArgumentsNode].freeze
|
|
15
|
+
|
|
16
|
+
def initialize(arid: []) # rubocop:disable Lint/MissingSuper
|
|
17
|
+
@arid = arid.map { |pattern| Regexp.new(pattern) }
|
|
18
|
+
end
|
|
19
|
+
|
|
20
|
+
def name
|
|
21
|
+
"argument_swap"
|
|
22
|
+
end
|
|
23
|
+
|
|
24
|
+
private
|
|
25
|
+
|
|
26
|
+
def mutate_node(node, subject, found)
|
|
27
|
+
return unless node.is_a?(Prism::CallNode) && node.opening_loc&.slice == "("
|
|
28
|
+
|
|
29
|
+
arguments = node.arguments&.arguments
|
|
30
|
+
return unless arguments&.size == 2
|
|
31
|
+
return if arguments.any? { |argument| excluded?(argument) }
|
|
32
|
+
|
|
33
|
+
left, right = arguments
|
|
34
|
+
return if left.slice == right.slice
|
|
35
|
+
|
|
36
|
+
# Prism forwards location offsets on nodes; removing .location here is equivalent.
|
|
37
|
+
edits = [[left, right], [right, left]].map do |original, replacement|
|
|
38
|
+
Edit.new(file: subject.file, start_byte: original.location.start_offset,
|
|
39
|
+
end_byte: original.location.end_offset, replacement: replacement.slice)
|
|
40
|
+
end
|
|
41
|
+
found << Mutation.new(subject_id: subject.id, operator: name, edits: edits, source: name)
|
|
42
|
+
end
|
|
43
|
+
|
|
44
|
+
def excluded?(argument)
|
|
45
|
+
EXCLUDED.any? { |type| argument.is_a?(type) } || heredoc?(argument)
|
|
46
|
+
end
|
|
47
|
+
|
|
48
|
+
def heredoc?(node)
|
|
49
|
+
(node.respond_to?(:opening_loc) && node.opening_loc&.slice&.start_with?("<<")) ||
|
|
50
|
+
node.compact_child_nodes.any? { |child| heredoc?(child) }
|
|
51
|
+
end
|
|
52
|
+
end
|
|
53
|
+
end
|
|
54
|
+
end
|
|
@@ -6,8 +6,10 @@ require_relative "base"
|
|
|
6
6
|
require_relative "filter_replacement"
|
|
7
7
|
require_relative "literal_replacement"
|
|
8
8
|
require_relative "argument_removal"
|
|
9
|
+
require_relative "argument_swap"
|
|
9
10
|
require_relative "chain_removal"
|
|
10
11
|
require_relative "pair_removal"
|
|
12
|
+
require_relative "element_removal"
|
|
11
13
|
require_relative "endpoint_replacement"
|
|
12
14
|
require_relative "operator_registry"
|
|
13
15
|
require_relative "range_replacement"
|
|
@@ -16,6 +18,7 @@ require_relative "condition_negation"
|
|
|
16
18
|
require_relative "method_swap"
|
|
17
19
|
require_relative "operand_replacement"
|
|
18
20
|
require_relative "interpolation_removal"
|
|
21
|
+
require_relative "regex_replacement"
|
|
19
22
|
|
|
20
23
|
module Saboteur
|
|
21
24
|
module MutationSource
|
|
@@ -63,7 +66,9 @@ module Saboteur
|
|
|
63
66
|
@filter_replacement = FilterReplacement.new(arid: arid)
|
|
64
67
|
@literal_replacement = LiteralReplacement.new(arid: arid)
|
|
65
68
|
@argument_removal = ArgumentRemoval.new(arid: arid)
|
|
69
|
+
@argument_swap = ArgumentSwap.new(arid: arid)
|
|
66
70
|
@pair_removal = PairRemoval.new(arid: arid)
|
|
71
|
+
@element_removal = ElementRemoval.new(arid: arid)
|
|
67
72
|
@chain_removal = ChainRemoval.new(arid: arid)
|
|
68
73
|
@endpoint_replacement = EndpointReplacement.new(arid: arid)
|
|
69
74
|
@range_replacement = RangeReplacement.new(arid: arid)
|
|
@@ -72,6 +77,7 @@ module Saboteur
|
|
|
72
77
|
@method_swap = MethodSwap.new(arid: arid)
|
|
73
78
|
@operand_replacement = OperandReplacement.new(arid: arid)
|
|
74
79
|
@interpolation_removal = InterpolationRemoval.new(arid: arid)
|
|
80
|
+
@regex_replacement = RegexReplacement.new(arid: arid)
|
|
75
81
|
end
|
|
76
82
|
|
|
77
83
|
def name
|
|
@@ -79,7 +85,7 @@ module Saboteur
|
|
|
79
85
|
end
|
|
80
86
|
|
|
81
87
|
def mutations_for(subject)
|
|
82
|
-
body = subject.
|
|
88
|
+
body = subject.body_node
|
|
83
89
|
@subject = subject
|
|
84
90
|
@found = []
|
|
85
91
|
coarse(body)
|
|
@@ -93,9 +99,10 @@ module Saboteur
|
|
|
93
99
|
# The single-purpose sources run after the shared walk, in registration
|
|
94
100
|
# order; each filters its own operator through the same policy below.
|
|
95
101
|
def delegated
|
|
96
|
-
[@filter_replacement, @literal_replacement, @argument_removal, @pair_removal, @
|
|
102
|
+
[@filter_replacement, @literal_replacement, @argument_removal, @pair_removal, @element_removal,
|
|
103
|
+
@chain_removal,
|
|
97
104
|
@endpoint_replacement, @range_replacement, @operator_assignment, @condition_negation,
|
|
98
|
-
@method_swap, @operand_replacement, @interpolation_removal]
|
|
105
|
+
@method_swap, @operand_replacement, @interpolation_removal, @regex_replacement, @argument_swap]
|
|
99
106
|
end
|
|
100
107
|
|
|
101
108
|
private
|
|
@@ -117,7 +124,7 @@ module Saboteur
|
|
|
117
124
|
|
|
118
125
|
# Nothing but logging: replacing it changes nothing a test can see.
|
|
119
126
|
def nothing_to_remove?(statements)
|
|
120
|
-
statements.body.all? { |statement|
|
|
127
|
+
statements.body.all? { |statement| arid_statement?(statement) }
|
|
121
128
|
end
|
|
122
129
|
|
|
123
130
|
# A raise that the method's own rescue catches is absorbed by design. It
|
|
@@ -156,7 +163,7 @@ module Saboteur
|
|
|
156
163
|
def visit_statements(node)
|
|
157
164
|
statements = node.body
|
|
158
165
|
statements.each do |statement|
|
|
159
|
-
next if
|
|
166
|
+
next if arid_statement?(statement)
|
|
160
167
|
|
|
161
168
|
removable_here = statements.drop(1).any? && removable?(statement)
|
|
162
169
|
add("statement_removal", statement, "nil") if removable_here
|
|
@@ -256,6 +263,22 @@ module Saboteur
|
|
|
256
263
|
@arid.any? { |pattern| pattern.match?(source) }
|
|
257
264
|
end
|
|
258
265
|
|
|
266
|
+
# An arid call is noise only in itself. A block on that call is real
|
|
267
|
+
# work (a notifier wrapping a computation), so the statement still
|
|
268
|
+
# mutates and keeps its body out of nothing_to_remove?; the walk
|
|
269
|
+
# descends into the block through the normal visit below.
|
|
270
|
+
def arid_statement?(statement)
|
|
271
|
+
return false unless arid?(statement)
|
|
272
|
+
|
|
273
|
+
block = statement.is_a?(Prism::CallNode) ? statement.block : nil
|
|
274
|
+
body = block.is_a?(Prism::BlockNode) ? block.body : nil
|
|
275
|
+
# A do-block with rescue/ensure parses its body as a BeginNode, not
|
|
276
|
+
# a StatementsNode (#464): its clauses are real work, not noise.
|
|
277
|
+
return false if body && !body.is_a?(Prism::StatementsNode)
|
|
278
|
+
|
|
279
|
+
body.nil? || body.body.all? { |inner| arid_statement?(inner) }
|
|
280
|
+
end
|
|
281
|
+
|
|
259
282
|
# node_or_location: a Prism node or location (both have offsets and slice).
|
|
260
283
|
def add(operator, node_or_location, replacement, coarse: false)
|
|
261
284
|
location = node_or_location
|
|
@@ -0,0 +1,66 @@
|
|
|
1
|
+
# SPDX-License-Identifier: MIT
|
|
2
|
+
# frozen_string_literal: true
|
|
3
|
+
|
|
4
|
+
require "prism"
|
|
5
|
+
require_relative "../text"
|
|
6
|
+
require_relative "base"
|
|
7
|
+
require_relative "pair_removal"
|
|
8
|
+
require_relative "element_removal"
|
|
9
|
+
require_relative "operator_registry"
|
|
10
|
+
|
|
11
|
+
module Saboteur
|
|
12
|
+
module MutationSource
|
|
13
|
+
# Mutations for a constant hash or array literal (a ConstantSubject,
|
|
14
|
+
# issue #306): pair_removal on the hash's entries, element_removal on
|
|
15
|
+
# the array's elements. The other operators stay method-only. Every
|
|
16
|
+
# entry is proposed here; in --since mode the diff scope narrows the
|
|
17
|
+
# proposals to the changed entries, and a configured operator exclusion
|
|
18
|
+
# applies as usual.
|
|
19
|
+
class ConstantLiterals < Base
|
|
20
|
+
def initialize(operators: [], exclude_operators: [], arid: [])
|
|
21
|
+
super()
|
|
22
|
+
@operators = OperatorRegistry.select(operators, exclude_operators)
|
|
23
|
+
@arid = arid.map { |pattern| Regexp.new(pattern) }
|
|
24
|
+
@pair_removal = PairRemoval.new
|
|
25
|
+
@element_removal = ElementRemoval.new
|
|
26
|
+
end
|
|
27
|
+
|
|
28
|
+
def name
|
|
29
|
+
"builtin"
|
|
30
|
+
end
|
|
31
|
+
|
|
32
|
+
def mutations_for(subject)
|
|
33
|
+
literal = subject.literal_node
|
|
34
|
+
found = []
|
|
35
|
+
if literal.is_a?(Prism::HashNode) && @operators.include?("pair_removal")
|
|
36
|
+
found.concat(@pair_removal.mutations_for(subject, root: literal))
|
|
37
|
+
end
|
|
38
|
+
if literal.is_a?(Prism::ArrayNode) && @operators.include?("element_removal")
|
|
39
|
+
found.concat(@element_removal.mutations_for(subject, root: literal))
|
|
40
|
+
end
|
|
41
|
+
found.reject { |mutation| arid?(subject, mutation) }
|
|
42
|
+
end
|
|
43
|
+
|
|
44
|
+
private
|
|
45
|
+
|
|
46
|
+
# An arid entry (logging or metrics config) mutates nothing a test
|
|
47
|
+
# can see - the same rule the operators apply to statements inside a
|
|
48
|
+
# method, here at entry level because a constant has no statements.
|
|
49
|
+
def arid?(subject, mutation)
|
|
50
|
+
mutation.edits.any? do |edit|
|
|
51
|
+
# file_source is a binread: retag losslessly before the UTF-8
|
|
52
|
+
# pattern match, and keep the mutant when the bytes still cannot
|
|
53
|
+
# be judged - a crash here would hide it instead.
|
|
54
|
+
slice = Text.utf8_identity(subject.file_source[edit.start_byte...edit.end_byte])
|
|
55
|
+
@arid.any? { |pattern| safe_match?(pattern, slice) }
|
|
56
|
+
end
|
|
57
|
+
end
|
|
58
|
+
|
|
59
|
+
def safe_match?(pattern, slice)
|
|
60
|
+
pattern.match?(slice)
|
|
61
|
+
rescue Encoding::CompatibilityError, ArgumentError
|
|
62
|
+
false
|
|
63
|
+
end
|
|
64
|
+
end
|
|
65
|
+
end
|
|
66
|
+
end
|
|
@@ -0,0 +1,73 @@
|
|
|
1
|
+
# SPDX-License-Identifier: MIT
|
|
2
|
+
# frozen_string_literal: true
|
|
3
|
+
|
|
4
|
+
require "prism"
|
|
5
|
+
require_relative "base"
|
|
6
|
+
require_relative "entry_edits"
|
|
7
|
+
|
|
8
|
+
module Saboteur
|
|
9
|
+
module MutationSource
|
|
10
|
+
# Deletes one element from an array literal.
|
|
11
|
+
class ElementRemoval < Base
|
|
12
|
+
include EntryEdits
|
|
13
|
+
|
|
14
|
+
# Nested scopes (a def, a class) are separate subjects; their arrays
|
|
15
|
+
# are not part of the enclosing subject.
|
|
16
|
+
BOUNDARIES = [Prism::DefNode, Prism::ClassNode, Prism::ModuleNode, Prism::SingletonClassNode,
|
|
17
|
+
Prism::ParametersNode, Prism::BlockParametersNode].freeze
|
|
18
|
+
|
|
19
|
+
def initialize(arid: []) # rubocop:disable Lint/MissingSuper
|
|
20
|
+
@arid = arid.map { |pattern| Regexp.new(pattern) }
|
|
21
|
+
end
|
|
22
|
+
|
|
23
|
+
def name
|
|
24
|
+
"element_removal"
|
|
25
|
+
end
|
|
26
|
+
|
|
27
|
+
# root: the node to walk instead of the subject's method body (a
|
|
28
|
+
# constant's literal, see MutationSource::ConstantLiterals).
|
|
29
|
+
def mutations_for(subject, root: nil)
|
|
30
|
+
found = []
|
|
31
|
+
tokens = Prism.lex(subject.file_source).value.map(&:first)
|
|
32
|
+
visit(root || subject.body_node, subject, found, tokens)
|
|
33
|
+
found
|
|
34
|
+
end
|
|
35
|
+
|
|
36
|
+
private
|
|
37
|
+
|
|
38
|
+
def visit(node, subject, found, tokens)
|
|
39
|
+
return if node.nil? || BOUNDARIES.any? { |type| node.is_a?(type) }
|
|
40
|
+
|
|
41
|
+
if node.is_a?(Prism::StatementsNode)
|
|
42
|
+
# An arid statement (logging, metrics) mutates nothing a test can
|
|
43
|
+
# see; leave its arrays alone, like the other removal operators.
|
|
44
|
+
node.body.each do |child|
|
|
45
|
+
visit(child, subject, found, tokens) unless @arid.any? { |p| p.match?(child.slice) }
|
|
46
|
+
end
|
|
47
|
+
return
|
|
48
|
+
end
|
|
49
|
+
remove_elements(node, subject, found, tokens) if node.is_a?(Prism::ArrayNode)
|
|
50
|
+
node.compact_child_nodes.each { |child| visit(child, subject, found, tokens) }
|
|
51
|
+
end
|
|
52
|
+
|
|
53
|
+
def remove_elements(node, subject, found, tokens)
|
|
54
|
+
elements = node.elements
|
|
55
|
+
elements.each_with_index do |element, index|
|
|
56
|
+
# A splat's separator handling is fragile (`[*A, 2]` vs `[2, *A]`)
|
|
57
|
+
# and removing one can silently change what expands where; leave
|
|
58
|
+
# splats intact, like pair_removal does.
|
|
59
|
+
next if heredoc?(element) || element.is_a?(Prism::SplatNode)
|
|
60
|
+
|
|
61
|
+
edits = [delete_entry(subject, element.location)]
|
|
62
|
+
separator = entry_separator(elements, index, ending_for(node), tokens)
|
|
63
|
+
edits << delete_entry(subject, separator) if separator
|
|
64
|
+
found << Mutation.new(subject_id: subject.id, operator: name, edits: edits, source: name)
|
|
65
|
+
end
|
|
66
|
+
end
|
|
67
|
+
|
|
68
|
+
def ending_for(node)
|
|
69
|
+
node.closing_loc&.start_offset || node.location.end_offset
|
|
70
|
+
end
|
|
71
|
+
end
|
|
72
|
+
end
|
|
73
|
+
end
|