saboteur 0.8.0 → 0.8.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +115 -0
- data/README.md +34 -6
- data/lib/saboteur/attribute_finder.rb +113 -0
- data/lib/saboteur/cli/presets.rb +41 -1
- data/lib/saboteur/cli/run_budget_options.rb +3 -1
- data/lib/saboteur/cli/source_set.rb +0 -18
- data/lib/saboteur/cli.rb +4 -2
- data/lib/saboteur/config.rb +4 -1
- data/lib/saboteur/config_file.rb +12 -8
- data/lib/saboteur/custom_map.rb +12 -1
- data/lib/saboteur/dirty_tree_warning.rb +5 -3
- data/lib/saboteur/doctor.rb +4 -0
- data/lib/saboteur/engine/baseline_guard.rb +76 -10
- data/lib/saboteur/engine/boot.rb +7 -3
- data/lib/saboteur/engine/bytecode_equivalence.rb +17 -1
- data/lib/saboteur/engine/candidate_selection.rb +1 -0
- data/lib/saboteur/engine/coverage_store.rb +25 -3
- data/lib/saboteur/engine/diagnostics.rb +1 -1
- data/lib/saboteur/engine/gated_run.rb +3 -1
- data/lib/saboteur/engine/load_coverage.rb +18 -0
- data/lib/saboteur/engine/pipelines.rb +8 -0
- data/lib/saboteur/engine/run_budget.rb +5 -2
- data/lib/saboteur/engine/sandbox_policy.rb +21 -3
- data/lib/saboteur/engine/scheduler.rb +32 -24
- data/lib/saboteur/engine/selection_announcement.rb +8 -3
- data/lib/saboteur/engine/subject_selection.rb +115 -31
- data/lib/saboteur/engine/test_guess.rb +106 -16
- data/lib/saboteur/engine.rb +3 -2
- data/lib/saboteur/execution/external_runner.rb +18 -0
- data/lib/saboteur/execution/fiddle_loader.rb +34 -0
- data/lib/saboteur/execution/marker_sweep.rb +2 -1
- data/lib/saboteur/execution/process_table.rb +7 -3
- data/lib/saboteur/execution/reevaluator.rb +7 -0
- data/lib/saboteur/execution/sandbox.rb +10 -4
- data/lib/saboteur/execution/seatbelt.rb +10 -3
- data/lib/saboteur/execution/supervisor.rb +2 -1
- data/lib/saboteur/execution/test_command_shape.rb +149 -0
- data/lib/saboteur/frameworks/sqlite_memory_database.rb +32 -2
- data/lib/saboteur/integration/rspec.rb +99 -11
- data/lib/saboteur/mutation_source/argument_swap.rb +54 -0
- data/lib/saboteur/mutation_source/builtin.rb +3 -1
- data/lib/saboteur/mutation_source/hazard_guard.rb +94 -13
- data/lib/saboteur/mutation_source/operator_examples.rb +1 -0
- data/lib/saboteur/mutation_source/operator_registry.rb +1 -0
- data/lib/saboteur/preview.rb +5 -0
- data/lib/saboteur/reporter/agent_incomplete.rb +20 -5
- data/lib/saboteur/reporter/terminal.rb +16 -1
- data/lib/saboteur/selection/coverage_map.rb +17 -2
- data/lib/saboteur/version.rb +1 -1
- data/lib/saboteur.rb +1 -0
- metadata +5 -15
|
@@ -0,0 +1,149 @@
|
|
|
1
|
+
# SPDX-License-Identifier: MIT
|
|
2
|
+
# frozen_string_literal: true
|
|
3
|
+
|
|
4
|
+
require "shellwords"
|
|
5
|
+
|
|
6
|
+
module Saboteur
|
|
7
|
+
module Execution
|
|
8
|
+
# Shape check for --test-command strings (#611). ruby executes only the
|
|
9
|
+
# FIRST script named on its command line; extra .rb arguments become
|
|
10
|
+
# ARGV, so "ruby -Itest a_test.rb b_test.rb" exercises a fraction of the
|
|
11
|
+
# suite and the unrun files' mutants silently survive. Only that exact
|
|
12
|
+
# shape warns: the command is split on shell operators (&&, ||, ;, |,
|
|
13
|
+
# redirections excluded with their targets) and each simple command inspected on
|
|
14
|
+
# its own; option operands (-r helper, -I dir, -C dir, -E/-F/-K values)
|
|
15
|
+
# are parsed out and -e/--eval ends the search (the inline script
|
|
16
|
+
# decides what it loads); the script must look like a test file, so a
|
|
17
|
+
# custom runner taking file arguments stays quiet. Wrappers (env,
|
|
18
|
+
# timeout, nice, time, bundle exec [--]) and versioned rubies
|
|
19
|
+
# (ruby3.4) are unwrapped. Known false negatives, accepted: nested
|
|
20
|
+
# shells (sh -c '...') and non-test file names.
|
|
21
|
+
module TestCommandShape
|
|
22
|
+
SHELL_OPERATOR_TOKENS = %w[&& || ; | &].freeze
|
|
23
|
+
|
|
24
|
+
# ruby options that swallow the NEXT token as their operand, detached
|
|
25
|
+
# form (-I lib); the attached form (-Ilib) is one token and needs no
|
|
26
|
+
# skip.
|
|
27
|
+
RUBY_OPERAND_OPTIONS = %w[-r -I -C -E -F -K --encoding --eval].freeze
|
|
28
|
+
|
|
29
|
+
TEST_FILE = /\A(test_|spec_).*|(_test|_spec)\.rb\z/
|
|
30
|
+
|
|
31
|
+
module_function
|
|
32
|
+
|
|
33
|
+
def multi_file_ruby?(command)
|
|
34
|
+
simple_commands(command).any? { |simple| ruby_multi_file_simple_command?(simple) }
|
|
35
|
+
rescue ArgumentError
|
|
36
|
+
false
|
|
37
|
+
end
|
|
38
|
+
|
|
39
|
+
# ";" is a shell separator whenever unquoted - even glued to the next
|
|
40
|
+
# word ("cd spec;ruby ..."). Masking quoted and escaped semicolons
|
|
41
|
+
# first keeps 'ruby "a;b_test.rb"' one script; other operators split
|
|
42
|
+
# only as standalone tokens.
|
|
43
|
+
def simple_commands(command)
|
|
44
|
+
commands = [[]]
|
|
45
|
+
semicolon_segments(command).each do |segment|
|
|
46
|
+
Shellwords.split(segment).each do |token|
|
|
47
|
+
if SHELL_OPERATOR_TOKENS.include?(token)
|
|
48
|
+
commands << []
|
|
49
|
+
else
|
|
50
|
+
commands.last << token
|
|
51
|
+
end
|
|
52
|
+
end
|
|
53
|
+
commands << []
|
|
54
|
+
end
|
|
55
|
+
commands.reject(&:empty?)
|
|
56
|
+
end
|
|
57
|
+
|
|
58
|
+
QUOTED_OR_ESCAPED = /"(?:\\.|[^"\\])*"|'[^']*'|\\./m
|
|
59
|
+
|
|
60
|
+
def semicolon_segments(command)
|
|
61
|
+
masked = command.gsub(QUOTED_OR_ESCAPED) { |span| span.tr(";", "\0") }
|
|
62
|
+
masked.split(";", -1).map { |segment| segment.tr("\0", ";") }
|
|
63
|
+
end
|
|
64
|
+
|
|
65
|
+
def ruby_multi_file_simple_command?(tokens)
|
|
66
|
+
tokens = unwrap(tokens)
|
|
67
|
+
return false unless tokens.first&.then { |prog| File.basename(prog).match?(/\Aruby[\d.]*\z/) }
|
|
68
|
+
|
|
69
|
+
args = tokens[1..]
|
|
70
|
+
return false if args.any? { |token| token == "-e" || token.start_with?("-e") || token == "--eval" }
|
|
71
|
+
|
|
72
|
+
script, *rest = script_operands(args)
|
|
73
|
+
!script.nil? && script.end_with?(".rb") && File.basename(script).match?(TEST_FILE) &&
|
|
74
|
+
rest.any? { |token| token.end_with?(".rb") }
|
|
75
|
+
end
|
|
76
|
+
|
|
77
|
+
def unwrap(tokens)
|
|
78
|
+
tokens = tokens.dup
|
|
79
|
+
loop { break unless shift_wrapper?(tokens) }
|
|
80
|
+
tokens
|
|
81
|
+
end
|
|
82
|
+
|
|
83
|
+
# Shifts one leading wrapper (env, timeout, nice, time, bundle exec,
|
|
84
|
+
# VAR=value, --) off the tokens; false once the head is the command.
|
|
85
|
+
def shift_wrapper?(tokens)
|
|
86
|
+
first = tokens.first
|
|
87
|
+
if first&.match?(/\A[A-Za-z_]\w*=/) || first == "--"
|
|
88
|
+
tokens.shift
|
|
89
|
+
elsif %w[env /usr/bin/env].include?(first)
|
|
90
|
+
tokens.shift
|
|
91
|
+
skip_env_flags(tokens)
|
|
92
|
+
elsif %w[timeout nice time].include?(first)
|
|
93
|
+
tokens.shift
|
|
94
|
+
skip_wrapper_operand(first, tokens)
|
|
95
|
+
elsif first == "bundle" && tokens[1] == "exec"
|
|
96
|
+
tokens.shift(2)
|
|
97
|
+
else
|
|
98
|
+
return false
|
|
99
|
+
end
|
|
100
|
+
true
|
|
101
|
+
end
|
|
102
|
+
|
|
103
|
+
# timeout skips its flags and duration; nice its adjustment (detached
|
|
104
|
+
# -n 5/--adjustment 5 or glued -n5/--adjustment=5/legacy -5); time -p.
|
|
105
|
+
def skip_wrapper_operand(wrapper, tokens)
|
|
106
|
+
case wrapper
|
|
107
|
+
when "timeout"
|
|
108
|
+
tokens.shift while tokens.first&.start_with?("-")
|
|
109
|
+
tokens.shift
|
|
110
|
+
when "nice"
|
|
111
|
+
tokens.shift(2) if %w[-n --adjustment].include?(tokens.first)
|
|
112
|
+
tokens.shift if tokens.first&.match?(/\A-n\S|\A--adjustment=|\A-\d+\z/)
|
|
113
|
+
when "time"
|
|
114
|
+
tokens.shift if tokens.first == "-p"
|
|
115
|
+
end
|
|
116
|
+
end
|
|
117
|
+
|
|
118
|
+
def skip_env_flags(tokens)
|
|
119
|
+
tokens.shift while tokens.first&.start_with?("-") && !%w[-u -C].include?(tokens.first)
|
|
120
|
+
tokens.shift(2) while %w[-u -C].include?(tokens.first)
|
|
121
|
+
end
|
|
122
|
+
|
|
123
|
+
# Redirections are shell syntax, not ruby operands: the operator and
|
|
124
|
+
# its target ("2> errors.rb") or the glued form ("2>errors.rb",
|
|
125
|
+
# ">out", "&>all") never count as scripts.
|
|
126
|
+
REDIRECT_OPS = %w[> >> < 2> 2>> 1> 1>> &> &>>].freeze
|
|
127
|
+
GLUED_REDIRECT = /\A\d*(?:>>?|<|&>)/
|
|
128
|
+
|
|
129
|
+
# The operands after option parsing, script first: -e/-E/-r/-I/-C and
|
|
130
|
+
# friends swallow their value.
|
|
131
|
+
def script_operands(args)
|
|
132
|
+
operands = []
|
|
133
|
+
index = 0
|
|
134
|
+
while index < args.size
|
|
135
|
+
token = args[index]
|
|
136
|
+
if REDIRECT_OPS.include?(token) || RUBY_OPERAND_OPTIONS.include?(token)
|
|
137
|
+
index += 2
|
|
138
|
+
elsif token.match?(GLUED_REDIRECT) || token.start_with?("-")
|
|
139
|
+
index += 1
|
|
140
|
+
else
|
|
141
|
+
operands << token
|
|
142
|
+
index += 1
|
|
143
|
+
end
|
|
144
|
+
end
|
|
145
|
+
operands
|
|
146
|
+
end
|
|
147
|
+
end
|
|
148
|
+
end
|
|
149
|
+
end
|
|
@@ -13,7 +13,18 @@ module Saboteur
|
|
|
13
13
|
module SqliteMemoryDatabase
|
|
14
14
|
REMEDY = "a sqlite :memory: database is empty in this forked child: in-memory databases do not " \
|
|
15
15
|
"survive saboteur's fork, so a schema loaded at require time is lost. Load the schema in " \
|
|
16
|
-
"a before(:suite) hook, or in an after_fork hook
|
|
16
|
+
"a before(:suite) hook, or in an after_fork hook in config/saboteur_hooks.rb; wire it in " \
|
|
17
|
+
'config/saboteur.rb with c.hooks = ["config/saboteur_hooks.rb"]'
|
|
18
|
+
|
|
19
|
+
# The #594 seed variant: the suite moved its schema into a hook (the
|
|
20
|
+
# schema remedy above), so children get tables - but rows seeded at
|
|
21
|
+
# require time (rolify's spec/support/data.rb) lived in the parent's
|
|
22
|
+
# database and are lost across the fork all the same.
|
|
23
|
+
SEED_REMEDY = "a sqlite :memory: database in this forked child has its schema but no rows: rows seeded at " \
|
|
24
|
+
"require time (seed data, fixtures loaded at boot) are lost across saboteur's fork, just like " \
|
|
25
|
+
"the schema. Seed in a before(:suite) hook, or in an after_fork hook in " \
|
|
26
|
+
"config/saboteur_hooks.rb; wire it in config/saboteur.rb with " \
|
|
27
|
+
'c.hooks = ["config/saboteur_hooks.rb"]'
|
|
17
28
|
|
|
18
29
|
module_function
|
|
19
30
|
|
|
@@ -24,7 +35,10 @@ module Saboteur
|
|
|
24
35
|
return unless defined?(ActiveRecord::Base)
|
|
25
36
|
|
|
26
37
|
pools = ActiveRecord::Base.connection_handler.connection_pool_list
|
|
27
|
-
pools.any? { |pool| empty_memory_database?(pool) }
|
|
38
|
+
return REMEDY if pools.any? { |pool| empty_memory_database?(pool) }
|
|
39
|
+
return SEED_REMEDY if pools.any? { |pool| unseeded_memory_database?(pool) }
|
|
40
|
+
|
|
41
|
+
nil
|
|
28
42
|
rescue StandardError, ScriptError
|
|
29
43
|
nil
|
|
30
44
|
end
|
|
@@ -38,6 +52,22 @@ module Saboteur
|
|
|
38
52
|
false
|
|
39
53
|
end
|
|
40
54
|
|
|
55
|
+
# Schema but not a single row anywhere: the hook-loaded schema
|
|
56
|
+
# survived the fork, the require-time seeds did not (#594).
|
|
57
|
+
def unseeded_memory_database?(pool)
|
|
58
|
+
config = pool.db_config
|
|
59
|
+
return false unless config.adapter.to_s == "sqlite3" && config.database.to_s == ":memory:"
|
|
60
|
+
|
|
61
|
+
pool.with_connection do |connection|
|
|
62
|
+
tables = user_tables(connection)
|
|
63
|
+
tables.any? && tables.all? do |table|
|
|
64
|
+
connection.select_value("SELECT 1 FROM #{connection.quote_table_name(table)} LIMIT 1").nil?
|
|
65
|
+
end
|
|
66
|
+
end
|
|
67
|
+
rescue StandardError
|
|
68
|
+
false
|
|
69
|
+
end
|
|
70
|
+
|
|
41
71
|
# Migration bookkeeping is not user schema: a database with only these
|
|
42
72
|
# tables still has no schema for the suite's models.
|
|
43
73
|
def user_tables(connection)
|
|
@@ -17,6 +17,7 @@ module Saboteur
|
|
|
17
17
|
begin
|
|
18
18
|
@runner.setup($stderr, load_output)
|
|
19
19
|
rescue SystemExit => e
|
|
20
|
+
load_output.replay
|
|
20
21
|
# A spec file calling exit at load makes RSpec print "While
|
|
21
22
|
# loading <file> ..." (which names the file) and re-raise.
|
|
22
23
|
# Uncaught, the parent exits with the spec's status: exit(0)
|
|
@@ -24,7 +25,15 @@ module Saboteur
|
|
|
24
25
|
raise Error, "a spec file called exit while loading (status #{e.status})"
|
|
25
26
|
end
|
|
26
27
|
configuration.order = "rand:#{config.fetch('seed', 1)}"
|
|
27
|
-
|
|
28
|
+
if quit_while_loading?(world)
|
|
29
|
+
# RSpec's load dump ends in the saboteur frames the load was
|
|
30
|
+
# triggered through and a "No examples found." from the reporter;
|
|
31
|
+
# both contradict the tool's own message. Print the suite's error
|
|
32
|
+
# without them (#608).
|
|
33
|
+
$stdout.write LoadErrorReport.filter(load_output.text, verbose: config.fetch("verbose", false))
|
|
34
|
+
raise Error, load_failure_message(load_output.failed_files, load_output.text)
|
|
35
|
+
end
|
|
36
|
+
load_output.replay
|
|
28
37
|
|
|
29
38
|
refuse_unintended_focus
|
|
30
39
|
end
|
|
@@ -80,7 +89,10 @@ module Saboteur
|
|
|
80
89
|
|
|
81
90
|
# A missing rspec-core is the target's setup error, not a saboteur
|
|
82
91
|
# defect (#482): refuse with the remedy instead of crashing into the
|
|
83
|
-
# internal-error report.
|
|
92
|
+
# internal-error report. It is a usage error (#610): ConfigError maps
|
|
93
|
+
# to exit 2, and the remedy names docs/minitest.md rather than
|
|
94
|
+
# c.integration - rspec is the only integration, so pointing at it
|
|
95
|
+
# sent Minitest users down a dead end. Only the framework's own LoadError converts -
|
|
84
96
|
# matched on the failed feature, not the message, so a missing
|
|
85
97
|
# rspec/core_ext does not pose as rspec/core; a LoadError the app's
|
|
86
98
|
# files raise keeps the crash route.
|
|
@@ -89,8 +101,9 @@ module Saboteur
|
|
|
89
101
|
rescue LoadError => e
|
|
90
102
|
raise unless e.path == "rspec/core"
|
|
91
103
|
|
|
92
|
-
raise
|
|
93
|
-
|
|
104
|
+
raise ConfigError, "rspec is not in the bundle: the RSpec integration needs the rspec-core gem. " \
|
|
105
|
+
"Add it to the bundle's test group. saboteur's built-in integration supports RSpec only - " \
|
|
106
|
+
"see docs/minitest.md for other suites"
|
|
94
107
|
end
|
|
95
108
|
|
|
96
109
|
def all_tests
|
|
@@ -137,6 +150,19 @@ module Saboteur
|
|
|
137
150
|
|
|
138
151
|
SHOWN_LOAD_FAILURES = 5
|
|
139
152
|
|
|
153
|
+
# Ruby's Coverage refuses a second start, so a suite whose
|
|
154
|
+
# spec_helper auto-starts SimpleCov/SingleCov collides with
|
|
155
|
+
# saboteur's own Coverage.start and the load quits with an opaque
|
|
156
|
+
# "No examples found" (#645, real run: audited @ dbf84326). Name the
|
|
157
|
+
# suspect and both guard styles. The gem's backtrace line is the
|
|
158
|
+
# fallback: the RuntimeError's text is Coverage's own, whoever
|
|
159
|
+
# started it second.
|
|
160
|
+
COVERAGE_DOUBLE_START = /coverage measurement is already setup/i
|
|
161
|
+
# A gem backtrace frame (path:line) or the start call itself - never a
|
|
162
|
+
# bare name, which a spec FILE name like simplecov_config_spec.rb
|
|
163
|
+
# would false-positive on (Pod 3 QA probe, #645).
|
|
164
|
+
COVERAGE_GEM_FRAME = %r{(?:single_cov|simplecov)[\w.\-/]*\.rb:\d+|\b(?:SimpleCov\.start|SingleCov\.setup)}i
|
|
165
|
+
|
|
140
166
|
private
|
|
141
167
|
|
|
142
168
|
# rspec-core 3.11 added rspec_is_quitting (SyntaxError and SystemExit
|
|
@@ -151,7 +177,7 @@ module Saboteur
|
|
|
151
177
|
# printed are what a user needs. Name them and the way out: a repo
|
|
152
178
|
# whose spec tree cannot load whole (an engine's dummy app, an
|
|
153
179
|
# env-dependent integration spec) must narrow what loads (#378).
|
|
154
|
-
def load_failure_message(failed_files)
|
|
180
|
+
def load_failure_message(failed_files, captured_text = nil)
|
|
155
181
|
message = +"rspec quit while loading spec files"
|
|
156
182
|
if failed_files.any?
|
|
157
183
|
shown = failed_files.first(SHOWN_LOAD_FAILURES)
|
|
@@ -164,6 +190,16 @@ module Saboteur
|
|
|
164
190
|
'c.rspec_args = ["--exclude-pattern", "spec/dummy/**/*_spec.rb"], or name the ' \
|
|
165
191
|
"suite with --test. Rails engines with an embedded dummy app (spec/dummy) need " \
|
|
166
192
|
'this; see README "Rails engines and dummy apps".'
|
|
193
|
+
if coverage_suspect?(captured_text)
|
|
194
|
+
message << "\nSuspect: your suite starts its own Coverage (saboteur starts one itself). " \
|
|
195
|
+
"Guard it - if ENV[\"COVERAGE\"], or SingleCov's conditional setup API - " \
|
|
196
|
+
"and see docs/recipes/README.md#simplecov."
|
|
197
|
+
end
|
|
198
|
+
message
|
|
199
|
+
end
|
|
200
|
+
|
|
201
|
+
def coverage_suspect?(captured_text)
|
|
202
|
+
captured_text&.match?(COVERAGE_DOUBLE_START) || captured_text&.match?(COVERAGE_GEM_FRAME) || false
|
|
167
203
|
end
|
|
168
204
|
|
|
169
205
|
def refuse_unintended_focus
|
|
@@ -293,26 +329,78 @@ module Saboteur
|
|
|
293
329
|
@captured = +""
|
|
294
330
|
end
|
|
295
331
|
|
|
332
|
+
# Buffer only: the caller decides after setup whether the captured
|
|
333
|
+
# text goes out as it is (a clean load), filtered (a load quit,
|
|
334
|
+
# #608) or verbatim on a SystemExit. Passing through during setup
|
|
335
|
+
# would print what a later filter needs to drop.
|
|
296
336
|
def write(text)
|
|
297
|
-
@captured << text
|
|
298
|
-
|
|
337
|
+
@replayed ? __getobj__.write(text) : @captured << text
|
|
338
|
+
self
|
|
299
339
|
end
|
|
300
340
|
|
|
301
341
|
def print(*args)
|
|
302
|
-
|
|
303
|
-
super
|
|
342
|
+
write(args.join)
|
|
304
343
|
end
|
|
305
344
|
|
|
306
345
|
def puts(*args)
|
|
307
|
-
|
|
308
|
-
super
|
|
346
|
+
write("#{args.flatten.join("\n")}\n")
|
|
309
347
|
end
|
|
310
348
|
|
|
349
|
+
def replay
|
|
350
|
+
__getobj__.write(@captured)
|
|
351
|
+
# RSpec retains this stream for later suite-hook errors.
|
|
352
|
+
@replayed = true
|
|
353
|
+
end
|
|
354
|
+
|
|
355
|
+
def text = @captured
|
|
356
|
+
|
|
311
357
|
def failed_files
|
|
312
358
|
LOAD_FAILURE.flat_map { |pattern| @captured.scan(pattern) }.flatten
|
|
313
359
|
end
|
|
314
360
|
end
|
|
315
361
|
|
|
362
|
+
# The load dump RSpec prints when a spec file raises at load, trimmed
|
|
363
|
+
# to what the user acts on: the first error with the suite's own
|
|
364
|
+
# frames. The saboteur frames the load was triggered through and the
|
|
365
|
+
# reporter's "No examples found." (nothing ran because nothing loaded)
|
|
366
|
+
# read as saboteur crashing and contradict the tool's message; both
|
|
367
|
+
# stay under --verbose (#608).
|
|
368
|
+
class LoadErrorReport
|
|
369
|
+
HEADER = /^An error occurred while loading /
|
|
370
|
+
# The running tool's install root (a checkout or an unpacked gem),
|
|
371
|
+
# with separator. Only frames under it are saboteur's: the same
|
|
372
|
+
# lib/saboteur/ layout under ANOTHER root is the target's vendored
|
|
373
|
+
# copy or a dogfooded checkout, and those frames are the suite's
|
|
374
|
+
# (#608).
|
|
375
|
+
TOOL_ROOT = "#{File.expand_path('../../..', __dir__)}/".freeze
|
|
376
|
+
private_constant :HEADER, :TOOL_ROOT
|
|
377
|
+
|
|
378
|
+
def self.filter(text, verbose: false)
|
|
379
|
+
return text if verbose
|
|
380
|
+
|
|
381
|
+
lines = text.lines.reject { |line| line.strip == "No examples found." || saboteur_frame?(line) }
|
|
382
|
+
heads = lines.each_index.select { |index| lines[index].match?(HEADER) }
|
|
383
|
+
return lines.join if heads.size <= 1
|
|
384
|
+
|
|
385
|
+
kept = lines[0...heads[1]]
|
|
386
|
+
kept.pop while kept.last&.strip == ""
|
|
387
|
+
omitted = heads.size - 1
|
|
388
|
+
kept << "(#{omitted} more load error#{'s' if omitted > 1} omitted; rerun with --verbose for the full list)\n"
|
|
389
|
+
kept.join
|
|
390
|
+
end
|
|
391
|
+
|
|
392
|
+
# A backtrace frame inside the tool's install root. Relative frames
|
|
393
|
+
# (the suite's own "./spec/...") are never the tool's.
|
|
394
|
+
def self.saboteur_frame?(line)
|
|
395
|
+
line = line.strip
|
|
396
|
+
return false unless line.start_with?("# ")
|
|
397
|
+
|
|
398
|
+
path = line.sub(/\A#\s+/, "").sub(/:\d+:in .*\z/, "")
|
|
399
|
+
path.start_with?("/") && File.expand_path(path).start_with?(TOOL_ROOT)
|
|
400
|
+
end
|
|
401
|
+
private_class_method :saboteur_frame?
|
|
402
|
+
end
|
|
403
|
+
|
|
316
404
|
class Listener
|
|
317
405
|
EVENTS = %i[example_started example_finished example_failed].freeze
|
|
318
406
|
|
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
# SPDX-License-Identifier: MIT
|
|
2
|
+
# frozen_string_literal: true
|
|
3
|
+
|
|
4
|
+
require_relative "base"
|
|
5
|
+
require_relative "operator_visit"
|
|
6
|
+
|
|
7
|
+
module Saboteur
|
|
8
|
+
module MutationSource
|
|
9
|
+
# Swap two explicit positional arguments without moving their separators.
|
|
10
|
+
# Order-insensitive calls (includes, attr_reader, validates) can be equivalent; arid patterns can exclude them.
|
|
11
|
+
class ArgumentSwap < Base
|
|
12
|
+
include OperatorVisit
|
|
13
|
+
|
|
14
|
+
EXCLUDED = [Prism::KeywordHashNode, Prism::SplatNode, Prism::ForwardingArgumentsNode].freeze
|
|
15
|
+
|
|
16
|
+
def initialize(arid: []) # rubocop:disable Lint/MissingSuper
|
|
17
|
+
@arid = arid.map { |pattern| Regexp.new(pattern) }
|
|
18
|
+
end
|
|
19
|
+
|
|
20
|
+
def name
|
|
21
|
+
"argument_swap"
|
|
22
|
+
end
|
|
23
|
+
|
|
24
|
+
private
|
|
25
|
+
|
|
26
|
+
def mutate_node(node, subject, found)
|
|
27
|
+
return unless node.is_a?(Prism::CallNode) && node.opening_loc&.slice == "("
|
|
28
|
+
|
|
29
|
+
arguments = node.arguments&.arguments
|
|
30
|
+
return unless arguments&.size == 2
|
|
31
|
+
return if arguments.any? { |argument| excluded?(argument) }
|
|
32
|
+
|
|
33
|
+
left, right = arguments
|
|
34
|
+
return if left.slice == right.slice
|
|
35
|
+
|
|
36
|
+
# Prism forwards location offsets on nodes; removing .location here is equivalent.
|
|
37
|
+
edits = [[left, right], [right, left]].map do |original, replacement|
|
|
38
|
+
Edit.new(file: subject.file, start_byte: original.location.start_offset,
|
|
39
|
+
end_byte: original.location.end_offset, replacement: replacement.slice)
|
|
40
|
+
end
|
|
41
|
+
found << Mutation.new(subject_id: subject.id, operator: name, edits: edits, source: name)
|
|
42
|
+
end
|
|
43
|
+
|
|
44
|
+
def excluded?(argument)
|
|
45
|
+
EXCLUDED.any? { |type| argument.is_a?(type) } || heredoc?(argument)
|
|
46
|
+
end
|
|
47
|
+
|
|
48
|
+
def heredoc?(node)
|
|
49
|
+
(node.respond_to?(:opening_loc) && node.opening_loc&.slice&.start_with?("<<")) ||
|
|
50
|
+
node.compact_child_nodes.any? { |child| heredoc?(child) }
|
|
51
|
+
end
|
|
52
|
+
end
|
|
53
|
+
end
|
|
54
|
+
end
|
|
@@ -6,6 +6,7 @@ require_relative "base"
|
|
|
6
6
|
require_relative "filter_replacement"
|
|
7
7
|
require_relative "literal_replacement"
|
|
8
8
|
require_relative "argument_removal"
|
|
9
|
+
require_relative "argument_swap"
|
|
9
10
|
require_relative "chain_removal"
|
|
10
11
|
require_relative "pair_removal"
|
|
11
12
|
require_relative "element_removal"
|
|
@@ -65,6 +66,7 @@ module Saboteur
|
|
|
65
66
|
@filter_replacement = FilterReplacement.new(arid: arid)
|
|
66
67
|
@literal_replacement = LiteralReplacement.new(arid: arid)
|
|
67
68
|
@argument_removal = ArgumentRemoval.new(arid: arid)
|
|
69
|
+
@argument_swap = ArgumentSwap.new(arid: arid)
|
|
68
70
|
@pair_removal = PairRemoval.new(arid: arid)
|
|
69
71
|
@element_removal = ElementRemoval.new(arid: arid)
|
|
70
72
|
@chain_removal = ChainRemoval.new(arid: arid)
|
|
@@ -100,7 +102,7 @@ module Saboteur
|
|
|
100
102
|
[@filter_replacement, @literal_replacement, @argument_removal, @pair_removal, @element_removal,
|
|
101
103
|
@chain_removal,
|
|
102
104
|
@endpoint_replacement, @range_replacement, @operator_assignment, @condition_negation,
|
|
103
|
-
@method_swap, @operand_replacement, @interpolation_removal, @regex_replacement]
|
|
105
|
+
@method_swap, @operand_replacement, @interpolation_removal, @regex_replacement, @argument_swap]
|
|
104
106
|
end
|
|
105
107
|
|
|
106
108
|
private
|
|
@@ -52,10 +52,11 @@ module Saboteur
|
|
|
52
52
|
stack = [root]
|
|
53
53
|
until stack.empty?
|
|
54
54
|
node = stack.pop
|
|
55
|
-
|
|
55
|
+
# def and lambda bodies run when called, not at load.
|
|
56
|
+
next if node.is_a?(Prism::DefNode) || node.is_a?(Prism::LambdaNode)
|
|
56
57
|
|
|
57
58
|
if hazardous?(node)
|
|
58
|
-
slices << node
|
|
59
|
+
slices << cite(node)
|
|
59
60
|
next
|
|
60
61
|
end
|
|
61
62
|
stack.concat(node.compact_child_nodes)
|
|
@@ -68,6 +69,10 @@ module Saboteur
|
|
|
68
69
|
# require_relative(begin ... end) runs arbitrary code on every
|
|
69
70
|
# replay, so it is unvetted below like any other helper call.
|
|
70
71
|
SAFE_LOAD_REQUIRES = %w[require require_relative].freeze
|
|
72
|
+
VISIBILITY_NAMES = %w[public private protected].freeze
|
|
73
|
+
CONSTANT_VISIBILITY_NAMES = %w[private_constant public_constant].freeze
|
|
74
|
+
# `X = Struct.new(:a)` / `Point = Data.define(:x)` without a block.
|
|
75
|
+
FACTORY_NEW = { "Struct" => :new, "Data" => :define }.freeze
|
|
71
76
|
|
|
72
77
|
# Assignments whose value a replay re-runs: vetted when the value is
|
|
73
78
|
# a plain literal construction.
|
|
@@ -81,11 +86,19 @@ module Saboteur
|
|
|
81
86
|
# at load runs again on every mutant, outside the method hazard guard.
|
|
82
87
|
# The vetted forms are narrow: require/require_relative, pure defs
|
|
83
88
|
# (a body runs when called, not at load), class/module nesting with a
|
|
84
|
-
# constant superclass,
|
|
85
|
-
#
|
|
86
|
-
#
|
|
87
|
-
#
|
|
88
|
-
#
|
|
89
|
+
# constant superclass, a `class << self` block of the same vetted
|
|
90
|
+
# forms, bare visibility calls (public/private/protected with no
|
|
91
|
+
# argument, literal symbols, or a def - they only re-flag methods),
|
|
92
|
+
# constant-visibility calls with literal names, Struct/Data class
|
|
93
|
+
# factories with literal arguments and no block, and constant or
|
|
94
|
+
# local writes of plain literal values (a `.freeze` on the literal is
|
|
95
|
+
# looked through; a lambda literal is data - its body runs when
|
|
96
|
+
# called, not at load). Everything else - helper calls,
|
|
97
|
+
# registrations, writes to disk, class factories, top-level blocks -
|
|
98
|
+
# is named here, NOT proven unsafe: the check only says saboteur
|
|
99
|
+
# cannot account for what re-running it does (#319). Each entry is a
|
|
100
|
+
# one-line citation: the statement's first line cut to 60 characters,
|
|
101
|
+
# with its line number (#664).
|
|
89
102
|
def unvetted_load_code(root)
|
|
90
103
|
bad = []
|
|
91
104
|
vet_statements(root.statements.body, bad)
|
|
@@ -112,21 +125,88 @@ module Saboteur
|
|
|
112
125
|
nil
|
|
113
126
|
when Prism::ClassNode, Prism::ModuleNode
|
|
114
127
|
vet_scope(node, bad)
|
|
128
|
+
when Prism::SingletonClassNode
|
|
129
|
+
if vetted_singleton_target?(node.expression)
|
|
130
|
+
vet_scope(node, bad)
|
|
131
|
+
else
|
|
132
|
+
bad << cite(node)
|
|
133
|
+
end
|
|
115
134
|
when *WRITE_NODES
|
|
116
|
-
bad << node
|
|
135
|
+
bad << cite(node) unless vetted_value?(node.value)
|
|
117
136
|
when Prism::CallNode
|
|
118
|
-
bad << node
|
|
137
|
+
bad << cite(node) unless vetted_call?(node) || visibility_call?(node)
|
|
119
138
|
else
|
|
120
|
-
bad << node
|
|
139
|
+
bad << cite(node)
|
|
121
140
|
end
|
|
122
141
|
end
|
|
123
142
|
|
|
143
|
+
# "first line of the statement (line N)": a whole class body or a
|
|
144
|
+
# long call must not flood the warning (#664).
|
|
145
|
+
def cite(node)
|
|
146
|
+
first = node.slice.lines.first.to_s.strip
|
|
147
|
+
first = "#{first[0, 57]}..." if first.length > 60
|
|
148
|
+
"#{first} (line #{node.location.start_line})"
|
|
149
|
+
end
|
|
150
|
+
|
|
151
|
+
# `class << self` opens self's singleton class, and a constant or
|
|
152
|
+
# ivar target is only read. Anything else - `class << factory.build` -
|
|
153
|
+
# RUNS the expression at load, so the whole statement is unvetted.
|
|
154
|
+
def vetted_singleton_target?(node)
|
|
155
|
+
node.is_a?(Prism::SelfNode) || constant_read?(node) || node.is_a?(Prism::InstanceVariableReadNode)
|
|
156
|
+
end
|
|
157
|
+
|
|
124
158
|
def vet_scope(node, bad)
|
|
125
159
|
superclass = node.respond_to?(:superclass) ? node.superclass : nil
|
|
126
|
-
bad << superclass
|
|
160
|
+
bad << cite(superclass) if superclass && !constant_read?(superclass)
|
|
127
161
|
vet_statements(node.body&.body || [], bad)
|
|
128
162
|
end
|
|
129
163
|
|
|
164
|
+
# A bare visibility call re-flags methods and nothing else, so a
|
|
165
|
+
# replay is harmless: no receiver, no block, and only literal symbol
|
|
166
|
+
# or def arguments (`private :x`, `private def x`). The constant
|
|
167
|
+
# twins (`private_constant :X`) re-flag a constant's visibility the
|
|
168
|
+
# same way, with literal symbol or string arguments (#537).
|
|
169
|
+
def visibility_call?(node)
|
|
170
|
+
return false unless node.receiver.nil? && node.block.nil?
|
|
171
|
+
|
|
172
|
+
arguments = node.arguments&.arguments || []
|
|
173
|
+
if VISIBILITY_NAMES.include?(node.name.to_s)
|
|
174
|
+
arguments.all? { |argument| argument.is_a?(Prism::SymbolNode) || argument.is_a?(Prism::DefNode) }
|
|
175
|
+
elsif CONSTANT_VISIBILITY_NAMES.include?(node.name.to_s)
|
|
176
|
+
arguments.all? { |argument| argument.is_a?(Prism::SymbolNode) || argument.is_a?(Prism::StringNode) }
|
|
177
|
+
else
|
|
178
|
+
false
|
|
179
|
+
end
|
|
180
|
+
end
|
|
181
|
+
|
|
182
|
+
# `.freeze` on a vetted literal freezes data: no side effect.
|
|
183
|
+
def freeze_call?(node)
|
|
184
|
+
node.name == :freeze && node.arguments.nil? && node.block.nil? && vetted_value?(node.receiver)
|
|
185
|
+
end
|
|
186
|
+
|
|
187
|
+
# A Struct/Data class factory with only literal arguments and no
|
|
188
|
+
# block just builds a class: replaying it rebuilds the class, so the
|
|
189
|
+
# file it sits in stays vetted (#537, real run: ruby-jwt #747's
|
|
190
|
+
# `VerificationContext = Struct.new(:payload, keyword_init: true)`).
|
|
191
|
+
# A block is class_eval'd at load and stays unvetted, as does any
|
|
192
|
+
# computed argument.
|
|
193
|
+
def factory_new?(node)
|
|
194
|
+
return false unless node.receiver.is_a?(Prism::ConstantReadNode) && node.block.nil?
|
|
195
|
+
return false unless FACTORY_NEW[node.receiver.name.to_s] == node.name
|
|
196
|
+
|
|
197
|
+
(node.arguments&.arguments || []).all? do |argument|
|
|
198
|
+
argument.is_a?(Prism::SymbolNode) || argument.is_a?(Prism::StringNode) ||
|
|
199
|
+
keyword_bool?(argument)
|
|
200
|
+
end
|
|
201
|
+
end
|
|
202
|
+
|
|
203
|
+
def keyword_bool?(argument)
|
|
204
|
+
argument.is_a?(Prism::KeywordHashNode) && argument.elements.all? do |pair|
|
|
205
|
+
pair.is_a?(Prism::AssocNode) && pair.key.is_a?(Prism::SymbolNode) &&
|
|
206
|
+
(pair.value.is_a?(Prism::TrueNode) || pair.value.is_a?(Prism::FalseNode))
|
|
207
|
+
end
|
|
208
|
+
end
|
|
209
|
+
|
|
130
210
|
def vetted_call?(node)
|
|
131
211
|
return false unless SAFE_LOAD_REQUIRES.include?(node.name.to_s) && node.receiver.nil?
|
|
132
212
|
return false unless node.block.nil?
|
|
@@ -146,7 +226,8 @@ module Saboteur
|
|
|
146
226
|
case node
|
|
147
227
|
when Prism::StringNode, Prism::SymbolNode, Prism::IntegerNode, Prism::FloatNode,
|
|
148
228
|
Prism::TrueNode, Prism::FalseNode, Prism::NilNode, Prism::SourceFileNode,
|
|
149
|
-
Prism::ConstantReadNode, Prism::ConstantPathNode
|
|
229
|
+
Prism::ConstantReadNode, Prism::ConstantPathNode, Prism::RegularExpressionNode,
|
|
230
|
+
Prism::LambdaNode
|
|
150
231
|
true
|
|
151
232
|
when Prism::ArrayNode
|
|
152
233
|
node.elements.all? { |element| vetted_value?(element) }
|
|
@@ -155,7 +236,7 @@ module Saboteur
|
|
|
155
236
|
pair.is_a?(Prism::AssocNode) && vetted_value?(pair.key) && vetted_value?(pair.value)
|
|
156
237
|
end
|
|
157
238
|
when Prism::CallNode
|
|
158
|
-
|
|
239
|
+
freeze_call?(node) || factory_new?(node)
|
|
159
240
|
else
|
|
160
241
|
false
|
|
161
242
|
end
|
|
@@ -22,6 +22,7 @@ module Saboteur
|
|
|
22
22
|
"filter_replacement" => "xs.select { |x| x } -> xs.reject { |x| x }",
|
|
23
23
|
"literal_string" => "\"hello\" -> \"\"",
|
|
24
24
|
"literal_boolean" => "true -> false",
|
|
25
|
+
"argument_swap" => "send(a, b) -> send(b, a)",
|
|
25
26
|
"argument_removal" => "send(a, b) -> send(a)",
|
|
26
27
|
"pair_removal" => "{a: 1, b: 2} -> {b: 2}",
|
|
27
28
|
"element_removal" => "TAGS = [a, b] -> TAGS = [b]",
|
|
@@ -29,6 +29,7 @@ module Saboteur
|
|
|
29
29
|
"filter_replacement" => "Swap select and reject",
|
|
30
30
|
"literal_string" => "Empty a string literal",
|
|
31
31
|
"literal_boolean" => "Swap a boolean literal",
|
|
32
|
+
"argument_swap" => "Swap two positional call arguments",
|
|
32
33
|
"argument_removal" => "Remove a call argument",
|
|
33
34
|
"pair_removal" => "Remove a hash pair",
|
|
34
35
|
"element_removal" => "Remove an element of a constant array literal",
|
data/lib/saboteur/preview.rb
CHANGED
|
@@ -125,6 +125,11 @@ module Saboteur
|
|
|
125
125
|
end
|
|
126
126
|
|
|
127
127
|
def planned(subject, candidates, scope)
|
|
128
|
+
if subject.skip_reason
|
|
129
|
+
@diagnostics.warn("#{subject.id}: skipped (#{subject.skip_reason}); excluded from the planned total")
|
|
130
|
+
return []
|
|
131
|
+
end
|
|
132
|
+
|
|
128
133
|
list = candidates.for(subject)
|
|
129
134
|
list = scope.restrict(subject, list) if scope
|
|
130
135
|
limit = @config["max_mutants_per_method"]
|