saboteur 0.7.0 → 0.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +657 -0
- data/README.md +198 -1077
- data/lib/saboteur/cli/exit_status.rb +32 -1
- data/lib/saboteur/cli/help_options.rb +2 -1
- data/lib/saboteur/cli/history_command.rb +135 -0
- data/lib/saboteur/cli/presets.rb +6 -2
- data/lib/saboteur/cli/selection_options.rb +31 -0
- data/lib/saboteur/cli/shard_options.rb +17 -0
- data/lib/saboteur/cli/source_set.rb +53 -0
- data/lib/saboteur/cli/survivors_command.rb +157 -0
- data/lib/saboteur/cli.rb +71 -54
- data/lib/saboteur/config.rb +98 -5
- data/lib/saboteur/config_file.rb +8 -0
- data/lib/saboteur/constant_reads.rb +182 -0
- data/lib/saboteur/constant_subject.rb +70 -0
- data/lib/saboteur/constant_subject_finder.rb +281 -0
- data/lib/saboteur/coverage_reads.rb +29 -0
- data/lib/saboteur/dirty_tree_warning.rb +128 -0
- data/lib/saboteur/doctor.rb +9 -0
- data/lib/saboteur/engine/baseline_guard.rb +62 -14
- data/lib/saboteur/engine/boot.rb +4 -0
- data/lib/saboteur/engine/candidate_selection.rb +19 -0
- data/lib/saboteur/engine/candidates.rb +43 -3
- data/lib/saboteur/engine/coverage_store.rb +32 -10
- data/lib/saboteur/engine/defined_order_prefix.rb +43 -0
- data/lib/saboteur/engine/diagnostics.rb +18 -0
- data/lib/saboteur/engine/distribution.rb +10 -0
- data/lib/saboteur/engine/early_stop_recording.rb +3 -2
- data/lib/saboteur/engine/gated_run.rb +35 -0
- data/lib/saboteur/engine/load_coverage.rb +45 -0
- data/lib/saboteur/engine/out_of_scope_guard.rb +21 -0
- data/lib/saboteur/engine/pipelines.rb +107 -0
- data/lib/saboteur/engine/result_cache.rb +65 -8
- data/lib/saboteur/engine/run_budget.rb +27 -0
- data/lib/saboteur/engine/sandbox_policy.rb +38 -0
- data/lib/saboteur/engine/scheduler.rb +109 -3
- data/lib/saboteur/engine/selection_announcement.rb +54 -0
- data/lib/saboteur/engine/shard.rb +69 -0
- data/lib/saboteur/engine/subject_selection.rb +295 -20
- data/lib/saboteur/engine/test_guess.rb +315 -25
- data/lib/saboteur/engine/unchanged_syntax.rb +17 -2
- data/lib/saboteur/engine.rb +100 -127
- data/lib/saboteur/execution/external_runner.rb +471 -0
- data/lib/saboteur/execution/marker_sweep.rb +79 -0
- data/lib/saboteur/execution/peak_memory.rb +28 -0
- data/lib/saboteur/execution/process_table.rb +12 -0
- data/lib/saboteur/execution/process_tree.rb +65 -5
- data/lib/saboteur/execution/reevaluator.rb +68 -12
- data/lib/saboteur/execution/runner.rb +46 -7
- data/lib/saboteur/execution/sandbox.rb +15 -2
- data/lib/saboteur/execution/seatbelt.rb +226 -0
- data/lib/saboteur/execution/supervisor.rb +27 -10
- data/lib/saboteur/execution/worker_pool.rb +23 -1
- data/lib/saboteur/frameworks/sqlite_memory_database.rb +49 -0
- data/lib/saboteur/gates.rb +43 -2
- data/lib/saboteur/help.rb +60 -9
- data/lib/saboteur/ignore_list.rb +29 -6
- data/lib/saboteur/integration/rspec.rb +135 -5
- data/lib/saboteur/mutation_source/argument_removal.rb +1 -1
- data/lib/saboteur/mutation_source/builtin.rb +26 -5
- data/lib/saboteur/mutation_source/chain_removal.rb +1 -1
- data/lib/saboteur/mutation_source/condition_negation.rb +1 -1
- data/lib/saboteur/mutation_source/constant_literals.rb +66 -0
- data/lib/saboteur/mutation_source/element_removal.rb +73 -0
- data/lib/saboteur/mutation_source/endpoint_replacement.rb +1 -1
- data/lib/saboteur/mutation_source/entry_edits.rb +44 -0
- data/lib/saboteur/mutation_source/filter_replacement.rb +1 -1
- data/lib/saboteur/mutation_source/hazard_guard.rb +108 -0
- data/lib/saboteur/mutation_source/literal_replacement.rb +1 -1
- data/lib/saboteur/mutation_source/method_swap.rb +148 -0
- data/lib/saboteur/mutation_source/operator_assignment.rb +1 -1
- data/lib/saboteur/mutation_source/operator_examples.rb +3 -1
- data/lib/saboteur/mutation_source/operator_registry.rb +4 -2
- data/lib/saboteur/mutation_source/operator_visit.rb +1 -1
- data/lib/saboteur/mutation_source/pair_removal.rb +13 -31
- data/lib/saboteur/mutation_source/range_replacement.rb +1 -1
- data/lib/saboteur/mutation_source/regex_replacement.rb +93 -0
- data/lib/saboteur/preview.rb +120 -12
- data/lib/saboteur/refinement_activations.rb +60 -0
- data/lib/saboteur/reporter/agent.rb +9 -1
- data/lib/saboteur/reporter/agent_commands.rb +10 -3
- data/lib/saboteur/reporter/agent_incomplete.rb +11 -0
- data/lib/saboteur/reporter/agent_summary.rb +4 -3
- data/lib/saboteur/reporter/findings.rb +5 -2
- data/lib/saboteur/reporter/history_policy.rb +11 -0
- data/lib/saboteur/reporter/history_reader.rb +47 -0
- data/lib/saboteur/reporter/html.rb +26 -4
- data/lib/saboteur/reporter/mutation_text.rb +6 -0
- data/lib/saboteur/reporter/stryker.rb +27 -3
- data/lib/saboteur/reporter/survivor_classification.rb +101 -0
- data/lib/saboteur/reporter/survivor_list.rb +15 -9
- data/lib/saboteur/reporter/terminal.rb +40 -7
- data/lib/saboteur/result.rb +19 -3
- data/lib/saboteur/selection/coverage_map.rb +105 -10
- data/lib/saboteur/selection/dependency_shield.rb +80 -0
- data/lib/saboteur/selection/external_test_map.rb +27 -0
- data/lib/saboteur/selection/git_diff.rb +93 -11
- data/lib/saboteur/selection/layout_only_lines.rb +80 -0
- data/lib/saboteur/selection/recorder.rb +22 -4
- data/lib/saboteur/selection/source_scope.rb +22 -0
- data/lib/saboteur/subject.rb +42 -7
- data/lib/saboteur/subject_finder.rb +173 -29
- data/lib/saboteur/version.rb +1 -1
- data/lib/saboteur/working_tree.rb +38 -15
- data/lib/saboteur.rb +13 -0
- metadata +35 -2
data/lib/saboteur/cli.rb
CHANGED
|
@@ -11,17 +11,27 @@ require_relative "baseline/command"
|
|
|
11
11
|
require_relative "cli/exit_status"
|
|
12
12
|
require_relative "preview"
|
|
13
13
|
require_relative "cli/run_reporting"
|
|
14
|
+
require_relative "cli/survivors_command"
|
|
15
|
+
require_relative "cli/source_set"
|
|
14
16
|
require_relative "cli/run_budget_options"
|
|
17
|
+
require_relative "cli/shard_options"
|
|
18
|
+
require_relative "cli/selection_options"
|
|
15
19
|
require_relative "cli/presets"
|
|
16
20
|
require_relative "cli/help_options"
|
|
21
|
+
require_relative "cli/history_command"
|
|
17
22
|
|
|
18
23
|
module Saboteur
|
|
19
24
|
class CLI
|
|
20
25
|
include Baseline::Command
|
|
21
26
|
include ExitStatus
|
|
22
27
|
include RunReporting
|
|
28
|
+
include SurvivorsCommand
|
|
29
|
+
include SourceSet
|
|
23
30
|
include RunBudgetOptions
|
|
31
|
+
include ShardOptions
|
|
32
|
+
include SelectionOptions
|
|
24
33
|
include Presets
|
|
34
|
+
include HistoryCommand
|
|
25
35
|
include HelpOptions
|
|
26
36
|
|
|
27
37
|
USAGE = "Usage: saboteur run [SOURCE...] [--test TEST...] [--all] [--threshold PERCENT] [--since REF] " \
|
|
@@ -58,11 +68,13 @@ module Saboteur
|
|
|
58
68
|
case command
|
|
59
69
|
when "run" then run_command
|
|
60
70
|
when "baseline" then baseline
|
|
61
|
-
when "
|
|
71
|
+
when "survivors" then survivors_command
|
|
72
|
+
when "doctor" then doctor_command
|
|
73
|
+
when "badge", "trend" then history_entry(command)
|
|
62
74
|
when "version", "--version" then @out.puts(VERSION) || EXIT_OK
|
|
63
75
|
when "help", "--help", "-h" then show_help(@argv.shift)
|
|
64
76
|
when "agent-skill" then @out.print(AgentSkill::TEXT) || EXIT_OK
|
|
65
|
-
else
|
|
77
|
+
else preset_or_usage_error(command)
|
|
66
78
|
end
|
|
67
79
|
rescue OptionParser::ParseError => e
|
|
68
80
|
usage_error(e.message)
|
|
@@ -98,13 +110,13 @@ module Saboteur
|
|
|
98
110
|
|
|
99
111
|
private
|
|
100
112
|
|
|
101
|
-
def run_command
|
|
102
|
-
help_requested? ? show_help("run") : run
|
|
103
|
-
end
|
|
113
|
+
def run_command = help_requested? ? show_help("run") : run
|
|
104
114
|
|
|
105
|
-
def help_requested?
|
|
106
|
-
|
|
107
|
-
|
|
115
|
+
def doctor_command = help_requested? ? show_help("doctor") : doctor
|
|
116
|
+
|
|
117
|
+
def history_entry(command) = help_requested? ? show_help(command) : history_command(command)
|
|
118
|
+
|
|
119
|
+
def help_requested? = @argv.intersect?(%w[--help -h])
|
|
108
120
|
|
|
109
121
|
def show_help(command_name)
|
|
110
122
|
help = Help.new(option_lines: option_lines)
|
|
@@ -121,9 +133,7 @@ module Saboteur
|
|
|
121
133
|
agent_failure(message || USAGE, EXIT_USAGE)
|
|
122
134
|
end
|
|
123
135
|
|
|
124
|
-
def agent?
|
|
125
|
-
@format == "agent"
|
|
126
|
-
end
|
|
136
|
+
def agent? = @format == "agent"
|
|
127
137
|
|
|
128
138
|
def agent_requested? = self.class.agent_requested?(@argv)
|
|
129
139
|
|
|
@@ -144,6 +154,8 @@ module Saboteur
|
|
|
144
154
|
|
|
145
155
|
return preview if options[:list_operators] || options[:dry_run]
|
|
146
156
|
|
|
157
|
+
refuse_empty_source_set
|
|
158
|
+
refuse_empty_since_range
|
|
147
159
|
execute_run
|
|
148
160
|
rescue ConfigError => e
|
|
149
161
|
@err.puts("saboteur: #{e.message}")
|
|
@@ -155,8 +167,12 @@ module Saboteur
|
|
|
155
167
|
|
|
156
168
|
# Mutants run your code, and a mutant of code that writes or deletes files
|
|
157
169
|
# can change the tree. Say so up front when uncommitted work is at stake.
|
|
170
|
+
# The scope the warning names is the configured one: positional source
|
|
171
|
+
# operands narrow the run after the warning, so the include/exclude they
|
|
172
|
+
# imply stay out of its config (#309, #503 review).
|
|
158
173
|
def warn_dirty_tree
|
|
159
|
-
|
|
174
|
+
warning = @sources ? Config.load(path: options[:config], overrides: overrides.except(:include, :exclude)) : config
|
|
175
|
+
DirtyTreeWarning.for(warning, config_path: options[:config])&.then { |text| @err.puts(text) }
|
|
160
176
|
end
|
|
161
177
|
|
|
162
178
|
# Contradictory selection flags are a usage error before any file or
|
|
@@ -165,30 +181,46 @@ module Saboteur
|
|
|
165
181
|
# --test wins there, so generated commands stay valid over any config.
|
|
166
182
|
def reject_test_selection_conflicts
|
|
167
183
|
raise UsageError, "--all cannot be combined with --test" if options[:all] && options[:tests]&.any?
|
|
184
|
+
|
|
185
|
+
reject_test_command_conflicts
|
|
168
186
|
raise UsageError, "--all cannot be combined with --guess-tests" if options[:all] && options[:guess_tests]
|
|
169
187
|
return unless options[:guess_tests] && options[:tests]&.any?
|
|
170
188
|
|
|
171
189
|
raise UsageError, "--guess-tests cannot be combined with --test"
|
|
172
190
|
end
|
|
173
191
|
|
|
192
|
+
# --test-command replaces the whole in-process test machinery: test
|
|
193
|
+
# selection, coverage, parallelism and the sandbox do not apply.
|
|
194
|
+
def reject_test_command_conflicts
|
|
195
|
+
return unless options[:test_command]
|
|
196
|
+
|
|
197
|
+
# options[:tests] is nil or a non-empty list, so truthiness is the check.
|
|
198
|
+
raise UsageError, "--test-command cannot be combined with --test" if options[:tests]
|
|
199
|
+
raise UsageError, "--test-command cannot be combined with --incremental" if options[:incremental]
|
|
200
|
+
end
|
|
201
|
+
|
|
174
202
|
# Usage errors found before the target boots.
|
|
175
203
|
def validate_run_options
|
|
176
|
-
|
|
204
|
+
reject_format_conflicts
|
|
205
|
+
validate_shard_options
|
|
177
206
|
Selection::GitDiff.verify_ref!(config["since"], root: config.root) if config["since"]
|
|
178
207
|
end
|
|
179
208
|
|
|
180
|
-
# Agent mode
|
|
181
|
-
def
|
|
182
|
-
|
|
209
|
+
# Agent mode allows no score gates (one strict exit 0); the json inventory is --dry-run planning only.
|
|
210
|
+
def reject_format_conflicts
|
|
211
|
+
if agent? && (config["threshold"] || gates.any?)
|
|
212
|
+
raise UsageError, "#{config['threshold'] ? '--threshold' : gates.agent_option_name} " \
|
|
213
|
+
"cannot be combined with --agent"
|
|
214
|
+
end
|
|
215
|
+
return unless @format == "json" && !options[:dry_run]
|
|
183
216
|
|
|
184
|
-
raise UsageError, "
|
|
217
|
+
raise UsageError, "--format json is the --dry-run mutant inventory; a run summary is --format agent"
|
|
185
218
|
end
|
|
186
219
|
|
|
187
220
|
# One line naming the reports written, paths relative to the root when
|
|
188
221
|
# they sit under it (a blank line before it, as the old JSON line had).
|
|
189
222
|
def announce_reports(reports)
|
|
190
|
-
written = reports.select { |path, _label| path }
|
|
191
|
-
return if written.empty?
|
|
223
|
+
return if (written = reports.select { |path, _label| path }).empty?
|
|
192
224
|
|
|
193
225
|
root = File.join(File.realpath(config.root), "")
|
|
194
226
|
names = written.map do |path, label|
|
|
@@ -233,10 +265,14 @@ module Saboteur
|
|
|
233
265
|
subjects: Array(config["subjects"]), tests: Array(engine.guessed_tests || config["tests"]),
|
|
234
266
|
sources: @sources || [],
|
|
235
267
|
config_path: options[:config], digest: options[:id], cache: config["cache"],
|
|
236
|
-
incremental: config["incremental"], settings: agent_settings
|
|
268
|
+
incremental: config["incremental"], settings: agent_settings, typed_settings: typed_settings
|
|
237
269
|
)
|
|
238
270
|
end
|
|
239
271
|
|
|
272
|
+
# The settings keys typed on the command line: a typed empty operator
|
|
273
|
+
# list may clear a config value, so the generated commands keep it.
|
|
274
|
+
def typed_settings = options.keys & Reporter::Agent::Commands::SETTING_FLAGS.keys
|
|
275
|
+
|
|
240
276
|
# A whole-suite run records its scope as --all, so the generated rerun
|
|
241
277
|
# and replay commands do not default back to inference (which can exit
|
|
242
278
|
# 2 in a project with a spec-less source file).
|
|
@@ -271,17 +307,15 @@ module Saboteur
|
|
|
271
307
|
# replace the configured include and exclude patterns. Each must exist,
|
|
272
308
|
# so a typo is an error and not a run over zero subjects.
|
|
273
309
|
def use_explicit_sources
|
|
274
|
-
|
|
275
|
-
|
|
276
|
-
|
|
277
|
-
|
|
278
|
-
|
|
279
|
-
end
|
|
310
|
+
Selection::SourceScope.verify_no_specs!(@argv)
|
|
311
|
+
options[:tests]&.each { |test| require_existing_test(test) }
|
|
312
|
+
# --test may stand alone: the configured include (or the --since
|
|
313
|
+
# diff) names the sources. The files must still exist.
|
|
314
|
+
return if @argv.empty?
|
|
280
315
|
|
|
281
316
|
@sources = @argv.dup
|
|
282
317
|
options[:include] = @argv.map { |source| source_pattern(source) }
|
|
283
318
|
options[:exclude] = []
|
|
284
|
-
options[:tests]&.each { |test| require_existing_test(test) }
|
|
285
319
|
end
|
|
286
320
|
|
|
287
321
|
def source_pattern(source)
|
|
@@ -303,12 +337,10 @@ module Saboteur
|
|
|
303
337
|
raise UsageError, "test file not found: #{test}" unless File.exist?(path)
|
|
304
338
|
end
|
|
305
339
|
|
|
306
|
-
def overrides
|
|
307
|
-
options.except(:config, :id, :suite, :list_operators, :dry_run)
|
|
308
|
-
end
|
|
340
|
+
def overrides = options.except(:config, :id, :suite, :list_operators, :dry_run)
|
|
309
341
|
|
|
310
342
|
def preview
|
|
311
|
-
raise UsageError, "--dry-run and --list-operators print text, not --agent JSON" if agent?
|
|
343
|
+
raise UsageError, "--dry-run and --list-operators print text, not --agent JSON; use --format json" if agent?
|
|
312
344
|
raise UsageError, "--id replays one mutant: not with --dry-run or --list-operators" if options[:id]
|
|
313
345
|
|
|
314
346
|
if options[:list_operators]
|
|
@@ -316,24 +348,10 @@ module Saboteur
|
|
|
316
348
|
return EXIT_OK
|
|
317
349
|
end
|
|
318
350
|
|
|
319
|
-
Preview.new(config, out: @out, err: @err).call
|
|
351
|
+
Preview.new(config, out: @out, err: @err, format: @format).call
|
|
320
352
|
EXIT_OK
|
|
321
353
|
end
|
|
322
354
|
|
|
323
|
-
def define_selection_options(opts)
|
|
324
|
-
opts.on("--operators NAMES", Array, "Choose operator names (default: all)") do |v|
|
|
325
|
-
set(:operators, v)
|
|
326
|
-
end
|
|
327
|
-
opts.on("--exclude-operators NAMES", Array, "Exclude operator names from the selected set") do |v|
|
|
328
|
-
set(:exclude_operators, v)
|
|
329
|
-
end
|
|
330
|
-
opts.on("--list-operators", "List operator names and descriptions") { set(:list_operators, true) }
|
|
331
|
-
opts.on("--dry-run", "Preview mutants without boot, tests, forks or report writes") { set(:dry_run, true) }
|
|
332
|
-
opts.on("--id DIGEST", "Replay exactly one current mutant by its full digest") { |v| set(:id, v) }
|
|
333
|
-
opts.on("--since REF", "Only mutate subjects changed since git REF") { |v| options[:since] = v }
|
|
334
|
-
opts.on("--subject EXPR", "Klass#meth, Klass.meth or a namespace; repeatable") { |v| add_subject(v) }
|
|
335
|
-
end
|
|
336
|
-
|
|
337
355
|
def define_execution_options(opts)
|
|
338
356
|
opts.on("--timeout-factor N", Float, "Multiple of baseline runtime (default 3)") { |v| set(:timeout_factor, v) }
|
|
339
357
|
opts.on("--jobs N", Integer, "Concurrent workers (default 1)") { |v| set(:jobs, v) }
|
|
@@ -352,6 +370,10 @@ module Saboteur
|
|
|
352
370
|
end
|
|
353
371
|
opts.on("--all", "Run the whole configured test suite instead of inferring the specs named after the " \
|
|
354
372
|
"mutated files") { set(:all, true) }
|
|
373
|
+
opts.on("--test-command CMD", "Run CMD per mutant in the project root; exit 0 means the mutant survived. " \
|
|
374
|
+
"For apps that cannot load in this Ruby; much slower, serial, no coverage") do |v|
|
|
375
|
+
set(:test_command, v)
|
|
376
|
+
end
|
|
355
377
|
opts.on("--[no-]guess-tests", "Deprecated alias: inferred specs are already the default; " \
|
|
356
378
|
"--no-guess-tests warns and is ignored - use --all for the whole suite") do |v|
|
|
357
379
|
set(:guess_tests, v)
|
|
@@ -368,8 +390,7 @@ module Saboteur
|
|
|
368
390
|
opts.on("--score-history PATH", "Append score history JSONL (opt-in)") { |v| set(:score_history, v) }
|
|
369
391
|
opts.on("--json PATH", "Stryker JSON report path") { |v| set(:json_report, v) }
|
|
370
392
|
opts.on("--html PATH", "Self-contained HTML report path") { |v| set(:html_report, v) }
|
|
371
|
-
opts.on("--format FORMAT", %w[text agent], "text
|
|
372
|
-
"strict exit codes") { |v| @format = v }
|
|
393
|
+
opts.on("--format FORMAT", %w[text agent json], "text, agent or json (--dry-run inventory)") { |v| @format = v }
|
|
373
394
|
opts.on("--agent", "Same as --format agent") { @format = "agent" }
|
|
374
395
|
opts.on("--verbose", "Text report: also print score split, timings and kill causes;",
|
|
375
396
|
"a crash prints its backtrace") { @verbose = true }
|
|
@@ -388,12 +409,8 @@ module Saboteur
|
|
|
388
409
|
Gates.define_options(opts) { |key, value| set(key, value) }
|
|
389
410
|
end
|
|
390
411
|
|
|
391
|
-
def add_subject(expr)
|
|
392
|
-
(options[:subjects] ||= []) << expr
|
|
393
|
-
end
|
|
412
|
+
def add_subject(expr) = (options[:subjects] ||= []) << expr
|
|
394
413
|
|
|
395
|
-
def set(key, value)
|
|
396
|
-
options[key] = value
|
|
397
|
-
end
|
|
414
|
+
def set(key, value) = (options[key] = value)
|
|
398
415
|
end
|
|
399
416
|
end
|
data/lib/saboteur/config.rb
CHANGED
|
@@ -22,13 +22,27 @@ module Saboteur
|
|
|
22
22
|
"integration" => "rspec",
|
|
23
23
|
"require" => [],
|
|
24
24
|
"include" => ["lib/**/*.rb"],
|
|
25
|
-
|
|
25
|
+
# Spec and test files vendored under lib/ (capybara ships its driver
|
|
26
|
+
# suite at lib/capybara/spec/**) are never mutation subjects (#504).
|
|
27
|
+
# A config's own c.exclude replaces this list wholesale.
|
|
28
|
+
"exclude" => ["lib/**/spec/**/*", "lib/**/test/**/*", "lib/**/tests/**/*",
|
|
29
|
+
"lib/**/*_spec.rb", "lib/**/*_test.rb", "lib/**/test_*.rb"],
|
|
26
30
|
"arid" => MutationSource::Builtin::DEFAULT_ARID,
|
|
27
31
|
"whole_file" => [],
|
|
32
|
+
# [{ "pattern" => "app/payments/**/*", "min_score" => 95.0 }], from
|
|
33
|
+
# repeatable c.score_gate calls (#473). --threshold stays the global
|
|
34
|
+
# fallback.
|
|
35
|
+
"score_gates" => [],
|
|
28
36
|
"hazard" => MutationSource::HazardGuard::DEFAULT_HAZARD,
|
|
29
37
|
"allow_hazardous" => false,
|
|
30
|
-
|
|
38
|
+
# true, false, "strict" or "seatbelt": strict refuses to run without full Landlock;
|
|
39
|
+
# seatbelt opts into the macOS Seatbelt sandbox (on Linux it behaves like true).
|
|
40
|
+
"sandbox" => true,
|
|
31
41
|
"sandbox_writable" => [],
|
|
42
|
+
# off, survivors, or decisive (#476): one reordered confirmation run
|
|
43
|
+
# per new survivor (and per fresh kill in decisive); a disagreement
|
|
44
|
+
# becomes unverified, outside the score.
|
|
45
|
+
"flaky_check" => "off",
|
|
32
46
|
"timeout_factor" => 3.0,
|
|
33
47
|
"timeout_min" => 5.0,
|
|
34
48
|
"baseline_timeout" => 300.0,
|
|
@@ -56,6 +70,7 @@ module Saboteur
|
|
|
56
70
|
"since" => nil,
|
|
57
71
|
"cache" => true,
|
|
58
72
|
"incremental" => false,
|
|
73
|
+
"shard" => nil,
|
|
59
74
|
"allow_focus" => false,
|
|
60
75
|
"fail_fast" => false,
|
|
61
76
|
"guess_tests" => false,
|
|
@@ -67,6 +82,7 @@ module Saboteur
|
|
|
67
82
|
"max_duration" => nil,
|
|
68
83
|
"max_duration_ok" => false,
|
|
69
84
|
"tests" => [],
|
|
85
|
+
"test_command" => nil,
|
|
70
86
|
"fingerprint_extra" => [],
|
|
71
87
|
"threshold" => nil,
|
|
72
88
|
"max_timeouts" => nil,
|
|
@@ -160,14 +176,30 @@ module Saboteur
|
|
|
160
176
|
matched = self["include"].flat_map { |pattern| Dir.glob(pattern, base: root) }
|
|
161
177
|
excluded = self["exclude"].flat_map { |pattern| Dir.glob(pattern, base: root) }
|
|
162
178
|
(matched - excluded).select { |file| file.end_with?(".rb") }
|
|
163
|
-
.
|
|
179
|
+
.filter_map { |file| real_source_path(file) }
|
|
164
180
|
.uniq.sort
|
|
165
181
|
end
|
|
166
182
|
|
|
183
|
+
# A broken symlink matches Dir.glob but has no real path: dangling
|
|
184
|
+
# (ENOENT), a loop (ELOOP) or through a regular file (ENOTDIR). It is
|
|
185
|
+
# not mutation source, so it is skipped rather than crashing the run
|
|
186
|
+
# (#503). A regular file that cannot be resolved still raises.
|
|
187
|
+
def real_source_path(file)
|
|
188
|
+
path = File.expand_path(Text.utf8_identity(file), root)
|
|
189
|
+
File.realpath(path)
|
|
190
|
+
rescue SystemCallError
|
|
191
|
+
raise unless File.symlink?(path)
|
|
192
|
+
end
|
|
193
|
+
|
|
167
194
|
# Patterns that match no file. A pattern goes stale when files move;
|
|
168
195
|
# an exclude that silently stops excluding is a hazard, so say so.
|
|
169
196
|
def unmatched_patterns
|
|
170
197
|
%w[include exclude].flat_map do |key|
|
|
198
|
+
# The default exclude shields (#504) match nothing on a healthy
|
|
199
|
+
# project - that is their job - so only a configured exclude can go
|
|
200
|
+
# stale enough to warn about.
|
|
201
|
+
next [] if key == "exclude" && !explicit?("exclude")
|
|
202
|
+
|
|
171
203
|
self[key].select { |pattern| Dir.glob(pattern, base: root).empty? }.map { |pattern| "#{key}: #{pattern}" }
|
|
172
204
|
end
|
|
173
205
|
end
|
|
@@ -237,6 +269,41 @@ module Saboteur
|
|
|
237
269
|
validate_matrix!
|
|
238
270
|
validate_test_selection!
|
|
239
271
|
validate_gates!
|
|
272
|
+
validate_score_gates!
|
|
273
|
+
end
|
|
274
|
+
|
|
275
|
+
# Per-path gates (#473): normalize each entry to string pattern /
|
|
276
|
+
# float floor, and fail loud on a pattern no included file matches -
|
|
277
|
+
# a gate that matches nothing would pass vacuously forever.
|
|
278
|
+
def validate_score_gates!
|
|
279
|
+
gates = self["score_gates"]
|
|
280
|
+
return if gates.empty?
|
|
281
|
+
|
|
282
|
+
normalized = gates.map { |gate| normalize_score_gate(gate) }
|
|
283
|
+
prefix = File.join(File.realpath(root), "")
|
|
284
|
+
relative = included_files.map { |file| file.delete_prefix(prefix) }
|
|
285
|
+
flags = File::FNM_PATHNAME | File::FNM_EXTGLOB
|
|
286
|
+
normalized.each do |gate|
|
|
287
|
+
next if relative.any? { |file| File.fnmatch?(gate["pattern"], file, flags) }
|
|
288
|
+
|
|
289
|
+
raise ConfigError, "score_gate #{gate['pattern']} matches no included file"
|
|
290
|
+
end
|
|
291
|
+
@values["score_gates"] = normalized
|
|
292
|
+
end
|
|
293
|
+
|
|
294
|
+
def normalize_score_gate(gate)
|
|
295
|
+
pattern = gate["pattern"].to_s
|
|
296
|
+
begin
|
|
297
|
+
floor = Float(gate["min_score"])
|
|
298
|
+
rescue ArgumentError, TypeError
|
|
299
|
+
raise ConfigError, "score_gate min_score must be between 0 and 100, got #{gate['min_score'].inspect}"
|
|
300
|
+
end
|
|
301
|
+
unless floor.finite? && floor.between?(0, 100)
|
|
302
|
+
raise ConfigError, "score_gate min_score must be between 0 and 100, got #{gate['min_score'].inspect}"
|
|
303
|
+
end
|
|
304
|
+
raise ConfigError, "score_gate pattern must not be empty" if pattern.empty?
|
|
305
|
+
|
|
306
|
+
{ "pattern" => pattern, "min_score" => floor }
|
|
240
307
|
end
|
|
241
308
|
|
|
242
309
|
def validate_gates!
|
|
@@ -270,6 +337,28 @@ module Saboteur
|
|
|
270
337
|
raise ConfigError, "test_selection must be one of: auto, all"
|
|
271
338
|
end
|
|
272
339
|
|
|
340
|
+
# --test-command replaces the whole in-process machinery, on the
|
|
341
|
+
# EFFECTIVE config: CLI flags and config-file entries alike (#305).
|
|
342
|
+
# The command runs unsandboxed, so "strict" cannot be honored.
|
|
343
|
+
def validate_test_command!
|
|
344
|
+
command = self["test_command"]
|
|
345
|
+
# nil alone means unset: false, arrays and numbers are all rejected,
|
|
346
|
+
# never coerced and never sent to String methods (#305 QA).
|
|
347
|
+
raise ConfigError, "test_command must be a string" unless command.nil? || command.is_a?(String)
|
|
348
|
+
raise ConfigError, "test_command must not be blank" if command&.strip&.empty?
|
|
349
|
+
|
|
350
|
+
return if command.to_s.empty?
|
|
351
|
+
raise ConfigError, "test_command cannot be combined with tests" unless self["tests"].empty?
|
|
352
|
+
raise ConfigError, "test_command cannot be combined with incremental" if self["incremental"]
|
|
353
|
+
unless self["flaky_check"] == "off"
|
|
354
|
+
raise ConfigError, "test_command cannot be combined with flaky_check: an external run has no " \
|
|
355
|
+
"in-process scheduler to rerun the decisive tests through"
|
|
356
|
+
end
|
|
357
|
+
return unless self["sandbox"] == "strict"
|
|
358
|
+
|
|
359
|
+
raise ConfigError, 'test_command runs the command unsandboxed, so sandbox "strict" cannot be honored'
|
|
360
|
+
end
|
|
361
|
+
|
|
273
362
|
def validate_reeval!
|
|
274
363
|
raise ConfigError, "reeval must be one of: #{Execution::Reevaluator::STRATEGIES.join(', ')}" unless
|
|
275
364
|
Execution::Reevaluator::STRATEGIES.include?(self["reeval"].to_s)
|
|
@@ -277,8 +366,11 @@ module Saboteur
|
|
|
277
366
|
|
|
278
367
|
def validate!
|
|
279
368
|
raise ConfigError, "jobs must be >= 1" if self["jobs"] < 1
|
|
280
|
-
unless [true, false, "strict"].include?(self["sandbox"])
|
|
281
|
-
raise ConfigError, 'sandbox must be true, false, or "
|
|
369
|
+
unless [true, false, "strict", "seatbelt"].include?(self["sandbox"])
|
|
370
|
+
raise ConfigError, 'sandbox must be true, false, "strict", or "seatbelt"'
|
|
371
|
+
end
|
|
372
|
+
unless %w[off survivors decisive].include?(self["flaky_check"])
|
|
373
|
+
raise ConfigError, "flaky_check must be one of: off, survivors, decisive"
|
|
282
374
|
end
|
|
283
375
|
|
|
284
376
|
validate_mutant_limit!
|
|
@@ -292,6 +384,7 @@ module Saboteur
|
|
|
292
384
|
|
|
293
385
|
validate_reeval!
|
|
294
386
|
validate_operator_config!
|
|
387
|
+
validate_test_command!
|
|
295
388
|
end
|
|
296
389
|
|
|
297
390
|
public
|
data/lib/saboteur/config_file.rb
CHANGED
|
@@ -26,6 +26,14 @@ module Saboteur
|
|
|
26
26
|
end
|
|
27
27
|
end
|
|
28
28
|
|
|
29
|
+
# Repeatable per-path score gates (#473), the one setting that takes
|
|
30
|
+
# arguments: c.score_gate "app/payments/**/*", min_score: 95. Values
|
|
31
|
+
# are validated later, on the effective Config.
|
|
32
|
+
def score_gate(pattern, min_score:)
|
|
33
|
+
(@values["score_gates"] ||= []) << { "pattern" => pattern, "min_score" => min_score }
|
|
34
|
+
@lines["score_gates"] = caller_locations(1, 1).first.lineno
|
|
35
|
+
end
|
|
36
|
+
|
|
29
37
|
Config::INTEGER_KEYS.each do |key|
|
|
30
38
|
define_method("#{key}=") do |value|
|
|
31
39
|
unless value.is_a?(Integer)
|
|
@@ -0,0 +1,182 @@
|
|
|
1
|
+
# SPDX-License-Identifier: MIT
|
|
2
|
+
# frozen_string_literal: true
|
|
3
|
+
|
|
4
|
+
require "prism"
|
|
5
|
+
require_relative "text"
|
|
6
|
+
|
|
7
|
+
module Saboteur
|
|
8
|
+
# The lines where a constant is read, per file: the tests covering those
|
|
9
|
+
# lines are the ones that can observe a change to the constant's value
|
|
10
|
+
# (issue #306).
|
|
11
|
+
#
|
|
12
|
+
# Matching is lexical and honest about its edges. A bare `RULES` matches
|
|
13
|
+
# when its scope is the constant's owner or nested under it (constant
|
|
14
|
+
# lookup climbs lexical scopes) or the constant is top-level; a
|
|
15
|
+
# `Shop::RULES` path matches by its full spelling. A scope that writes its
|
|
16
|
+
# own `RULES` shadows the outer constant: reads under it resolve to the
|
|
17
|
+
# inner one and do not count. Reads reached through a superclass,
|
|
18
|
+
# `const_get`, or a compact `class Shop::Item` scope are not resolved: a
|
|
19
|
+
# constant only read that way finds no consumer here, and the run reports
|
|
20
|
+
# it as ran-at-load instead of guessing.
|
|
21
|
+
#
|
|
22
|
+
# Runtime reads (inside a def body) observe the constant when the mutant
|
|
23
|
+
# runs; load-time reads outside the constant's own file do not: the other
|
|
24
|
+
# file was loaded once with the original value and re-evaluating the
|
|
25
|
+
# constant's file does not run it again, so a read like
|
|
26
|
+
# `COPY = A::RULES` there keeps the original (#319). A capture inside a
|
|
27
|
+
# boot-time block (`1.times { COPY = A::RULES }`) is treated the same:
|
|
28
|
+
# statement-level blocks run while the file loads. That is a heuristic -
|
|
29
|
+
# a block passed to a deferring call (at_exit, define_method) runs later,
|
|
30
|
+
# but treating it as load-time errs toward dropping a constant rather
|
|
31
|
+
# than reporting a mutant nobody can observe.
|
|
32
|
+
module ConstantReads
|
|
33
|
+
Reads = Data.define(:runtime, :load, :defs)
|
|
34
|
+
|
|
35
|
+
module_function
|
|
36
|
+
|
|
37
|
+
# { "/abs/file.rb" => Reads } for the files that read the constant.
|
|
38
|
+
# runtime: def-body and block lines - a block runs when its caller runs
|
|
39
|
+
# it (an `it` block during the example), and line coverage decides
|
|
40
|
+
# whether any test reached it. load: top-level or class-body statement
|
|
41
|
+
# lines (observable only in the constant's own file, which the mutant
|
|
42
|
+
# re-evaluates). defs: the start line of each def a runtime read sits
|
|
43
|
+
# in, for the method-call counter fallback (endless defs have no line
|
|
44
|
+
# coverage). Only files whose text mentions the name are parsed.
|
|
45
|
+
def across(files, constant)
|
|
46
|
+
mention = /\b#{Regexp.escape(constant.name.to_s)}\b/
|
|
47
|
+
files.each_with_object({}) do |file, hits|
|
|
48
|
+
source = File.binread(file)
|
|
49
|
+
next unless mentions?(source, mention)
|
|
50
|
+
|
|
51
|
+
reads = reads_in(source, constant)
|
|
52
|
+
hits[file] = reads unless reads.runtime.empty? && reads.load.empty?
|
|
53
|
+
end
|
|
54
|
+
end
|
|
55
|
+
|
|
56
|
+
# The lines a mutant of the constant can be observed through: runtime
|
|
57
|
+
# reads everywhere, load reads in the constant's own file only.
|
|
58
|
+
def observable_lines(reads_by_file, constant)
|
|
59
|
+
reads_by_file.each_with_object({}) do |(file, reads), hits|
|
|
60
|
+
lines = reads.runtime.dup
|
|
61
|
+
lines.concat(reads.load) if file == constant.file
|
|
62
|
+
hits[file] = lines.uniq.sort unless lines.empty?
|
|
63
|
+
end
|
|
64
|
+
end
|
|
65
|
+
|
|
66
|
+
def mentions?(source, mention)
|
|
67
|
+
Text.utf8_identity(source).match?(mention)
|
|
68
|
+
rescue Encoding::CompatibilityError, ArgumentError
|
|
69
|
+
false
|
|
70
|
+
end
|
|
71
|
+
|
|
72
|
+
def reads_in(source, constant)
|
|
73
|
+
result = Prism.parse(source)
|
|
74
|
+
return Reads.new([], [], []) if result.failure?
|
|
75
|
+
|
|
76
|
+
out = { runtime: [], load: [], defs: [] }
|
|
77
|
+
walk(result.value, [], constant, out, in_def: nil, in_block: false, shadowed: nil)
|
|
78
|
+
Reads.new(out[:runtime].uniq.sort, out[:load].uniq.sort, out[:defs].uniq.sort)
|
|
79
|
+
end
|
|
80
|
+
|
|
81
|
+
def walk(node, nesting, constant, out, in_def:, in_block:, shadowed:, capture: false)
|
|
82
|
+
return if node.nil?
|
|
83
|
+
|
|
84
|
+
case node
|
|
85
|
+
when Prism::DefNode
|
|
86
|
+
walk(node.body, nesting, constant, out, in_def: node.location.start_line, in_block: false,
|
|
87
|
+
shadowed: shadowed)
|
|
88
|
+
when Prism::BlockNode, Prism::LambdaNode
|
|
89
|
+
walk(node.body, nesting, constant, out, in_def: in_def, in_block: true, capture: capture,
|
|
90
|
+
shadowed: shadowed)
|
|
91
|
+
when Prism::ConstantWriteNode, Prism::ConstantPathWriteNode
|
|
92
|
+
# A capture (`COPY = <expr>`): the value reads the constant once,
|
|
93
|
+
# when the write runs. Walk the value flagged as a capture so a
|
|
94
|
+
# boot-time block write (`1.times { COPY = A::RULES }`) lands in
|
|
95
|
+
# load, not runtime (#319).
|
|
96
|
+
walk(node.value, nesting, constant, out, in_def: in_def, in_block: in_block, capture: true,
|
|
97
|
+
shadowed: shadowed)
|
|
98
|
+
when Prism::ConstantReadNode
|
|
99
|
+
record(node, out, in_def, in_block, capture) if !shadowed && bare_match?(node, nesting, constant)
|
|
100
|
+
when Prism::ConstantPathNode
|
|
101
|
+
# `Shop::RULES` holds no ConstantReadNode for RULES, only for Shop;
|
|
102
|
+
# match the path's full spelling.
|
|
103
|
+
record(node, out, in_def, in_block, capture) if node.slice.delete_prefix("::") == constant.id
|
|
104
|
+
walk_children(node, nesting, constant, out, in_def: in_def, in_block: in_block, capture: capture,
|
|
105
|
+
shadowed: shadowed)
|
|
106
|
+
when Prism::ModuleNode, Prism::ClassNode, Prism::SingletonClassNode
|
|
107
|
+
walk_scope(node, nesting, constant, out, shadowed)
|
|
108
|
+
else
|
|
109
|
+
walk_children(node, nesting, constant, out, in_def: in_def, in_block: in_block, capture: capture,
|
|
110
|
+
shadowed: shadowed)
|
|
111
|
+
end
|
|
112
|
+
end
|
|
113
|
+
|
|
114
|
+
# `class << self` joins the nesting as "<attached>.singleton_class",
|
|
115
|
+
# matching how ConstantSubjectFinder names that owner (#380); a non-self
|
|
116
|
+
# receiver or a top-level singleton has no static name and adds nothing.
|
|
117
|
+
# The shadow carries the path of the scope that set it. Entering the
|
|
118
|
+
# constant's owner scope lifts a shadow set outside it: the owner's
|
|
119
|
+
# own definition is lexically closer there (#395). A shadow set
|
|
120
|
+
# inside the owner subtree still binds below the nested scope that
|
|
121
|
+
# wrote the constant.
|
|
122
|
+
def walk_scope(node, nesting, constant, out, shadowed)
|
|
123
|
+
inner = scope_nesting(node, nesting)
|
|
124
|
+
path = inner.join("::")
|
|
125
|
+
scopes = [constant.owner.to_s]
|
|
126
|
+
shadowed = nil if shadowed && scopes.any? { |scope| path == scope || path.start_with?("#{scope}::") } &&
|
|
127
|
+
scopes.none? { |scope| shadowed == scope || shadowed.start_with?("#{scope}::") }
|
|
128
|
+
shadowed = path if shadowed.nil? && path != scopes.first && writes_constant?(node.body, constant.name)
|
|
129
|
+
walk(node.body, inner, constant, out, in_def: nil, in_block: false, shadowed: shadowed)
|
|
130
|
+
end
|
|
131
|
+
|
|
132
|
+
def scope_nesting(node, nesting)
|
|
133
|
+
return nesting + [node.constant_path.slice.delete_prefix("::")] unless node.is_a?(Prism::SingletonClassNode)
|
|
134
|
+
return nesting unless node.expression.is_a?(Prism::SelfNode) && nesting.any?
|
|
135
|
+
|
|
136
|
+
nesting[0...-1] + ["#{nesting.last}.singleton_class"]
|
|
137
|
+
end
|
|
138
|
+
|
|
139
|
+
def walk_children(node, nesting, constant, out, in_def:, in_block:, capture:, shadowed:)
|
|
140
|
+
node.compact_child_nodes.each do |child|
|
|
141
|
+
walk(child, nesting, constant, out, in_def: in_def, in_block: in_block, capture: capture,
|
|
142
|
+
shadowed: shadowed)
|
|
143
|
+
end
|
|
144
|
+
end
|
|
145
|
+
|
|
146
|
+
def record(node, out, in_def, in_block, capture)
|
|
147
|
+
line = node.location.start_line
|
|
148
|
+
if in_def
|
|
149
|
+
out[:runtime] << line
|
|
150
|
+
out[:defs] << in_def
|
|
151
|
+
elsif in_block && !capture
|
|
152
|
+
out[:runtime] << line
|
|
153
|
+
else
|
|
154
|
+
# A bare statement read, or a capture in a boot-time block: both
|
|
155
|
+
# resolved while the file loaded, with the original value.
|
|
156
|
+
out[:load] << line
|
|
157
|
+
end
|
|
158
|
+
end
|
|
159
|
+
|
|
160
|
+
# A bare `RULES` reads this constant when the read's scope is the
|
|
161
|
+
# constant's owner (or its singleton scope, whose ancestry reaches the
|
|
162
|
+
# attached class) or nested under either, or it is top-level (#380).
|
|
163
|
+
def bare_match?(node, nesting, constant)
|
|
164
|
+
return false unless node.name == constant.name
|
|
165
|
+
return true if constant.owner.nil?
|
|
166
|
+
|
|
167
|
+
path = nesting.join("::")
|
|
168
|
+
[constant.owner, "#{constant.owner}.singleton_class"].any? do |scope|
|
|
169
|
+
path == scope || path.start_with?("#{scope}::")
|
|
170
|
+
end
|
|
171
|
+
end
|
|
172
|
+
|
|
173
|
+
# True when the scope body assigns the constant name at its own level
|
|
174
|
+
# (not inside a nested def or scope).
|
|
175
|
+
def writes_constant?(body, name)
|
|
176
|
+
statements = body&.body || []
|
|
177
|
+
statements.any? do |statement|
|
|
178
|
+
statement.is_a?(Prism::ConstantWriteNode) && statement.name == name
|
|
179
|
+
end
|
|
180
|
+
end
|
|
181
|
+
end
|
|
182
|
+
end
|