saboteur 0.7.0 → 0.8.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (107) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +657 -0
  3. data/README.md +198 -1077
  4. data/lib/saboteur/cli/exit_status.rb +32 -1
  5. data/lib/saboteur/cli/help_options.rb +2 -1
  6. data/lib/saboteur/cli/history_command.rb +135 -0
  7. data/lib/saboteur/cli/presets.rb +6 -2
  8. data/lib/saboteur/cli/selection_options.rb +31 -0
  9. data/lib/saboteur/cli/shard_options.rb +17 -0
  10. data/lib/saboteur/cli/source_set.rb +53 -0
  11. data/lib/saboteur/cli/survivors_command.rb +157 -0
  12. data/lib/saboteur/cli.rb +71 -54
  13. data/lib/saboteur/config.rb +98 -5
  14. data/lib/saboteur/config_file.rb +8 -0
  15. data/lib/saboteur/constant_reads.rb +182 -0
  16. data/lib/saboteur/constant_subject.rb +70 -0
  17. data/lib/saboteur/constant_subject_finder.rb +281 -0
  18. data/lib/saboteur/coverage_reads.rb +29 -0
  19. data/lib/saboteur/dirty_tree_warning.rb +128 -0
  20. data/lib/saboteur/doctor.rb +9 -0
  21. data/lib/saboteur/engine/baseline_guard.rb +62 -14
  22. data/lib/saboteur/engine/boot.rb +4 -0
  23. data/lib/saboteur/engine/candidate_selection.rb +19 -0
  24. data/lib/saboteur/engine/candidates.rb +43 -3
  25. data/lib/saboteur/engine/coverage_store.rb +32 -10
  26. data/lib/saboteur/engine/defined_order_prefix.rb +43 -0
  27. data/lib/saboteur/engine/diagnostics.rb +18 -0
  28. data/lib/saboteur/engine/distribution.rb +10 -0
  29. data/lib/saboteur/engine/early_stop_recording.rb +3 -2
  30. data/lib/saboteur/engine/gated_run.rb +35 -0
  31. data/lib/saboteur/engine/load_coverage.rb +45 -0
  32. data/lib/saboteur/engine/out_of_scope_guard.rb +21 -0
  33. data/lib/saboteur/engine/pipelines.rb +107 -0
  34. data/lib/saboteur/engine/result_cache.rb +65 -8
  35. data/lib/saboteur/engine/run_budget.rb +27 -0
  36. data/lib/saboteur/engine/sandbox_policy.rb +38 -0
  37. data/lib/saboteur/engine/scheduler.rb +109 -3
  38. data/lib/saboteur/engine/selection_announcement.rb +54 -0
  39. data/lib/saboteur/engine/shard.rb +69 -0
  40. data/lib/saboteur/engine/subject_selection.rb +295 -20
  41. data/lib/saboteur/engine/test_guess.rb +315 -25
  42. data/lib/saboteur/engine/unchanged_syntax.rb +17 -2
  43. data/lib/saboteur/engine.rb +100 -127
  44. data/lib/saboteur/execution/external_runner.rb +471 -0
  45. data/lib/saboteur/execution/marker_sweep.rb +79 -0
  46. data/lib/saboteur/execution/peak_memory.rb +28 -0
  47. data/lib/saboteur/execution/process_table.rb +12 -0
  48. data/lib/saboteur/execution/process_tree.rb +65 -5
  49. data/lib/saboteur/execution/reevaluator.rb +68 -12
  50. data/lib/saboteur/execution/runner.rb +46 -7
  51. data/lib/saboteur/execution/sandbox.rb +15 -2
  52. data/lib/saboteur/execution/seatbelt.rb +226 -0
  53. data/lib/saboteur/execution/supervisor.rb +27 -10
  54. data/lib/saboteur/execution/worker_pool.rb +23 -1
  55. data/lib/saboteur/frameworks/sqlite_memory_database.rb +49 -0
  56. data/lib/saboteur/gates.rb +43 -2
  57. data/lib/saboteur/help.rb +60 -9
  58. data/lib/saboteur/ignore_list.rb +29 -6
  59. data/lib/saboteur/integration/rspec.rb +135 -5
  60. data/lib/saboteur/mutation_source/argument_removal.rb +1 -1
  61. data/lib/saboteur/mutation_source/builtin.rb +26 -5
  62. data/lib/saboteur/mutation_source/chain_removal.rb +1 -1
  63. data/lib/saboteur/mutation_source/condition_negation.rb +1 -1
  64. data/lib/saboteur/mutation_source/constant_literals.rb +66 -0
  65. data/lib/saboteur/mutation_source/element_removal.rb +73 -0
  66. data/lib/saboteur/mutation_source/endpoint_replacement.rb +1 -1
  67. data/lib/saboteur/mutation_source/entry_edits.rb +44 -0
  68. data/lib/saboteur/mutation_source/filter_replacement.rb +1 -1
  69. data/lib/saboteur/mutation_source/hazard_guard.rb +108 -0
  70. data/lib/saboteur/mutation_source/literal_replacement.rb +1 -1
  71. data/lib/saboteur/mutation_source/method_swap.rb +148 -0
  72. data/lib/saboteur/mutation_source/operator_assignment.rb +1 -1
  73. data/lib/saboteur/mutation_source/operator_examples.rb +3 -1
  74. data/lib/saboteur/mutation_source/operator_registry.rb +4 -2
  75. data/lib/saboteur/mutation_source/operator_visit.rb +1 -1
  76. data/lib/saboteur/mutation_source/pair_removal.rb +13 -31
  77. data/lib/saboteur/mutation_source/range_replacement.rb +1 -1
  78. data/lib/saboteur/mutation_source/regex_replacement.rb +93 -0
  79. data/lib/saboteur/preview.rb +120 -12
  80. data/lib/saboteur/refinement_activations.rb +60 -0
  81. data/lib/saboteur/reporter/agent.rb +9 -1
  82. data/lib/saboteur/reporter/agent_commands.rb +10 -3
  83. data/lib/saboteur/reporter/agent_incomplete.rb +11 -0
  84. data/lib/saboteur/reporter/agent_summary.rb +4 -3
  85. data/lib/saboteur/reporter/findings.rb +5 -2
  86. data/lib/saboteur/reporter/history_policy.rb +11 -0
  87. data/lib/saboteur/reporter/history_reader.rb +47 -0
  88. data/lib/saboteur/reporter/html.rb +26 -4
  89. data/lib/saboteur/reporter/mutation_text.rb +6 -0
  90. data/lib/saboteur/reporter/stryker.rb +27 -3
  91. data/lib/saboteur/reporter/survivor_classification.rb +101 -0
  92. data/lib/saboteur/reporter/survivor_list.rb +15 -9
  93. data/lib/saboteur/reporter/terminal.rb +40 -7
  94. data/lib/saboteur/result.rb +19 -3
  95. data/lib/saboteur/selection/coverage_map.rb +105 -10
  96. data/lib/saboteur/selection/dependency_shield.rb +80 -0
  97. data/lib/saboteur/selection/external_test_map.rb +27 -0
  98. data/lib/saboteur/selection/git_diff.rb +93 -11
  99. data/lib/saboteur/selection/layout_only_lines.rb +80 -0
  100. data/lib/saboteur/selection/recorder.rb +22 -4
  101. data/lib/saboteur/selection/source_scope.rb +22 -0
  102. data/lib/saboteur/subject.rb +42 -7
  103. data/lib/saboteur/subject_finder.rb +173 -29
  104. data/lib/saboteur/version.rb +1 -1
  105. data/lib/saboteur/working_tree.rb +38 -15
  106. data/lib/saboteur.rb +13 -0
  107. metadata +35 -2
data/lib/saboteur/cli.rb CHANGED
@@ -11,17 +11,27 @@ require_relative "baseline/command"
11
11
  require_relative "cli/exit_status"
12
12
  require_relative "preview"
13
13
  require_relative "cli/run_reporting"
14
+ require_relative "cli/survivors_command"
15
+ require_relative "cli/source_set"
14
16
  require_relative "cli/run_budget_options"
17
+ require_relative "cli/shard_options"
18
+ require_relative "cli/selection_options"
15
19
  require_relative "cli/presets"
16
20
  require_relative "cli/help_options"
21
+ require_relative "cli/history_command"
17
22
 
18
23
  module Saboteur
19
24
  class CLI
20
25
  include Baseline::Command
21
26
  include ExitStatus
22
27
  include RunReporting
28
+ include SurvivorsCommand
29
+ include SourceSet
23
30
  include RunBudgetOptions
31
+ include ShardOptions
32
+ include SelectionOptions
24
33
  include Presets
34
+ include HistoryCommand
25
35
  include HelpOptions
26
36
 
27
37
  USAGE = "Usage: saboteur run [SOURCE...] [--test TEST...] [--all] [--threshold PERCENT] [--since REF] " \
@@ -58,11 +68,13 @@ module Saboteur
58
68
  case command
59
69
  when "run" then run_command
60
70
  when "baseline" then baseline
61
- when "doctor" then help_requested? ? show_help("doctor") : doctor
71
+ when "survivors" then survivors_command
72
+ when "doctor" then doctor_command
73
+ when "badge", "trend" then history_entry(command)
62
74
  when "version", "--version" then @out.puts(VERSION) || EXIT_OK
63
75
  when "help", "--help", "-h" then show_help(@argv.shift)
64
76
  when "agent-skill" then @out.print(AgentSkill::TEXT) || EXIT_OK
65
- else PRESETS.key?(command) ? preset(command) : usage_error
77
+ else preset_or_usage_error(command)
66
78
  end
67
79
  rescue OptionParser::ParseError => e
68
80
  usage_error(e.message)
@@ -98,13 +110,13 @@ module Saboteur
98
110
 
99
111
  private
100
112
 
101
- def run_command
102
- help_requested? ? show_help("run") : run
103
- end
113
+ def run_command = help_requested? ? show_help("run") : run
104
114
 
105
- def help_requested?
106
- @argv.intersect?(%w[--help -h])
107
- end
115
+ def doctor_command = help_requested? ? show_help("doctor") : doctor
116
+
117
+ def history_entry(command) = help_requested? ? show_help(command) : history_command(command)
118
+
119
+ def help_requested? = @argv.intersect?(%w[--help -h])
108
120
 
109
121
  def show_help(command_name)
110
122
  help = Help.new(option_lines: option_lines)
@@ -121,9 +133,7 @@ module Saboteur
121
133
  agent_failure(message || USAGE, EXIT_USAGE)
122
134
  end
123
135
 
124
- def agent?
125
- @format == "agent"
126
- end
136
+ def agent? = @format == "agent"
127
137
 
128
138
  def agent_requested? = self.class.agent_requested?(@argv)
129
139
 
@@ -144,6 +154,8 @@ module Saboteur
144
154
 
145
155
  return preview if options[:list_operators] || options[:dry_run]
146
156
 
157
+ refuse_empty_source_set
158
+ refuse_empty_since_range
147
159
  execute_run
148
160
  rescue ConfigError => e
149
161
  @err.puts("saboteur: #{e.message}")
@@ -155,8 +167,12 @@ module Saboteur
155
167
 
156
168
  # Mutants run your code, and a mutant of code that writes or deletes files
157
169
  # can change the tree. Say so up front when uncommitted work is at stake.
170
+ # The scope the warning names is the configured one: positional source
171
+ # operands narrow the run after the warning, so the include/exclude they
172
+ # imply stay out of its config (#309, #503 review).
158
173
  def warn_dirty_tree
159
- WorkingTree.new(config.root, ignored: config.output_paths).state.warning&.then { |text| @err.puts(text) }
174
+ warning = @sources ? Config.load(path: options[:config], overrides: overrides.except(:include, :exclude)) : config
175
+ DirtyTreeWarning.for(warning, config_path: options[:config])&.then { |text| @err.puts(text) }
160
176
  end
161
177
 
162
178
  # Contradictory selection flags are a usage error before any file or
@@ -165,30 +181,46 @@ module Saboteur
165
181
  # --test wins there, so generated commands stay valid over any config.
166
182
  def reject_test_selection_conflicts
167
183
  raise UsageError, "--all cannot be combined with --test" if options[:all] && options[:tests]&.any?
184
+
185
+ reject_test_command_conflicts
168
186
  raise UsageError, "--all cannot be combined with --guess-tests" if options[:all] && options[:guess_tests]
169
187
  return unless options[:guess_tests] && options[:tests]&.any?
170
188
 
171
189
  raise UsageError, "--guess-tests cannot be combined with --test"
172
190
  end
173
191
 
192
+ # --test-command replaces the whole in-process test machinery: test
193
+ # selection, coverage, parallelism and the sandbox do not apply.
194
+ def reject_test_command_conflicts
195
+ return unless options[:test_command]
196
+
197
+ # options[:tests] is nil or a non-empty list, so truthiness is the check.
198
+ raise UsageError, "--test-command cannot be combined with --test" if options[:tests]
199
+ raise UsageError, "--test-command cannot be combined with --incremental" if options[:incremental]
200
+ end
201
+
174
202
  # Usage errors found before the target boots.
175
203
  def validate_run_options
176
- reject_threshold_with_agent_format
204
+ reject_format_conflicts
205
+ validate_shard_options
177
206
  Selection::GitDiff.verify_ref!(config["since"], root: config.root) if config["since"]
178
207
  end
179
208
 
180
- # Agent mode has one strict meaning for exit 0, so a score threshold cannot also decide it.
181
- def reject_threshold_with_agent_format
182
- return unless agent? && (config["threshold"] || gates.any?)
209
+ # Agent mode allows no score gates (one strict exit 0); the json inventory is --dry-run planning only.
210
+ def reject_format_conflicts
211
+ if agent? && (config["threshold"] || gates.any?)
212
+ raise UsageError, "#{config['threshold'] ? '--threshold' : gates.agent_option_name} " \
213
+ "cannot be combined with --agent"
214
+ end
215
+ return unless @format == "json" && !options[:dry_run]
183
216
 
184
- raise UsageError, "#{config['threshold'] ? '--threshold' : '--max-* gates'} cannot be combined with --agent"
217
+ raise UsageError, "--format json is the --dry-run mutant inventory; a run summary is --format agent"
185
218
  end
186
219
 
187
220
  # One line naming the reports written, paths relative to the root when
188
221
  # they sit under it (a blank line before it, as the old JSON line had).
189
222
  def announce_reports(reports)
190
- written = reports.select { |path, _label| path }
191
- return if written.empty?
223
+ return if (written = reports.select { |path, _label| path }).empty?
192
224
 
193
225
  root = File.join(File.realpath(config.root), "")
194
226
  names = written.map do |path, label|
@@ -233,10 +265,14 @@ module Saboteur
233
265
  subjects: Array(config["subjects"]), tests: Array(engine.guessed_tests || config["tests"]),
234
266
  sources: @sources || [],
235
267
  config_path: options[:config], digest: options[:id], cache: config["cache"],
236
- incremental: config["incremental"], settings: agent_settings
268
+ incremental: config["incremental"], settings: agent_settings, typed_settings: typed_settings
237
269
  )
238
270
  end
239
271
 
272
+ # The settings keys typed on the command line: a typed empty operator
273
+ # list may clear a config value, so the generated commands keep it.
274
+ def typed_settings = options.keys & Reporter::Agent::Commands::SETTING_FLAGS.keys
275
+
240
276
  # A whole-suite run records its scope as --all, so the generated rerun
241
277
  # and replay commands do not default back to inference (which can exit
242
278
  # 2 in a project with a spec-less source file).
@@ -271,17 +307,15 @@ module Saboteur
271
307
  # replace the configured include and exclude patterns. Each must exist,
272
308
  # so a typo is an error and not a run over zero subjects.
273
309
  def use_explicit_sources
274
- if @argv.empty?
275
- # --test may stand alone: the configured include (or the --since
276
- # diff) names the sources. The files must still exist.
277
- options[:tests]&.each { |test| require_existing_test(test) }
278
- return
279
- end
310
+ Selection::SourceScope.verify_no_specs!(@argv)
311
+ options[:tests]&.each { |test| require_existing_test(test) }
312
+ # --test may stand alone: the configured include (or the --since
313
+ # diff) names the sources. The files must still exist.
314
+ return if @argv.empty?
280
315
 
281
316
  @sources = @argv.dup
282
317
  options[:include] = @argv.map { |source| source_pattern(source) }
283
318
  options[:exclude] = []
284
- options[:tests]&.each { |test| require_existing_test(test) }
285
319
  end
286
320
 
287
321
  def source_pattern(source)
@@ -303,12 +337,10 @@ module Saboteur
303
337
  raise UsageError, "test file not found: #{test}" unless File.exist?(path)
304
338
  end
305
339
 
306
- def overrides
307
- options.except(:config, :id, :suite, :list_operators, :dry_run)
308
- end
340
+ def overrides = options.except(:config, :id, :suite, :list_operators, :dry_run)
309
341
 
310
342
  def preview
311
- raise UsageError, "--dry-run and --list-operators print text, not --agent JSON" if agent?
343
+ raise UsageError, "--dry-run and --list-operators print text, not --agent JSON; use --format json" if agent?
312
344
  raise UsageError, "--id replays one mutant: not with --dry-run or --list-operators" if options[:id]
313
345
 
314
346
  if options[:list_operators]
@@ -316,24 +348,10 @@ module Saboteur
316
348
  return EXIT_OK
317
349
  end
318
350
 
319
- Preview.new(config, out: @out, err: @err).call
351
+ Preview.new(config, out: @out, err: @err, format: @format).call
320
352
  EXIT_OK
321
353
  end
322
354
 
323
- def define_selection_options(opts)
324
- opts.on("--operators NAMES", Array, "Choose operator names (default: all)") do |v|
325
- set(:operators, v)
326
- end
327
- opts.on("--exclude-operators NAMES", Array, "Exclude operator names from the selected set") do |v|
328
- set(:exclude_operators, v)
329
- end
330
- opts.on("--list-operators", "List operator names and descriptions") { set(:list_operators, true) }
331
- opts.on("--dry-run", "Preview mutants without boot, tests, forks or report writes") { set(:dry_run, true) }
332
- opts.on("--id DIGEST", "Replay exactly one current mutant by its full digest") { |v| set(:id, v) }
333
- opts.on("--since REF", "Only mutate subjects changed since git REF") { |v| options[:since] = v }
334
- opts.on("--subject EXPR", "Klass#meth, Klass.meth or a namespace; repeatable") { |v| add_subject(v) }
335
- end
336
-
337
355
  def define_execution_options(opts)
338
356
  opts.on("--timeout-factor N", Float, "Multiple of baseline runtime (default 3)") { |v| set(:timeout_factor, v) }
339
357
  opts.on("--jobs N", Integer, "Concurrent workers (default 1)") { |v| set(:jobs, v) }
@@ -352,6 +370,10 @@ module Saboteur
352
370
  end
353
371
  opts.on("--all", "Run the whole configured test suite instead of inferring the specs named after the " \
354
372
  "mutated files") { set(:all, true) }
373
+ opts.on("--test-command CMD", "Run CMD per mutant in the project root; exit 0 means the mutant survived. " \
374
+ "For apps that cannot load in this Ruby; much slower, serial, no coverage") do |v|
375
+ set(:test_command, v)
376
+ end
355
377
  opts.on("--[no-]guess-tests", "Deprecated alias: inferred specs are already the default; " \
356
378
  "--no-guess-tests warns and is ignored - use --all for the whole suite") do |v|
357
379
  set(:guess_tests, v)
@@ -368,8 +390,7 @@ module Saboteur
368
390
  opts.on("--score-history PATH", "Append score history JSONL (opt-in)") { |v| set(:score_history, v) }
369
391
  opts.on("--json PATH", "Stryker JSON report path") { |v| set(:json_report, v) }
370
392
  opts.on("--html PATH", "Self-contained HTML report path") { |v| set(:html_report, v) }
371
- opts.on("--format FORMAT", %w[text agent], "text (default) or agent: one bounded JSON summary on stdout, " \
372
- "strict exit codes") { |v| @format = v }
393
+ opts.on("--format FORMAT", %w[text agent json], "text, agent or json (--dry-run inventory)") { |v| @format = v }
373
394
  opts.on("--agent", "Same as --format agent") { @format = "agent" }
374
395
  opts.on("--verbose", "Text report: also print score split, timings and kill causes;",
375
396
  "a crash prints its backtrace") { @verbose = true }
@@ -388,12 +409,8 @@ module Saboteur
388
409
  Gates.define_options(opts) { |key, value| set(key, value) }
389
410
  end
390
411
 
391
- def add_subject(expr)
392
- (options[:subjects] ||= []) << expr
393
- end
412
+ def add_subject(expr) = (options[:subjects] ||= []) << expr
394
413
 
395
- def set(key, value)
396
- options[key] = value
397
- end
414
+ def set(key, value) = (options[key] = value)
398
415
  end
399
416
  end
@@ -22,13 +22,27 @@ module Saboteur
22
22
  "integration" => "rspec",
23
23
  "require" => [],
24
24
  "include" => ["lib/**/*.rb"],
25
- "exclude" => [],
25
+ # Spec and test files vendored under lib/ (capybara ships its driver
26
+ # suite at lib/capybara/spec/**) are never mutation subjects (#504).
27
+ # A config's own c.exclude replaces this list wholesale.
28
+ "exclude" => ["lib/**/spec/**/*", "lib/**/test/**/*", "lib/**/tests/**/*",
29
+ "lib/**/*_spec.rb", "lib/**/*_test.rb", "lib/**/test_*.rb"],
26
30
  "arid" => MutationSource::Builtin::DEFAULT_ARID,
27
31
  "whole_file" => [],
32
+ # [{ "pattern" => "app/payments/**/*", "min_score" => 95.0 }], from
33
+ # repeatable c.score_gate calls (#473). --threshold stays the global
34
+ # fallback.
35
+ "score_gates" => [],
28
36
  "hazard" => MutationSource::HazardGuard::DEFAULT_HAZARD,
29
37
  "allow_hazardous" => false,
30
- "sandbox" => true, # true, false, or "strict": strict refuses to run without full Landlock
38
+ # true, false, "strict" or "seatbelt": strict refuses to run without full Landlock;
39
+ # seatbelt opts into the macOS Seatbelt sandbox (on Linux it behaves like true).
40
+ "sandbox" => true,
31
41
  "sandbox_writable" => [],
42
+ # off, survivors, or decisive (#476): one reordered confirmation run
43
+ # per new survivor (and per fresh kill in decisive); a disagreement
44
+ # becomes unverified, outside the score.
45
+ "flaky_check" => "off",
32
46
  "timeout_factor" => 3.0,
33
47
  "timeout_min" => 5.0,
34
48
  "baseline_timeout" => 300.0,
@@ -56,6 +70,7 @@ module Saboteur
56
70
  "since" => nil,
57
71
  "cache" => true,
58
72
  "incremental" => false,
73
+ "shard" => nil,
59
74
  "allow_focus" => false,
60
75
  "fail_fast" => false,
61
76
  "guess_tests" => false,
@@ -67,6 +82,7 @@ module Saboteur
67
82
  "max_duration" => nil,
68
83
  "max_duration_ok" => false,
69
84
  "tests" => [],
85
+ "test_command" => nil,
70
86
  "fingerprint_extra" => [],
71
87
  "threshold" => nil,
72
88
  "max_timeouts" => nil,
@@ -160,14 +176,30 @@ module Saboteur
160
176
  matched = self["include"].flat_map { |pattern| Dir.glob(pattern, base: root) }
161
177
  excluded = self["exclude"].flat_map { |pattern| Dir.glob(pattern, base: root) }
162
178
  (matched - excluded).select { |file| file.end_with?(".rb") }
163
- .map { |file| File.realpath(File.expand_path(Text.utf8_identity(file), root)) }
179
+ .filter_map { |file| real_source_path(file) }
164
180
  .uniq.sort
165
181
  end
166
182
 
183
+ # A broken symlink matches Dir.glob but has no real path: dangling
184
+ # (ENOENT), a loop (ELOOP) or through a regular file (ENOTDIR). It is
185
+ # not mutation source, so it is skipped rather than crashing the run
186
+ # (#503). A regular file that cannot be resolved still raises.
187
+ def real_source_path(file)
188
+ path = File.expand_path(Text.utf8_identity(file), root)
189
+ File.realpath(path)
190
+ rescue SystemCallError
191
+ raise unless File.symlink?(path)
192
+ end
193
+
167
194
  # Patterns that match no file. A pattern goes stale when files move;
168
195
  # an exclude that silently stops excluding is a hazard, so say so.
169
196
  def unmatched_patterns
170
197
  %w[include exclude].flat_map do |key|
198
+ # The default exclude shields (#504) match nothing on a healthy
199
+ # project - that is their job - so only a configured exclude can go
200
+ # stale enough to warn about.
201
+ next [] if key == "exclude" && !explicit?("exclude")
202
+
171
203
  self[key].select { |pattern| Dir.glob(pattern, base: root).empty? }.map { |pattern| "#{key}: #{pattern}" }
172
204
  end
173
205
  end
@@ -237,6 +269,41 @@ module Saboteur
237
269
  validate_matrix!
238
270
  validate_test_selection!
239
271
  validate_gates!
272
+ validate_score_gates!
273
+ end
274
+
275
+ # Per-path gates (#473): normalize each entry to string pattern /
276
+ # float floor, and fail loud on a pattern no included file matches -
277
+ # a gate that matches nothing would pass vacuously forever.
278
+ def validate_score_gates!
279
+ gates = self["score_gates"]
280
+ return if gates.empty?
281
+
282
+ normalized = gates.map { |gate| normalize_score_gate(gate) }
283
+ prefix = File.join(File.realpath(root), "")
284
+ relative = included_files.map { |file| file.delete_prefix(prefix) }
285
+ flags = File::FNM_PATHNAME | File::FNM_EXTGLOB
286
+ normalized.each do |gate|
287
+ next if relative.any? { |file| File.fnmatch?(gate["pattern"], file, flags) }
288
+
289
+ raise ConfigError, "score_gate #{gate['pattern']} matches no included file"
290
+ end
291
+ @values["score_gates"] = normalized
292
+ end
293
+
294
+ def normalize_score_gate(gate)
295
+ pattern = gate["pattern"].to_s
296
+ begin
297
+ floor = Float(gate["min_score"])
298
+ rescue ArgumentError, TypeError
299
+ raise ConfigError, "score_gate min_score must be between 0 and 100, got #{gate['min_score'].inspect}"
300
+ end
301
+ unless floor.finite? && floor.between?(0, 100)
302
+ raise ConfigError, "score_gate min_score must be between 0 and 100, got #{gate['min_score'].inspect}"
303
+ end
304
+ raise ConfigError, "score_gate pattern must not be empty" if pattern.empty?
305
+
306
+ { "pattern" => pattern, "min_score" => floor }
240
307
  end
241
308
 
242
309
  def validate_gates!
@@ -270,6 +337,28 @@ module Saboteur
270
337
  raise ConfigError, "test_selection must be one of: auto, all"
271
338
  end
272
339
 
340
+ # --test-command replaces the whole in-process machinery, on the
341
+ # EFFECTIVE config: CLI flags and config-file entries alike (#305).
342
+ # The command runs unsandboxed, so "strict" cannot be honored.
343
+ def validate_test_command!
344
+ command = self["test_command"]
345
+ # nil alone means unset: false, arrays and numbers are all rejected,
346
+ # never coerced and never sent to String methods (#305 QA).
347
+ raise ConfigError, "test_command must be a string" unless command.nil? || command.is_a?(String)
348
+ raise ConfigError, "test_command must not be blank" if command&.strip&.empty?
349
+
350
+ return if command.to_s.empty?
351
+ raise ConfigError, "test_command cannot be combined with tests" unless self["tests"].empty?
352
+ raise ConfigError, "test_command cannot be combined with incremental" if self["incremental"]
353
+ unless self["flaky_check"] == "off"
354
+ raise ConfigError, "test_command cannot be combined with flaky_check: an external run has no " \
355
+ "in-process scheduler to rerun the decisive tests through"
356
+ end
357
+ return unless self["sandbox"] == "strict"
358
+
359
+ raise ConfigError, 'test_command runs the command unsandboxed, so sandbox "strict" cannot be honored'
360
+ end
361
+
273
362
  def validate_reeval!
274
363
  raise ConfigError, "reeval must be one of: #{Execution::Reevaluator::STRATEGIES.join(', ')}" unless
275
364
  Execution::Reevaluator::STRATEGIES.include?(self["reeval"].to_s)
@@ -277,8 +366,11 @@ module Saboteur
277
366
 
278
367
  def validate!
279
368
  raise ConfigError, "jobs must be >= 1" if self["jobs"] < 1
280
- unless [true, false, "strict"].include?(self["sandbox"])
281
- raise ConfigError, 'sandbox must be true, false, or "strict"'
369
+ unless [true, false, "strict", "seatbelt"].include?(self["sandbox"])
370
+ raise ConfigError, 'sandbox must be true, false, "strict", or "seatbelt"'
371
+ end
372
+ unless %w[off survivors decisive].include?(self["flaky_check"])
373
+ raise ConfigError, "flaky_check must be one of: off, survivors, decisive"
282
374
  end
283
375
 
284
376
  validate_mutant_limit!
@@ -292,6 +384,7 @@ module Saboteur
292
384
 
293
385
  validate_reeval!
294
386
  validate_operator_config!
387
+ validate_test_command!
295
388
  end
296
389
 
297
390
  public
@@ -26,6 +26,14 @@ module Saboteur
26
26
  end
27
27
  end
28
28
 
29
+ # Repeatable per-path score gates (#473), the one setting that takes
30
+ # arguments: c.score_gate "app/payments/**/*", min_score: 95. Values
31
+ # are validated later, on the effective Config.
32
+ def score_gate(pattern, min_score:)
33
+ (@values["score_gates"] ||= []) << { "pattern" => pattern, "min_score" => min_score }
34
+ @lines["score_gates"] = caller_locations(1, 1).first.lineno
35
+ end
36
+
29
37
  Config::INTEGER_KEYS.each do |key|
30
38
  define_method("#{key}=") do |value|
31
39
  unless value.is_a?(Integer)
@@ -0,0 +1,182 @@
1
+ # SPDX-License-Identifier: MIT
2
+ # frozen_string_literal: true
3
+
4
+ require "prism"
5
+ require_relative "text"
6
+
7
+ module Saboteur
8
+ # The lines where a constant is read, per file: the tests covering those
9
+ # lines are the ones that can observe a change to the constant's value
10
+ # (issue #306).
11
+ #
12
+ # Matching is lexical and honest about its edges. A bare `RULES` matches
13
+ # when its scope is the constant's owner or nested under it (constant
14
+ # lookup climbs lexical scopes) or the constant is top-level; a
15
+ # `Shop::RULES` path matches by its full spelling. A scope that writes its
16
+ # own `RULES` shadows the outer constant: reads under it resolve to the
17
+ # inner one and do not count. Reads reached through a superclass,
18
+ # `const_get`, or a compact `class Shop::Item` scope are not resolved: a
19
+ # constant only read that way finds no consumer here, and the run reports
20
+ # it as ran-at-load instead of guessing.
21
+ #
22
+ # Runtime reads (inside a def body) observe the constant when the mutant
23
+ # runs; load-time reads outside the constant's own file do not: the other
24
+ # file was loaded once with the original value and re-evaluating the
25
+ # constant's file does not run it again, so a read like
26
+ # `COPY = A::RULES` there keeps the original (#319). A capture inside a
27
+ # boot-time block (`1.times { COPY = A::RULES }`) is treated the same:
28
+ # statement-level blocks run while the file loads. That is a heuristic -
29
+ # a block passed to a deferring call (at_exit, define_method) runs later,
30
+ # but treating it as load-time errs toward dropping a constant rather
31
+ # than reporting a mutant nobody can observe.
32
+ module ConstantReads
33
+ Reads = Data.define(:runtime, :load, :defs)
34
+
35
+ module_function
36
+
37
+ # { "/abs/file.rb" => Reads } for the files that read the constant.
38
+ # runtime: def-body and block lines - a block runs when its caller runs
39
+ # it (an `it` block during the example), and line coverage decides
40
+ # whether any test reached it. load: top-level or class-body statement
41
+ # lines (observable only in the constant's own file, which the mutant
42
+ # re-evaluates). defs: the start line of each def a runtime read sits
43
+ # in, for the method-call counter fallback (endless defs have no line
44
+ # coverage). Only files whose text mentions the name are parsed.
45
+ def across(files, constant)
46
+ mention = /\b#{Regexp.escape(constant.name.to_s)}\b/
47
+ files.each_with_object({}) do |file, hits|
48
+ source = File.binread(file)
49
+ next unless mentions?(source, mention)
50
+
51
+ reads = reads_in(source, constant)
52
+ hits[file] = reads unless reads.runtime.empty? && reads.load.empty?
53
+ end
54
+ end
55
+
56
+ # The lines a mutant of the constant can be observed through: runtime
57
+ # reads everywhere, load reads in the constant's own file only.
58
+ def observable_lines(reads_by_file, constant)
59
+ reads_by_file.each_with_object({}) do |(file, reads), hits|
60
+ lines = reads.runtime.dup
61
+ lines.concat(reads.load) if file == constant.file
62
+ hits[file] = lines.uniq.sort unless lines.empty?
63
+ end
64
+ end
65
+
66
+ def mentions?(source, mention)
67
+ Text.utf8_identity(source).match?(mention)
68
+ rescue Encoding::CompatibilityError, ArgumentError
69
+ false
70
+ end
71
+
72
+ def reads_in(source, constant)
73
+ result = Prism.parse(source)
74
+ return Reads.new([], [], []) if result.failure?
75
+
76
+ out = { runtime: [], load: [], defs: [] }
77
+ walk(result.value, [], constant, out, in_def: nil, in_block: false, shadowed: nil)
78
+ Reads.new(out[:runtime].uniq.sort, out[:load].uniq.sort, out[:defs].uniq.sort)
79
+ end
80
+
81
+ def walk(node, nesting, constant, out, in_def:, in_block:, shadowed:, capture: false)
82
+ return if node.nil?
83
+
84
+ case node
85
+ when Prism::DefNode
86
+ walk(node.body, nesting, constant, out, in_def: node.location.start_line, in_block: false,
87
+ shadowed: shadowed)
88
+ when Prism::BlockNode, Prism::LambdaNode
89
+ walk(node.body, nesting, constant, out, in_def: in_def, in_block: true, capture: capture,
90
+ shadowed: shadowed)
91
+ when Prism::ConstantWriteNode, Prism::ConstantPathWriteNode
92
+ # A capture (`COPY = <expr>`): the value reads the constant once,
93
+ # when the write runs. Walk the value flagged as a capture so a
94
+ # boot-time block write (`1.times { COPY = A::RULES }`) lands in
95
+ # load, not runtime (#319).
96
+ walk(node.value, nesting, constant, out, in_def: in_def, in_block: in_block, capture: true,
97
+ shadowed: shadowed)
98
+ when Prism::ConstantReadNode
99
+ record(node, out, in_def, in_block, capture) if !shadowed && bare_match?(node, nesting, constant)
100
+ when Prism::ConstantPathNode
101
+ # `Shop::RULES` holds no ConstantReadNode for RULES, only for Shop;
102
+ # match the path's full spelling.
103
+ record(node, out, in_def, in_block, capture) if node.slice.delete_prefix("::") == constant.id
104
+ walk_children(node, nesting, constant, out, in_def: in_def, in_block: in_block, capture: capture,
105
+ shadowed: shadowed)
106
+ when Prism::ModuleNode, Prism::ClassNode, Prism::SingletonClassNode
107
+ walk_scope(node, nesting, constant, out, shadowed)
108
+ else
109
+ walk_children(node, nesting, constant, out, in_def: in_def, in_block: in_block, capture: capture,
110
+ shadowed: shadowed)
111
+ end
112
+ end
113
+
114
+ # `class << self` joins the nesting as "<attached>.singleton_class",
115
+ # matching how ConstantSubjectFinder names that owner (#380); a non-self
116
+ # receiver or a top-level singleton has no static name and adds nothing.
117
+ # The shadow carries the path of the scope that set it. Entering the
118
+ # constant's owner scope lifts a shadow set outside it: the owner's
119
+ # own definition is lexically closer there (#395). A shadow set
120
+ # inside the owner subtree still binds below the nested scope that
121
+ # wrote the constant.
122
+ def walk_scope(node, nesting, constant, out, shadowed)
123
+ inner = scope_nesting(node, nesting)
124
+ path = inner.join("::")
125
+ scopes = [constant.owner.to_s]
126
+ shadowed = nil if shadowed && scopes.any? { |scope| path == scope || path.start_with?("#{scope}::") } &&
127
+ scopes.none? { |scope| shadowed == scope || shadowed.start_with?("#{scope}::") }
128
+ shadowed = path if shadowed.nil? && path != scopes.first && writes_constant?(node.body, constant.name)
129
+ walk(node.body, inner, constant, out, in_def: nil, in_block: false, shadowed: shadowed)
130
+ end
131
+
132
+ def scope_nesting(node, nesting)
133
+ return nesting + [node.constant_path.slice.delete_prefix("::")] unless node.is_a?(Prism::SingletonClassNode)
134
+ return nesting unless node.expression.is_a?(Prism::SelfNode) && nesting.any?
135
+
136
+ nesting[0...-1] + ["#{nesting.last}.singleton_class"]
137
+ end
138
+
139
+ def walk_children(node, nesting, constant, out, in_def:, in_block:, capture:, shadowed:)
140
+ node.compact_child_nodes.each do |child|
141
+ walk(child, nesting, constant, out, in_def: in_def, in_block: in_block, capture: capture,
142
+ shadowed: shadowed)
143
+ end
144
+ end
145
+
146
+ def record(node, out, in_def, in_block, capture)
147
+ line = node.location.start_line
148
+ if in_def
149
+ out[:runtime] << line
150
+ out[:defs] << in_def
151
+ elsif in_block && !capture
152
+ out[:runtime] << line
153
+ else
154
+ # A bare statement read, or a capture in a boot-time block: both
155
+ # resolved while the file loaded, with the original value.
156
+ out[:load] << line
157
+ end
158
+ end
159
+
160
+ # A bare `RULES` reads this constant when the read's scope is the
161
+ # constant's owner (or its singleton scope, whose ancestry reaches the
162
+ # attached class) or nested under either, or it is top-level (#380).
163
+ def bare_match?(node, nesting, constant)
164
+ return false unless node.name == constant.name
165
+ return true if constant.owner.nil?
166
+
167
+ path = nesting.join("::")
168
+ [constant.owner, "#{constant.owner}.singleton_class"].any? do |scope|
169
+ path == scope || path.start_with?("#{scope}::")
170
+ end
171
+ end
172
+
173
+ # True when the scope body assigns the constant name at its own level
174
+ # (not inside a nested def or scope).
175
+ def writes_constant?(body, name)
176
+ statements = body&.body || []
177
+ statements.any? do |statement|
178
+ statement.is_a?(Prism::ConstantWriteNode) && statement.name == name
179
+ end
180
+ end
181
+ end
182
+ end