saboteur 0.6.0 → 0.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +1024 -0
- data/README.md +199 -952
- data/exe/saboteur +34 -7
- data/lib/saboteur/baseline/workflow.rb +2 -1
- data/lib/saboteur/baseline_health.rb +40 -6
- data/lib/saboteur/cli/exit_status.rb +80 -4
- data/lib/saboteur/cli/help_options.rb +17 -0
- data/lib/saboteur/cli/history_command.rb +135 -0
- data/lib/saboteur/cli/presets.rb +40 -0
- data/lib/saboteur/cli/run_budget_options.rb +37 -0
- data/lib/saboteur/cli/run_reporting.rb +74 -5
- data/lib/saboteur/cli/selection_options.rb +31 -0
- data/lib/saboteur/cli/shard_options.rb +17 -0
- data/lib/saboteur/cli/source_set.rb +53 -0
- data/lib/saboteur/cli/survivors_command.rb +157 -0
- data/lib/saboteur/cli.rb +136 -98
- data/lib/saboteur/config.rb +163 -15
- data/lib/saboteur/config_file.rb +8 -0
- data/lib/saboteur/constant_reads.rb +182 -0
- data/lib/saboteur/constant_subject.rb +70 -0
- data/lib/saboteur/constant_subject_finder.rb +281 -0
- data/lib/saboteur/coverage_reads.rb +29 -0
- data/lib/saboteur/crash_report.rb +154 -0
- data/lib/saboteur/custom_map.rb +3 -1
- data/lib/saboteur/dirty_tree_warning.rb +128 -0
- data/lib/saboteur/doctor.rb +49 -1
- data/lib/saboteur/engine/baseline_guard.rb +136 -14
- data/lib/saboteur/engine/boot.rb +16 -1
- data/lib/saboteur/engine/canary.rb +41 -10
- data/lib/saboteur/engine/candidate_selection.rb +28 -1
- data/lib/saboteur/engine/candidates.rb +43 -3
- data/lib/saboteur/engine/coverage_guard.rb +29 -0
- data/lib/saboteur/engine/coverage_store.rb +79 -15
- data/lib/saboteur/engine/defined_order_prefix.rb +43 -0
- data/lib/saboteur/engine/diagnostics.rb +41 -2
- data/lib/saboteur/engine/diff_scope.rb +24 -5
- data/lib/saboteur/engine/distribution.rb +134 -8
- data/lib/saboteur/engine/early_stop_recording.rb +47 -0
- data/lib/saboteur/engine/gated_run.rb +35 -0
- data/lib/saboteur/engine/load_coverage.rb +45 -0
- data/lib/saboteur/engine/out_of_scope_guard.rb +21 -0
- data/lib/saboteur/engine/pipelines.rb +107 -0
- data/lib/saboteur/engine/result_cache.rb +67 -9
- data/lib/saboteur/engine/run_budget.rb +90 -0
- data/lib/saboteur/engine/run_fingerprint.rb +4 -0
- data/lib/saboteur/engine/sandbox_policy.rb +56 -11
- data/lib/saboteur/engine/scheduler.rb +222 -20
- data/lib/saboteur/engine/selection_announcement.rb +54 -0
- data/lib/saboteur/engine/shard.rb +69 -0
- data/lib/saboteur/engine/subject_selection.rb +373 -19
- data/lib/saboteur/engine/suite_files.rb +3 -1
- data/lib/saboteur/engine/test_guess.rb +438 -25
- data/lib/saboteur/engine/thread_warnings.rb +53 -0
- data/lib/saboteur/engine/unchanged_syntax.rb +17 -2
- data/lib/saboteur/engine/unscoped_suite_warning.rb +41 -0
- data/lib/saboteur/engine/verdict_check.rb +54 -18
- data/lib/saboteur/engine.rb +202 -158
- data/lib/saboteur/execution/external_runner.rb +471 -0
- data/lib/saboteur/execution/marker_sweep.rb +79 -0
- data/lib/saboteur/execution/peak_memory.rb +28 -0
- data/lib/saboteur/execution/process_table.rb +12 -0
- data/lib/saboteur/execution/process_tree.rb +65 -5
- data/lib/saboteur/execution/protocol.rb +2 -1
- data/lib/saboteur/execution/reevaluator.rb +68 -12
- data/lib/saboteur/execution/runner.rb +225 -24
- data/lib/saboteur/execution/sandbox.rb +21 -3
- data/lib/saboteur/execution/seatbelt.rb +226 -0
- data/lib/saboteur/execution/supervisor.rb +45 -13
- data/lib/saboteur/execution/worker_pool.rb +110 -8
- data/lib/saboteur/frameworks/sqlite_memory_database.rb +49 -0
- data/lib/saboteur/gates.rb +43 -2
- data/lib/saboteur/help.rb +108 -13
- data/lib/saboteur/ignore_list.rb +40 -7
- data/lib/saboteur/install.rb +4 -3
- data/lib/saboteur/integration/base.rb +7 -2
- data/lib/saboteur/integration/rspec.rb +175 -9
- data/lib/saboteur/mutation_source/argument_removal.rb +1 -1
- data/lib/saboteur/mutation_source/builtin.rb +44 -14
- data/lib/saboteur/mutation_source/chain_removal.rb +1 -1
- data/lib/saboteur/mutation_source/condition_negation.rb +1 -1
- data/lib/saboteur/mutation_source/constant_literals.rb +66 -0
- data/lib/saboteur/mutation_source/element_removal.rb +73 -0
- data/lib/saboteur/mutation_source/endpoint_replacement.rb +1 -1
- data/lib/saboteur/mutation_source/entry_edits.rb +44 -0
- data/lib/saboteur/mutation_source/filter_replacement.rb +1 -1
- data/lib/saboteur/mutation_source/hazard_guard.rb +129 -3
- data/lib/saboteur/mutation_source/interpolation_removal.rb +41 -0
- data/lib/saboteur/mutation_source/literal_replacement.rb +1 -1
- data/lib/saboteur/mutation_source/method_swap.rb +193 -0
- data/lib/saboteur/mutation_source/mutation.rb +5 -3
- data/lib/saboteur/mutation_source/operand_replacement.rb +54 -0
- data/lib/saboteur/mutation_source/operator_assignment.rb +1 -1
- data/lib/saboteur/mutation_source/operator_examples.rb +6 -1
- data/lib/saboteur/mutation_source/operator_registry.rb +7 -2
- data/lib/saboteur/mutation_source/operator_visit.rb +39 -0
- data/lib/saboteur/mutation_source/pair_removal.rb +13 -31
- data/lib/saboteur/mutation_source/range_replacement.rb +1 -1
- data/lib/saboteur/mutation_source/regex_replacement.rb +93 -0
- data/lib/saboteur/preview.rb +122 -13
- data/lib/saboteur/refinement_activations.rb +60 -0
- data/lib/saboteur/reporter/agent.rb +9 -1
- data/lib/saboteur/reporter/agent_commands.rb +47 -15
- data/lib/saboteur/reporter/agent_incomplete.rb +38 -2
- data/lib/saboteur/reporter/agent_summary.rb +4 -3
- data/lib/saboteur/reporter/findings.rb +5 -2
- data/lib/saboteur/reporter/history_policy.rb +11 -0
- data/lib/saboteur/reporter/history_reader.rb +47 -0
- data/lib/saboteur/reporter/html.rb +26 -4
- data/lib/saboteur/reporter/mutation_text.rb +6 -0
- data/lib/saboteur/reporter/next_steps.rb +14 -4
- data/lib/saboteur/reporter/score_history.rb +5 -1
- data/lib/saboteur/reporter/stryker.rb +40 -7
- data/lib/saboteur/reporter/survivor_classification.rb +101 -0
- data/lib/saboteur/reporter/survivor_hints.rb +3 -0
- data/lib/saboteur/reporter/survivor_list.rb +15 -9
- data/lib/saboteur/reporter/terminal.rb +74 -17
- data/lib/saboteur/result.rb +38 -7
- data/lib/saboteur/selection/coverage_map.rb +107 -11
- data/lib/saboteur/selection/dependency_shield.rb +80 -0
- data/lib/saboteur/selection/external_test_map.rb +27 -0
- data/lib/saboteur/selection/git_diff.rb +105 -13
- data/lib/saboteur/selection/layout_only_lines.rb +80 -0
- data/lib/saboteur/selection/recorder.rb +22 -4
- data/lib/saboteur/selection/source_scope.rb +22 -0
- data/lib/saboteur/selection/untracked_changes.rb +126 -0
- data/lib/saboteur/subject.rb +53 -7
- data/lib/saboteur/subject_finder.rb +173 -29
- data/lib/saboteur/text.rb +42 -0
- data/lib/saboteur/version.rb +1 -1
- data/lib/saboteur/working_tree.rb +38 -15
- data/lib/saboteur.rb +29 -1
- metadata +50 -2
data/lib/saboteur/config.rb
CHANGED
|
@@ -1,6 +1,8 @@
|
|
|
1
1
|
# SPDX-License-Identifier: MIT
|
|
2
2
|
# frozen_string_literal: true
|
|
3
3
|
|
|
4
|
+
require_relative "text"
|
|
5
|
+
|
|
4
6
|
require_relative "mutation_source/builtin"
|
|
5
7
|
require_relative "mutation_source/operator_registry"
|
|
6
8
|
require_relative "mutation_source/hazard_guard"
|
|
@@ -20,13 +22,27 @@ module Saboteur
|
|
|
20
22
|
"integration" => "rspec",
|
|
21
23
|
"require" => [],
|
|
22
24
|
"include" => ["lib/**/*.rb"],
|
|
23
|
-
|
|
25
|
+
# Spec and test files vendored under lib/ (capybara ships its driver
|
|
26
|
+
# suite at lib/capybara/spec/**) are never mutation subjects (#504).
|
|
27
|
+
# A config's own c.exclude replaces this list wholesale.
|
|
28
|
+
"exclude" => ["lib/**/spec/**/*", "lib/**/test/**/*", "lib/**/tests/**/*",
|
|
29
|
+
"lib/**/*_spec.rb", "lib/**/*_test.rb", "lib/**/test_*.rb"],
|
|
24
30
|
"arid" => MutationSource::Builtin::DEFAULT_ARID,
|
|
25
31
|
"whole_file" => [],
|
|
32
|
+
# [{ "pattern" => "app/payments/**/*", "min_score" => 95.0 }], from
|
|
33
|
+
# repeatable c.score_gate calls (#473). --threshold stays the global
|
|
34
|
+
# fallback.
|
|
35
|
+
"score_gates" => [],
|
|
26
36
|
"hazard" => MutationSource::HazardGuard::DEFAULT_HAZARD,
|
|
27
37
|
"allow_hazardous" => false,
|
|
28
|
-
|
|
38
|
+
# true, false, "strict" or "seatbelt": strict refuses to run without full Landlock;
|
|
39
|
+
# seatbelt opts into the macOS Seatbelt sandbox (on Linux it behaves like true).
|
|
40
|
+
"sandbox" => true,
|
|
29
41
|
"sandbox_writable" => [],
|
|
42
|
+
# off, survivors, or decisive (#476): one reordered confirmation run
|
|
43
|
+
# per new survivor (and per fresh kill in decisive); a disagreement
|
|
44
|
+
# becomes unverified, outside the score.
|
|
45
|
+
"flaky_check" => "off",
|
|
30
46
|
"timeout_factor" => 3.0,
|
|
31
47
|
"timeout_min" => 5.0,
|
|
32
48
|
"baseline_timeout" => 300.0,
|
|
@@ -54,20 +70,28 @@ module Saboteur
|
|
|
54
70
|
"since" => nil,
|
|
55
71
|
"cache" => true,
|
|
56
72
|
"incremental" => false,
|
|
73
|
+
"shard" => nil,
|
|
57
74
|
"allow_focus" => false,
|
|
58
75
|
"fail_fast" => false,
|
|
59
76
|
"guess_tests" => false,
|
|
77
|
+
"all" => false,
|
|
78
|
+
"test_selection" => "auto",
|
|
60
79
|
"matrix" => false,
|
|
61
80
|
"seed" => 1,
|
|
81
|
+
"baseline_passes" => 2,
|
|
82
|
+
"max_duration" => nil,
|
|
83
|
+
"max_duration_ok" => false,
|
|
62
84
|
"tests" => [],
|
|
85
|
+
"test_command" => nil,
|
|
63
86
|
"fingerprint_extra" => [],
|
|
64
87
|
"threshold" => nil,
|
|
65
88
|
"max_timeouts" => nil,
|
|
66
89
|
"max_errors" => nil
|
|
67
90
|
}.freeze
|
|
68
91
|
|
|
69
|
-
INTEGER_KEYS = %w[jobs max_memory_mb max_processes max_mutants_per_method seed].freeze
|
|
92
|
+
INTEGER_KEYS = %w[jobs max_memory_mb max_processes max_mutants_per_method seed baseline_passes].freeze
|
|
70
93
|
FLOAT_KEYS = %w[timeout_factor timeout_min baseline_timeout coverage_timeout].freeze
|
|
94
|
+
NILABLE_FLOAT_KEYS = %w[threshold max_duration].freeze
|
|
71
95
|
THRESHOLD_RANGE = (0.0..100.0)
|
|
72
96
|
|
|
73
97
|
LIST_KEYS = %w[require include exclude arid hazard sources hooks rspec_args subjects tests
|
|
@@ -96,11 +120,19 @@ module Saboteur
|
|
|
96
120
|
hash.to_h { |key, value| [key.to_s, value] }.compact
|
|
97
121
|
end
|
|
98
122
|
|
|
99
|
-
|
|
100
|
-
|
|
123
|
+
# Deprecations found while loading: printed by the CLI, never fatal.
|
|
124
|
+
attr_reader :deprecation_warnings
|
|
125
|
+
|
|
126
|
+
def initialize(values = {}, root: Dir.pwd, explicit: nil)
|
|
127
|
+
# Retag the root losslessly: under a non-UTF-8 locale it arrives
|
|
128
|
+
# locale-tagged, and every glob, join and expand below would mix
|
|
129
|
+
# encodings with the UTF-8 patterns from the config file (#231).
|
|
130
|
+
@root = Text.utf8_identity(File.expand_path(root))
|
|
101
131
|
unknown = values.keys - DEFAULTS.keys
|
|
102
132
|
raise ConfigError, "unknown config key(s): #{unknown.join(', ')}" if unknown.any?
|
|
103
133
|
|
|
134
|
+
@deprecation_warnings = deprecations(values)
|
|
135
|
+
@explicit_keys = (explicit || values.keys).map(&:to_s).freeze
|
|
104
136
|
@values = DEFAULTS.merge(values)
|
|
105
137
|
coerce!
|
|
106
138
|
validate!
|
|
@@ -115,6 +147,26 @@ module Saboteur
|
|
|
115
147
|
@values.fetch(key.to_s, default)
|
|
116
148
|
end
|
|
117
149
|
|
|
150
|
+
# Keys the user set (config file or CLI flag) rather than relying on
|
|
151
|
+
# the default; carried through Engine's config clone so provenance
|
|
152
|
+
# survives (#296).
|
|
153
|
+
attr_reader :explicit_keys
|
|
154
|
+
|
|
155
|
+
# A copy with some values replaced or dropped, keeping which keys the
|
|
156
|
+
# user set explicitly: the replacements count as explicit, the dropped
|
|
157
|
+
# keys lose it (#296).
|
|
158
|
+
def with(values, except: [])
|
|
159
|
+
merged = to_h.except(*except).merge(values)
|
|
160
|
+
Config.new(merged, root: root, explicit: (explicit_keys - except.map(&:to_s)) | values.keys.map(&:to_s))
|
|
161
|
+
end
|
|
162
|
+
|
|
163
|
+
# Whether the user set a key (config file or CLI flag) rather than
|
|
164
|
+
# relying on the default. An explicitly requested report artifact is
|
|
165
|
+
# completion evidence, so its write failure is a hard error (#296).
|
|
166
|
+
def explicit?(key)
|
|
167
|
+
@explicit_keys.include?(key.to_s)
|
|
168
|
+
end
|
|
169
|
+
|
|
118
170
|
def to_h
|
|
119
171
|
@values.dup
|
|
120
172
|
end
|
|
@@ -124,13 +176,30 @@ module Saboteur
|
|
|
124
176
|
matched = self["include"].flat_map { |pattern| Dir.glob(pattern, base: root) }
|
|
125
177
|
excluded = self["exclude"].flat_map { |pattern| Dir.glob(pattern, base: root) }
|
|
126
178
|
(matched - excluded).select { |file| file.end_with?(".rb") }
|
|
127
|
-
.
|
|
179
|
+
.filter_map { |file| real_source_path(file) }
|
|
180
|
+
.uniq.sort
|
|
181
|
+
end
|
|
182
|
+
|
|
183
|
+
# A broken symlink matches Dir.glob but has no real path: dangling
|
|
184
|
+
# (ENOENT), a loop (ELOOP) or through a regular file (ENOTDIR). It is
|
|
185
|
+
# not mutation source, so it is skipped rather than crashing the run
|
|
186
|
+
# (#503). A regular file that cannot be resolved still raises.
|
|
187
|
+
def real_source_path(file)
|
|
188
|
+
path = File.expand_path(Text.utf8_identity(file), root)
|
|
189
|
+
File.realpath(path)
|
|
190
|
+
rescue SystemCallError
|
|
191
|
+
raise unless File.symlink?(path)
|
|
128
192
|
end
|
|
129
193
|
|
|
130
194
|
# Patterns that match no file. A pattern goes stale when files move;
|
|
131
195
|
# an exclude that silently stops excluding is a hazard, so say so.
|
|
132
196
|
def unmatched_patterns
|
|
133
197
|
%w[include exclude].flat_map do |key|
|
|
198
|
+
# The default exclude shields (#504) match nothing on a healthy
|
|
199
|
+
# project - that is their job - so only a configured exclude can go
|
|
200
|
+
# stale enough to warn about.
|
|
201
|
+
next [] if key == "exclude" && !explicit?("exclude")
|
|
202
|
+
|
|
134
203
|
self[key].select { |pattern| Dir.glob(pattern, base: root).empty? }.map { |pattern| "#{key}: #{pattern}" }
|
|
135
204
|
end
|
|
136
205
|
end
|
|
@@ -151,11 +220,23 @@ module Saboteur
|
|
|
151
220
|
|
|
152
221
|
private
|
|
153
222
|
|
|
223
|
+
# An explicit guess_tests = false (config file or --no-guess-tests) used
|
|
224
|
+
# to keep the whole suite; inference is the default now, so the opt-out
|
|
225
|
+
# is --all / c.test_selection = :all. guess_tests = true is the default
|
|
226
|
+
# behavior spelled out, so it stays silent.
|
|
227
|
+
def deprecations(values)
|
|
228
|
+
return [] unless values["guess_tests"] == false
|
|
229
|
+
|
|
230
|
+
["guess_tests = false is deprecated: inferred specs are the default and it no longer restores the " \
|
|
231
|
+
"whole suite; use --all or c.test_selection = :all to run every spec"]
|
|
232
|
+
end
|
|
233
|
+
|
|
154
234
|
def coerce!
|
|
155
235
|
INTEGER_KEYS.each { |key| @values[key] = Integer(@values[key]) }
|
|
156
236
|
FLOAT_KEYS.each { |key| @values[key] = Float(@values[key]) }
|
|
157
237
|
LIST_KEYS.each { |key| @values[key] = Array(@values[key]).map(&:to_s) }
|
|
158
|
-
@values[
|
|
238
|
+
NILABLE_FLOAT_KEYS.each { |key| @values[key] = Float(@values[key]) unless @values[key].nil? }
|
|
239
|
+
@values["test_selection"] = @values["test_selection"].to_s
|
|
159
240
|
rescue ArgumentError, TypeError => e
|
|
160
241
|
raise ConfigError, "invalid config value: #{e.message}"
|
|
161
242
|
end
|
|
@@ -164,6 +245,13 @@ module Saboteur
|
|
|
164
245
|
raise ConfigError, "max_mutants_per_method must be > 0" unless self["max_mutants_per_method"].positive?
|
|
165
246
|
end
|
|
166
247
|
|
|
248
|
+
# The unchanged-suite checks: how many, and the whole run's clock (#55, #245).
|
|
249
|
+
def validate_baseline!
|
|
250
|
+
raise ConfigError, "seed must be >= 0" if self["seed"].negative?
|
|
251
|
+
raise ConfigError, "baseline_passes must be >= 1" if self["baseline_passes"] < 1
|
|
252
|
+
raise ConfigError, "max_duration must be > 0" if self["max_duration"] && !self["max_duration"].positive?
|
|
253
|
+
end
|
|
254
|
+
|
|
167
255
|
def validate_report_paths!
|
|
168
256
|
json = path("json_report")
|
|
169
257
|
if json && json == path("html_report")
|
|
@@ -179,8 +267,43 @@ module Saboteur
|
|
|
179
267
|
def validate_modes!
|
|
180
268
|
validate_fail_fast!
|
|
181
269
|
validate_matrix!
|
|
182
|
-
|
|
270
|
+
validate_test_selection!
|
|
183
271
|
validate_gates!
|
|
272
|
+
validate_score_gates!
|
|
273
|
+
end
|
|
274
|
+
|
|
275
|
+
# Per-path gates (#473): normalize each entry to string pattern /
|
|
276
|
+
# float floor, and fail loud on a pattern no included file matches -
|
|
277
|
+
# a gate that matches nothing would pass vacuously forever.
|
|
278
|
+
def validate_score_gates!
|
|
279
|
+
gates = self["score_gates"]
|
|
280
|
+
return if gates.empty?
|
|
281
|
+
|
|
282
|
+
normalized = gates.map { |gate| normalize_score_gate(gate) }
|
|
283
|
+
prefix = File.join(File.realpath(root), "")
|
|
284
|
+
relative = included_files.map { |file| file.delete_prefix(prefix) }
|
|
285
|
+
flags = File::FNM_PATHNAME | File::FNM_EXTGLOB
|
|
286
|
+
normalized.each do |gate|
|
|
287
|
+
next if relative.any? { |file| File.fnmatch?(gate["pattern"], file, flags) }
|
|
288
|
+
|
|
289
|
+
raise ConfigError, "score_gate #{gate['pattern']} matches no included file"
|
|
290
|
+
end
|
|
291
|
+
@values["score_gates"] = normalized
|
|
292
|
+
end
|
|
293
|
+
|
|
294
|
+
def normalize_score_gate(gate)
|
|
295
|
+
pattern = gate["pattern"].to_s
|
|
296
|
+
begin
|
|
297
|
+
floor = Float(gate["min_score"])
|
|
298
|
+
rescue ArgumentError, TypeError
|
|
299
|
+
raise ConfigError, "score_gate min_score must be between 0 and 100, got #{gate['min_score'].inspect}"
|
|
300
|
+
end
|
|
301
|
+
unless floor.finite? && floor.between?(0, 100)
|
|
302
|
+
raise ConfigError, "score_gate min_score must be between 0 and 100, got #{gate['min_score'].inspect}"
|
|
303
|
+
end
|
|
304
|
+
raise ConfigError, "score_gate pattern must not be empty" if pattern.empty?
|
|
305
|
+
|
|
306
|
+
{ "pattern" => pattern, "min_score" => floor }
|
|
184
307
|
end
|
|
185
308
|
|
|
186
309
|
def validate_gates!
|
|
@@ -208,11 +331,32 @@ module Saboteur
|
|
|
208
331
|
raise ConfigError, "threshold must be between 0 and 100"
|
|
209
332
|
end
|
|
210
333
|
|
|
211
|
-
def
|
|
212
|
-
return
|
|
334
|
+
def validate_test_selection!
|
|
335
|
+
return if %w[auto all].include?(self["test_selection"])
|
|
336
|
+
|
|
337
|
+
raise ConfigError, "test_selection must be one of: auto, all"
|
|
338
|
+
end
|
|
339
|
+
|
|
340
|
+
# --test-command replaces the whole in-process machinery, on the
|
|
341
|
+
# EFFECTIVE config: CLI flags and config-file entries alike (#305).
|
|
342
|
+
# The command runs unsandboxed, so "strict" cannot be honored.
|
|
343
|
+
def validate_test_command!
|
|
344
|
+
command = self["test_command"]
|
|
345
|
+
# nil alone means unset: false, arrays and numbers are all rejected,
|
|
346
|
+
# never coerced and never sent to String methods (#305 QA).
|
|
347
|
+
raise ConfigError, "test_command must be a string" unless command.nil? || command.is_a?(String)
|
|
348
|
+
raise ConfigError, "test_command must not be blank" if command&.strip&.empty?
|
|
349
|
+
|
|
350
|
+
return if command.to_s.empty?
|
|
351
|
+
raise ConfigError, "test_command cannot be combined with tests" unless self["tests"].empty?
|
|
352
|
+
raise ConfigError, "test_command cannot be combined with incremental" if self["incremental"]
|
|
353
|
+
unless self["flaky_check"] == "off"
|
|
354
|
+
raise ConfigError, "test_command cannot be combined with flaky_check: an external run has no " \
|
|
355
|
+
"in-process scheduler to rerun the decisive tests through"
|
|
356
|
+
end
|
|
357
|
+
return unless self["sandbox"] == "strict"
|
|
213
358
|
|
|
214
|
-
raise ConfigError,
|
|
215
|
-
raise ConfigError, "guess_tests cannot be combined with tests" if self["tests"].any?
|
|
359
|
+
raise ConfigError, 'test_command runs the command unsandboxed, so sandbox "strict" cannot be honored'
|
|
216
360
|
end
|
|
217
361
|
|
|
218
362
|
def validate_reeval!
|
|
@@ -222,14 +366,17 @@ module Saboteur
|
|
|
222
366
|
|
|
223
367
|
def validate!
|
|
224
368
|
raise ConfigError, "jobs must be >= 1" if self["jobs"] < 1
|
|
225
|
-
unless [true, false, "strict"].include?(self["sandbox"])
|
|
226
|
-
raise ConfigError, 'sandbox must be true, false, or "
|
|
369
|
+
unless [true, false, "strict", "seatbelt"].include?(self["sandbox"])
|
|
370
|
+
raise ConfigError, 'sandbox must be true, false, "strict", or "seatbelt"'
|
|
371
|
+
end
|
|
372
|
+
unless %w[off survivors decisive].include?(self["flaky_check"])
|
|
373
|
+
raise ConfigError, "flaky_check must be one of: off, survivors, decisive"
|
|
227
374
|
end
|
|
228
375
|
|
|
229
376
|
validate_mutant_limit!
|
|
230
377
|
validate_threshold!
|
|
231
378
|
validate_modes!
|
|
232
|
-
|
|
379
|
+
validate_baseline!
|
|
233
380
|
raise ConfigError, "max_memory_mb must be > 0" unless self["max_memory_mb"].positive?
|
|
234
381
|
raise ConfigError, "max_processes must be > 0" unless self["max_processes"].positive?
|
|
235
382
|
raise ConfigError, "timeout_factor must be > 0" unless self["timeout_factor"].positive?
|
|
@@ -237,6 +384,7 @@ module Saboteur
|
|
|
237
384
|
|
|
238
385
|
validate_reeval!
|
|
239
386
|
validate_operator_config!
|
|
387
|
+
validate_test_command!
|
|
240
388
|
end
|
|
241
389
|
|
|
242
390
|
public
|
data/lib/saboteur/config_file.rb
CHANGED
|
@@ -26,6 +26,14 @@ module Saboteur
|
|
|
26
26
|
end
|
|
27
27
|
end
|
|
28
28
|
|
|
29
|
+
# Repeatable per-path score gates (#473), the one setting that takes
|
|
30
|
+
# arguments: c.score_gate "app/payments/**/*", min_score: 95. Values
|
|
31
|
+
# are validated later, on the effective Config.
|
|
32
|
+
def score_gate(pattern, min_score:)
|
|
33
|
+
(@values["score_gates"] ||= []) << { "pattern" => pattern, "min_score" => min_score }
|
|
34
|
+
@lines["score_gates"] = caller_locations(1, 1).first.lineno
|
|
35
|
+
end
|
|
36
|
+
|
|
29
37
|
Config::INTEGER_KEYS.each do |key|
|
|
30
38
|
define_method("#{key}=") do |value|
|
|
31
39
|
unless value.is_a?(Integer)
|
|
@@ -0,0 +1,182 @@
|
|
|
1
|
+
# SPDX-License-Identifier: MIT
|
|
2
|
+
# frozen_string_literal: true
|
|
3
|
+
|
|
4
|
+
require "prism"
|
|
5
|
+
require_relative "text"
|
|
6
|
+
|
|
7
|
+
module Saboteur
|
|
8
|
+
# The lines where a constant is read, per file: the tests covering those
|
|
9
|
+
# lines are the ones that can observe a change to the constant's value
|
|
10
|
+
# (issue #306).
|
|
11
|
+
#
|
|
12
|
+
# Matching is lexical and honest about its edges. A bare `RULES` matches
|
|
13
|
+
# when its scope is the constant's owner or nested under it (constant
|
|
14
|
+
# lookup climbs lexical scopes) or the constant is top-level; a
|
|
15
|
+
# `Shop::RULES` path matches by its full spelling. A scope that writes its
|
|
16
|
+
# own `RULES` shadows the outer constant: reads under it resolve to the
|
|
17
|
+
# inner one and do not count. Reads reached through a superclass,
|
|
18
|
+
# `const_get`, or a compact `class Shop::Item` scope are not resolved: a
|
|
19
|
+
# constant only read that way finds no consumer here, and the run reports
|
|
20
|
+
# it as ran-at-load instead of guessing.
|
|
21
|
+
#
|
|
22
|
+
# Runtime reads (inside a def body) observe the constant when the mutant
|
|
23
|
+
# runs; load-time reads outside the constant's own file do not: the other
|
|
24
|
+
# file was loaded once with the original value and re-evaluating the
|
|
25
|
+
# constant's file does not run it again, so a read like
|
|
26
|
+
# `COPY = A::RULES` there keeps the original (#319). A capture inside a
|
|
27
|
+
# boot-time block (`1.times { COPY = A::RULES }`) is treated the same:
|
|
28
|
+
# statement-level blocks run while the file loads. That is a heuristic -
|
|
29
|
+
# a block passed to a deferring call (at_exit, define_method) runs later,
|
|
30
|
+
# but treating it as load-time errs toward dropping a constant rather
|
|
31
|
+
# than reporting a mutant nobody can observe.
|
|
32
|
+
module ConstantReads
|
|
33
|
+
Reads = Data.define(:runtime, :load, :defs)
|
|
34
|
+
|
|
35
|
+
module_function
|
|
36
|
+
|
|
37
|
+
# { "/abs/file.rb" => Reads } for the files that read the constant.
|
|
38
|
+
# runtime: def-body and block lines - a block runs when its caller runs
|
|
39
|
+
# it (an `it` block during the example), and line coverage decides
|
|
40
|
+
# whether any test reached it. load: top-level or class-body statement
|
|
41
|
+
# lines (observable only in the constant's own file, which the mutant
|
|
42
|
+
# re-evaluates). defs: the start line of each def a runtime read sits
|
|
43
|
+
# in, for the method-call counter fallback (endless defs have no line
|
|
44
|
+
# coverage). Only files whose text mentions the name are parsed.
|
|
45
|
+
def across(files, constant)
|
|
46
|
+
mention = /\b#{Regexp.escape(constant.name.to_s)}\b/
|
|
47
|
+
files.each_with_object({}) do |file, hits|
|
|
48
|
+
source = File.binread(file)
|
|
49
|
+
next unless mentions?(source, mention)
|
|
50
|
+
|
|
51
|
+
reads = reads_in(source, constant)
|
|
52
|
+
hits[file] = reads unless reads.runtime.empty? && reads.load.empty?
|
|
53
|
+
end
|
|
54
|
+
end
|
|
55
|
+
|
|
56
|
+
# The lines a mutant of the constant can be observed through: runtime
|
|
57
|
+
# reads everywhere, load reads in the constant's own file only.
|
|
58
|
+
def observable_lines(reads_by_file, constant)
|
|
59
|
+
reads_by_file.each_with_object({}) do |(file, reads), hits|
|
|
60
|
+
lines = reads.runtime.dup
|
|
61
|
+
lines.concat(reads.load) if file == constant.file
|
|
62
|
+
hits[file] = lines.uniq.sort unless lines.empty?
|
|
63
|
+
end
|
|
64
|
+
end
|
|
65
|
+
|
|
66
|
+
def mentions?(source, mention)
|
|
67
|
+
Text.utf8_identity(source).match?(mention)
|
|
68
|
+
rescue Encoding::CompatibilityError, ArgumentError
|
|
69
|
+
false
|
|
70
|
+
end
|
|
71
|
+
|
|
72
|
+
def reads_in(source, constant)
|
|
73
|
+
result = Prism.parse(source)
|
|
74
|
+
return Reads.new([], [], []) if result.failure?
|
|
75
|
+
|
|
76
|
+
out = { runtime: [], load: [], defs: [] }
|
|
77
|
+
walk(result.value, [], constant, out, in_def: nil, in_block: false, shadowed: nil)
|
|
78
|
+
Reads.new(out[:runtime].uniq.sort, out[:load].uniq.sort, out[:defs].uniq.sort)
|
|
79
|
+
end
|
|
80
|
+
|
|
81
|
+
def walk(node, nesting, constant, out, in_def:, in_block:, shadowed:, capture: false)
|
|
82
|
+
return if node.nil?
|
|
83
|
+
|
|
84
|
+
case node
|
|
85
|
+
when Prism::DefNode
|
|
86
|
+
walk(node.body, nesting, constant, out, in_def: node.location.start_line, in_block: false,
|
|
87
|
+
shadowed: shadowed)
|
|
88
|
+
when Prism::BlockNode, Prism::LambdaNode
|
|
89
|
+
walk(node.body, nesting, constant, out, in_def: in_def, in_block: true, capture: capture,
|
|
90
|
+
shadowed: shadowed)
|
|
91
|
+
when Prism::ConstantWriteNode, Prism::ConstantPathWriteNode
|
|
92
|
+
# A capture (`COPY = <expr>`): the value reads the constant once,
|
|
93
|
+
# when the write runs. Walk the value flagged as a capture so a
|
|
94
|
+
# boot-time block write (`1.times { COPY = A::RULES }`) lands in
|
|
95
|
+
# load, not runtime (#319).
|
|
96
|
+
walk(node.value, nesting, constant, out, in_def: in_def, in_block: in_block, capture: true,
|
|
97
|
+
shadowed: shadowed)
|
|
98
|
+
when Prism::ConstantReadNode
|
|
99
|
+
record(node, out, in_def, in_block, capture) if !shadowed && bare_match?(node, nesting, constant)
|
|
100
|
+
when Prism::ConstantPathNode
|
|
101
|
+
# `Shop::RULES` holds no ConstantReadNode for RULES, only for Shop;
|
|
102
|
+
# match the path's full spelling.
|
|
103
|
+
record(node, out, in_def, in_block, capture) if node.slice.delete_prefix("::") == constant.id
|
|
104
|
+
walk_children(node, nesting, constant, out, in_def: in_def, in_block: in_block, capture: capture,
|
|
105
|
+
shadowed: shadowed)
|
|
106
|
+
when Prism::ModuleNode, Prism::ClassNode, Prism::SingletonClassNode
|
|
107
|
+
walk_scope(node, nesting, constant, out, shadowed)
|
|
108
|
+
else
|
|
109
|
+
walk_children(node, nesting, constant, out, in_def: in_def, in_block: in_block, capture: capture,
|
|
110
|
+
shadowed: shadowed)
|
|
111
|
+
end
|
|
112
|
+
end
|
|
113
|
+
|
|
114
|
+
# `class << self` joins the nesting as "<attached>.singleton_class",
|
|
115
|
+
# matching how ConstantSubjectFinder names that owner (#380); a non-self
|
|
116
|
+
# receiver or a top-level singleton has no static name and adds nothing.
|
|
117
|
+
# The shadow carries the path of the scope that set it. Entering the
|
|
118
|
+
# constant's owner scope lifts a shadow set outside it: the owner's
|
|
119
|
+
# own definition is lexically closer there (#395). A shadow set
|
|
120
|
+
# inside the owner subtree still binds below the nested scope that
|
|
121
|
+
# wrote the constant.
|
|
122
|
+
def walk_scope(node, nesting, constant, out, shadowed)
|
|
123
|
+
inner = scope_nesting(node, nesting)
|
|
124
|
+
path = inner.join("::")
|
|
125
|
+
scopes = [constant.owner.to_s]
|
|
126
|
+
shadowed = nil if shadowed && scopes.any? { |scope| path == scope || path.start_with?("#{scope}::") } &&
|
|
127
|
+
scopes.none? { |scope| shadowed == scope || shadowed.start_with?("#{scope}::") }
|
|
128
|
+
shadowed = path if shadowed.nil? && path != scopes.first && writes_constant?(node.body, constant.name)
|
|
129
|
+
walk(node.body, inner, constant, out, in_def: nil, in_block: false, shadowed: shadowed)
|
|
130
|
+
end
|
|
131
|
+
|
|
132
|
+
def scope_nesting(node, nesting)
|
|
133
|
+
return nesting + [node.constant_path.slice.delete_prefix("::")] unless node.is_a?(Prism::SingletonClassNode)
|
|
134
|
+
return nesting unless node.expression.is_a?(Prism::SelfNode) && nesting.any?
|
|
135
|
+
|
|
136
|
+
nesting[0...-1] + ["#{nesting.last}.singleton_class"]
|
|
137
|
+
end
|
|
138
|
+
|
|
139
|
+
def walk_children(node, nesting, constant, out, in_def:, in_block:, capture:, shadowed:)
|
|
140
|
+
node.compact_child_nodes.each do |child|
|
|
141
|
+
walk(child, nesting, constant, out, in_def: in_def, in_block: in_block, capture: capture,
|
|
142
|
+
shadowed: shadowed)
|
|
143
|
+
end
|
|
144
|
+
end
|
|
145
|
+
|
|
146
|
+
def record(node, out, in_def, in_block, capture)
|
|
147
|
+
line = node.location.start_line
|
|
148
|
+
if in_def
|
|
149
|
+
out[:runtime] << line
|
|
150
|
+
out[:defs] << in_def
|
|
151
|
+
elsif in_block && !capture
|
|
152
|
+
out[:runtime] << line
|
|
153
|
+
else
|
|
154
|
+
# A bare statement read, or a capture in a boot-time block: both
|
|
155
|
+
# resolved while the file loaded, with the original value.
|
|
156
|
+
out[:load] << line
|
|
157
|
+
end
|
|
158
|
+
end
|
|
159
|
+
|
|
160
|
+
# A bare `RULES` reads this constant when the read's scope is the
|
|
161
|
+
# constant's owner (or its singleton scope, whose ancestry reaches the
|
|
162
|
+
# attached class) or nested under either, or it is top-level (#380).
|
|
163
|
+
def bare_match?(node, nesting, constant)
|
|
164
|
+
return false unless node.name == constant.name
|
|
165
|
+
return true if constant.owner.nil?
|
|
166
|
+
|
|
167
|
+
path = nesting.join("::")
|
|
168
|
+
[constant.owner, "#{constant.owner}.singleton_class"].any? do |scope|
|
|
169
|
+
path == scope || path.start_with?("#{scope}::")
|
|
170
|
+
end
|
|
171
|
+
end
|
|
172
|
+
|
|
173
|
+
# True when the scope body assigns the constant name at its own level
|
|
174
|
+
# (not inside a nested def or scope).
|
|
175
|
+
def writes_constant?(body, name)
|
|
176
|
+
statements = body&.body || []
|
|
177
|
+
statements.any? do |statement|
|
|
178
|
+
statement.is_a?(Prism::ConstantWriteNode) && statement.name == name
|
|
179
|
+
end
|
|
180
|
+
end
|
|
181
|
+
end
|
|
182
|
+
end
|
|
@@ -0,0 +1,70 @@
|
|
|
1
|
+
# SPDX-License-Identifier: MIT
|
|
2
|
+
# frozen_string_literal: true
|
|
3
|
+
|
|
4
|
+
require_relative "subject"
|
|
5
|
+
|
|
6
|
+
module Saboteur
|
|
7
|
+
# A constant assigned a hash or array literal, as a mutation subject
|
|
8
|
+
# (issue #306): `RULES = { ... }` or `TAGS = [ ... ]`, with an optional
|
|
9
|
+
# trailing `.freeze`. Only --since selection creates these (see
|
|
10
|
+
# Engine::SubjectSelection), and the diff scope narrows their mutants to
|
|
11
|
+
# the entries the diff changed.
|
|
12
|
+
#
|
|
13
|
+
# A constant is not a method, so the surgical def re-evaluation does not
|
|
14
|
+
# apply: mutants run by re-evaluating the whole file with the mutated
|
|
15
|
+
# assignment spliced in (Execution::Reevaluator#eval_whole_file, which
|
|
16
|
+
# silences the "already initialized constant" warning). Re-evaluation
|
|
17
|
+
# replaces the constant's object; code that reads the constant at test
|
|
18
|
+
# time sees the mutant, while anything that copied the old object at load
|
|
19
|
+
# time keeps it. consumer_lines names the reads a test can cover; the
|
|
20
|
+
# coverage map turns them into the mutant's tests.
|
|
21
|
+
class ConstantSubject < Subject
|
|
22
|
+
attr_reader :literal_node
|
|
23
|
+
# { "/abs/file.rb" => [lines reading the constant] } across the include
|
|
24
|
+
# set, filled in after selection (the finder sees one file at a time).
|
|
25
|
+
attr_accessor :consumer_lines
|
|
26
|
+
# { "/abs/file.rb" => [def start lines] }: the def each runtime read
|
|
27
|
+
# sits in, for the method-call counter fallback (endless defs have no
|
|
28
|
+
# line coverage).
|
|
29
|
+
attr_accessor :consumer_defs
|
|
30
|
+
|
|
31
|
+
def initialize(name:, owner:, file:, node:, literal_node:, file_source:, nesting:, frozen_literals:,
|
|
32
|
+
consumer_lines: {}, consumer_defs: {})
|
|
33
|
+
super(name:, owner:, singleton: false, sclass: false, file:, node:, file_source:, nesting:,
|
|
34
|
+
frozen_literals:, sig_span: nil)
|
|
35
|
+
@literal_node = literal_node
|
|
36
|
+
@consumer_lines = consumer_lines
|
|
37
|
+
@consumer_defs = consumer_defs
|
|
38
|
+
end
|
|
39
|
+
|
|
40
|
+
# "Shop::RULES", or "RULES" at the top level. The :: separator keeps a
|
|
41
|
+
# constant's id apart from the Klass#method and Klass.method spellings.
|
|
42
|
+
def id
|
|
43
|
+
owner ? "#{owner}::#{name}" : name.to_s
|
|
44
|
+
end
|
|
45
|
+
|
|
46
|
+
# There is no method body to redefine: constants always take the
|
|
47
|
+
# whole-file evaluation path.
|
|
48
|
+
def whole_file_only?
|
|
49
|
+
true
|
|
50
|
+
end
|
|
51
|
+
|
|
52
|
+
# The assignment runs while the file loads, so its lines carry the load
|
|
53
|
+
# evidence; the reads of the constant in this file are the lines tests
|
|
54
|
+
# can cover.
|
|
55
|
+
def coverage_lines
|
|
56
|
+
((start_line..end_line).to_a + Array(consumer_lines[file])).uniq
|
|
57
|
+
end
|
|
58
|
+
|
|
59
|
+
# Every edit sits inside the literal and is observable wherever the
|
|
60
|
+
# constant is read, so the assignment's own lines are the evidence that
|
|
61
|
+
# the edit ran (they execute at load, never inside a test).
|
|
62
|
+
def coverage_lines_for(_edit)
|
|
63
|
+
(start_line..end_line).to_a
|
|
64
|
+
end
|
|
65
|
+
|
|
66
|
+
def recovers?
|
|
67
|
+
false
|
|
68
|
+
end
|
|
69
|
+
end
|
|
70
|
+
end
|