saboteur 0.7.0 → 0.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +657 -0
- data/README.md +198 -1077
- data/lib/saboteur/cli/exit_status.rb +32 -1
- data/lib/saboteur/cli/help_options.rb +2 -1
- data/lib/saboteur/cli/history_command.rb +135 -0
- data/lib/saboteur/cli/presets.rb +6 -2
- data/lib/saboteur/cli/selection_options.rb +31 -0
- data/lib/saboteur/cli/shard_options.rb +17 -0
- data/lib/saboteur/cli/source_set.rb +53 -0
- data/lib/saboteur/cli/survivors_command.rb +157 -0
- data/lib/saboteur/cli.rb +71 -54
- data/lib/saboteur/config.rb +98 -5
- data/lib/saboteur/config_file.rb +8 -0
- data/lib/saboteur/constant_reads.rb +182 -0
- data/lib/saboteur/constant_subject.rb +70 -0
- data/lib/saboteur/constant_subject_finder.rb +281 -0
- data/lib/saboteur/coverage_reads.rb +29 -0
- data/lib/saboteur/dirty_tree_warning.rb +128 -0
- data/lib/saboteur/doctor.rb +9 -0
- data/lib/saboteur/engine/baseline_guard.rb +62 -14
- data/lib/saboteur/engine/boot.rb +4 -0
- data/lib/saboteur/engine/candidate_selection.rb +19 -0
- data/lib/saboteur/engine/candidates.rb +43 -3
- data/lib/saboteur/engine/coverage_store.rb +32 -10
- data/lib/saboteur/engine/defined_order_prefix.rb +43 -0
- data/lib/saboteur/engine/diagnostics.rb +18 -0
- data/lib/saboteur/engine/distribution.rb +10 -0
- data/lib/saboteur/engine/early_stop_recording.rb +3 -2
- data/lib/saboteur/engine/gated_run.rb +35 -0
- data/lib/saboteur/engine/load_coverage.rb +45 -0
- data/lib/saboteur/engine/out_of_scope_guard.rb +21 -0
- data/lib/saboteur/engine/pipelines.rb +107 -0
- data/lib/saboteur/engine/result_cache.rb +65 -8
- data/lib/saboteur/engine/run_budget.rb +27 -0
- data/lib/saboteur/engine/sandbox_policy.rb +38 -0
- data/lib/saboteur/engine/scheduler.rb +109 -3
- data/lib/saboteur/engine/selection_announcement.rb +54 -0
- data/lib/saboteur/engine/shard.rb +69 -0
- data/lib/saboteur/engine/subject_selection.rb +295 -20
- data/lib/saboteur/engine/test_guess.rb +315 -25
- data/lib/saboteur/engine/unchanged_syntax.rb +17 -2
- data/lib/saboteur/engine.rb +100 -127
- data/lib/saboteur/execution/external_runner.rb +471 -0
- data/lib/saboteur/execution/marker_sweep.rb +79 -0
- data/lib/saboteur/execution/peak_memory.rb +28 -0
- data/lib/saboteur/execution/process_table.rb +12 -0
- data/lib/saboteur/execution/process_tree.rb +65 -5
- data/lib/saboteur/execution/reevaluator.rb +68 -12
- data/lib/saboteur/execution/runner.rb +46 -7
- data/lib/saboteur/execution/sandbox.rb +15 -2
- data/lib/saboteur/execution/seatbelt.rb +226 -0
- data/lib/saboteur/execution/supervisor.rb +27 -10
- data/lib/saboteur/execution/worker_pool.rb +23 -1
- data/lib/saboteur/frameworks/sqlite_memory_database.rb +49 -0
- data/lib/saboteur/gates.rb +43 -2
- data/lib/saboteur/help.rb +60 -9
- data/lib/saboteur/ignore_list.rb +29 -6
- data/lib/saboteur/integration/rspec.rb +135 -5
- data/lib/saboteur/mutation_source/argument_removal.rb +1 -1
- data/lib/saboteur/mutation_source/builtin.rb +26 -5
- data/lib/saboteur/mutation_source/chain_removal.rb +1 -1
- data/lib/saboteur/mutation_source/condition_negation.rb +1 -1
- data/lib/saboteur/mutation_source/constant_literals.rb +66 -0
- data/lib/saboteur/mutation_source/element_removal.rb +73 -0
- data/lib/saboteur/mutation_source/endpoint_replacement.rb +1 -1
- data/lib/saboteur/mutation_source/entry_edits.rb +44 -0
- data/lib/saboteur/mutation_source/filter_replacement.rb +1 -1
- data/lib/saboteur/mutation_source/hazard_guard.rb +108 -0
- data/lib/saboteur/mutation_source/literal_replacement.rb +1 -1
- data/lib/saboteur/mutation_source/method_swap.rb +148 -0
- data/lib/saboteur/mutation_source/operator_assignment.rb +1 -1
- data/lib/saboteur/mutation_source/operator_examples.rb +3 -1
- data/lib/saboteur/mutation_source/operator_registry.rb +4 -2
- data/lib/saboteur/mutation_source/operator_visit.rb +1 -1
- data/lib/saboteur/mutation_source/pair_removal.rb +13 -31
- data/lib/saboteur/mutation_source/range_replacement.rb +1 -1
- data/lib/saboteur/mutation_source/regex_replacement.rb +93 -0
- data/lib/saboteur/preview.rb +120 -12
- data/lib/saboteur/refinement_activations.rb +60 -0
- data/lib/saboteur/reporter/agent.rb +9 -1
- data/lib/saboteur/reporter/agent_commands.rb +10 -3
- data/lib/saboteur/reporter/agent_incomplete.rb +11 -0
- data/lib/saboteur/reporter/agent_summary.rb +4 -3
- data/lib/saboteur/reporter/findings.rb +5 -2
- data/lib/saboteur/reporter/history_policy.rb +11 -0
- data/lib/saboteur/reporter/history_reader.rb +47 -0
- data/lib/saboteur/reporter/html.rb +26 -4
- data/lib/saboteur/reporter/mutation_text.rb +6 -0
- data/lib/saboteur/reporter/stryker.rb +27 -3
- data/lib/saboteur/reporter/survivor_classification.rb +101 -0
- data/lib/saboteur/reporter/survivor_list.rb +15 -9
- data/lib/saboteur/reporter/terminal.rb +40 -7
- data/lib/saboteur/result.rb +19 -3
- data/lib/saboteur/selection/coverage_map.rb +105 -10
- data/lib/saboteur/selection/dependency_shield.rb +80 -0
- data/lib/saboteur/selection/external_test_map.rb +27 -0
- data/lib/saboteur/selection/git_diff.rb +93 -11
- data/lib/saboteur/selection/layout_only_lines.rb +80 -0
- data/lib/saboteur/selection/recorder.rb +22 -4
- data/lib/saboteur/selection/source_scope.rb +22 -0
- data/lib/saboteur/subject.rb +42 -7
- data/lib/saboteur/subject_finder.rb +173 -29
- data/lib/saboteur/version.rb +1 -1
- data/lib/saboteur/working_tree.rb +38 -15
- data/lib/saboteur.rb +13 -0
- metadata +35 -2
data/lib/saboteur/engine/boot.rb
CHANGED
|
@@ -10,6 +10,10 @@ module Saboteur
|
|
|
10
10
|
# A test suite that (perhaps only under a mutant) starts a saboteur run
|
|
11
11
|
# would otherwise run saboteur inside a saboteur child, recursively.
|
|
12
12
|
def self.refuse_recursion!
|
|
13
|
+
if ENV[Execution::ExternalRunner::CHILD_MARKER]
|
|
14
|
+
raise Error, "refusing to start: this process runs inside a saboteur external test command " \
|
|
15
|
+
"(#{Execution::ExternalRunner::CHILD_MARKER} is set); a nested run could race its tree"
|
|
16
|
+
end
|
|
13
17
|
return unless ENV[Execution::Supervisor::CHILD_ENV]
|
|
14
18
|
|
|
15
19
|
raise Error, "refusing to start: this process is a saboteur child " \
|
|
@@ -22,6 +22,10 @@ module Saboteur
|
|
|
22
22
|
reason = subject.skip_reason || (skip_reason_for(subject) if @map.tests_for(subject).empty?)
|
|
23
23
|
if reason
|
|
24
24
|
@skipped_subjects[subject] = reason
|
|
25
|
+
# In a scoped run every subject is part of the change under
|
|
26
|
+
# review, so a skip is said up front, not only in the end-of-run
|
|
27
|
+
# summary (#574).
|
|
28
|
+
@diagnostics.announce(skip_announcement(subject, reason)) if scope
|
|
25
29
|
return @selected[subject] = []
|
|
26
30
|
end
|
|
27
31
|
|
|
@@ -44,6 +48,21 @@ module Saboteur
|
|
|
44
48
|
|
|
45
49
|
private
|
|
46
50
|
|
|
51
|
+
# A scoped no-coverage skip names the specs the run loaded: coverage
|
|
52
|
+
# living in a sibling spec file is otherwise invisible, and the false
|
|
53
|
+
# read is "untestable" when the truth is "not loaded" (#636). Names
|
|
54
|
+
# at most five, like the other selection notes.
|
|
55
|
+
def skip_announcement(subject, reason)
|
|
56
|
+
line = "#{subject.id}: skipped (#{reason}); its change is not verified by this run"
|
|
57
|
+
return line unless reason == "no coverage"
|
|
58
|
+
|
|
59
|
+
loaded = @map.tests.map { |id| id.split("[").first.delete_prefix("./") }.uniq.sort
|
|
60
|
+
more = loaded.size > 5 ? ", ..." : ""
|
|
61
|
+
names = loaded.empty? ? "none" : loaded.first(5).join(", ")
|
|
62
|
+
"#{line} - loaded specs: #{names}#{more}; " \
|
|
63
|
+
"other spec files may cover it, pass --test to load them"
|
|
64
|
+
end
|
|
65
|
+
|
|
47
66
|
# Load-only code is not "no coverage": it ran, just never inside a
|
|
48
67
|
# test, and no forked test can re-run it (see CoverageMap::RAN_AT_LOAD).
|
|
49
68
|
def skip_reason_for(subject)
|
|
@@ -10,6 +10,25 @@ module Saboteur
|
|
|
10
10
|
# a subject into runnable candidates: applied, re-parsed, de-duplicated,
|
|
11
11
|
# coarse first. Returns [[mutation, mutated_text]].
|
|
12
12
|
class Candidates
|
|
13
|
+
# The generator for constant hash/array literal subjects (#306); the
|
|
14
|
+
# method operators walk a def body a constant does not have.
|
|
15
|
+
def self.build(config, diagnostics, hazard_guard:)
|
|
16
|
+
new(build_sources(config), diagnostics, hazard_guard: hazard_guard,
|
|
17
|
+
constant_source: build_constant_source(config))
|
|
18
|
+
end
|
|
19
|
+
|
|
20
|
+
# Constants are generated by the builtin operator family, so a
|
|
21
|
+
# config that drops "builtin" from c.sources drops constant mutation
|
|
22
|
+
# too: the configured sources are the boundary for every subject
|
|
23
|
+
# kind (#319). Custom sources never see constants.
|
|
24
|
+
def self.build_constant_source(config)
|
|
25
|
+
return nil unless config["sources"].include?("builtin")
|
|
26
|
+
|
|
27
|
+
MutationSource::ConstantLiterals.new(operators: config["operators"],
|
|
28
|
+
exclude_operators: config["exclude_operators"],
|
|
29
|
+
arid: config["arid"])
|
|
30
|
+
end
|
|
31
|
+
|
|
13
32
|
def self.build_sources(config)
|
|
14
33
|
root = File.realpath(config.root)
|
|
15
34
|
config["sources"].map do |entry|
|
|
@@ -19,15 +38,26 @@ module Saboteur
|
|
|
19
38
|
end
|
|
20
39
|
|
|
21
40
|
file, const = entry.split(":", 2)
|
|
22
|
-
|
|
41
|
+
unless const
|
|
42
|
+
if entry.include?("*")
|
|
43
|
+
raise ConfigError, "mutation source #{entry.inspect} looks like a source glob; " \
|
|
44
|
+
"did you mean c.include? c.sources registers custom operator sources " \
|
|
45
|
+
"(builtin or path.rb:ClassName)"
|
|
46
|
+
end
|
|
47
|
+
raise ConfigError, "mutation source #{entry.inspect} must be builtin or path.rb:ClassName"
|
|
48
|
+
end
|
|
23
49
|
|
|
24
50
|
require File.expand_path(file, root)
|
|
25
51
|
Object.const_get(const).new
|
|
26
52
|
end
|
|
27
53
|
end
|
|
28
54
|
|
|
29
|
-
|
|
55
|
+
# constant_source: the generator for ConstantSubject instances; nil
|
|
56
|
+
# leaves constants with no candidates (a safe empty list).
|
|
57
|
+
def initialize(sources, diagnostics, hazard_guard: nil, bytecode: BytecodeEquivalence.new,
|
|
58
|
+
constant_source: nil)
|
|
30
59
|
@sources = sources
|
|
60
|
+
@constant_source = constant_source
|
|
31
61
|
@diagnostics = diagnostics
|
|
32
62
|
@hazard_guard = hazard_guard
|
|
33
63
|
@bytecode = bytecode
|
|
@@ -40,7 +70,7 @@ module Saboteur
|
|
|
40
70
|
end
|
|
41
71
|
|
|
42
72
|
seen = Set[subject.source]
|
|
43
|
-
list =
|
|
73
|
+
list = sources_for(subject).flat_map { |source| proposals(source, subject) }.filter_map do |mutation|
|
|
44
74
|
candidate(subject, mutation, seen)
|
|
45
75
|
end
|
|
46
76
|
list = suppress_inline(list, subject)
|
|
@@ -50,6 +80,15 @@ module Saboteur
|
|
|
50
80
|
|
|
51
81
|
private
|
|
52
82
|
|
|
83
|
+
# Constant literals have their own generator; external custom sources
|
|
84
|
+
# implement the def-subject contract and never see constants (#306).
|
|
85
|
+
def sources_for(subject)
|
|
86
|
+
return @sources unless subject.is_a?(ConstantSubject)
|
|
87
|
+
return [] unless @constant_source
|
|
88
|
+
|
|
89
|
+
[@constant_source]
|
|
90
|
+
end
|
|
91
|
+
|
|
53
92
|
def suppress_inline(list, subject)
|
|
54
93
|
directives = InlineSuppression.new(subject.file, subject.file_source)
|
|
55
94
|
list.reject do |mutation, _text|
|
|
@@ -95,6 +134,7 @@ module Saboteur
|
|
|
95
134
|
seen << text
|
|
96
135
|
unless parses?(text)
|
|
97
136
|
@diagnostics.count_invalid_mutation
|
|
137
|
+
@diagnostics.warn("#{subject.id}: dropped #{mutation.operator} mutation (did not parse)")
|
|
98
138
|
return record_drop(subject, mutation, "invalid_syntax")
|
|
99
139
|
end
|
|
100
140
|
|
|
@@ -15,7 +15,8 @@ module Saboteur
|
|
|
15
15
|
LOCKFILES = %w[Gemfile.lock gems.locked].freeze
|
|
16
16
|
LOG = "coverage-build.log"
|
|
17
17
|
|
|
18
|
-
def initialize(config:, integration:, hooks:, supervisor:, diagnostics:, load_lines: {},
|
|
18
|
+
def initialize(config:, integration:, hooks:, supervisor:, diagnostics:, load_lines: {}, load_methods: {},
|
|
19
|
+
timeout_cap: nil)
|
|
19
20
|
@config = config
|
|
20
21
|
@integration = integration
|
|
21
22
|
@hooks = hooks
|
|
@@ -23,7 +24,9 @@ module Saboteur
|
|
|
23
24
|
@diagnostics = diagnostics
|
|
24
25
|
@timeout_cap = timeout_cap
|
|
25
26
|
@load_lines = load_lines
|
|
27
|
+
@load_methods = load_methods
|
|
26
28
|
@root = File.realpath(config.root)
|
|
29
|
+
@root_prefixes = Selection::CoverageMap.root_prefixes(config.root)
|
|
27
30
|
end
|
|
28
31
|
|
|
29
32
|
# True when fetch would reuse the cached map instead of running the
|
|
@@ -49,6 +52,21 @@ module Saboteur
|
|
|
49
52
|
|
|
50
53
|
private
|
|
51
54
|
|
|
55
|
+
def write_map(writer, recorder, build_seconds)
|
|
56
|
+
writer.write(recorder.result(build_seconds: build_seconds, loaded_files: loaded_project_files).to_json)
|
|
57
|
+
writer.close
|
|
58
|
+
end
|
|
59
|
+
|
|
60
|
+
# Every project source this child loaded, even a lazy require outside the
|
|
61
|
+
# include list that the recorder drops: --incremental must watch it, or a
|
|
62
|
+
# change there keeps stale verdicts (#353).
|
|
63
|
+
def loaded_project_files
|
|
64
|
+
$LOADED_FEATURES.filter_map do |file|
|
|
65
|
+
path = Text.utf8_strict(Selection::CoverageMap.canonical(file))
|
|
66
|
+
path if path.end_with?(".rb") && @root_prefixes.any? { |prefix| path.start_with?(prefix) }
|
|
67
|
+
end
|
|
68
|
+
end
|
|
69
|
+
|
|
52
70
|
def cache_path(files)
|
|
53
71
|
File.join(@config.path("cache_dir"), "coverage-#{cache_key(files)}.json")
|
|
54
72
|
end
|
|
@@ -122,13 +140,16 @@ module Saboteur
|
|
|
122
140
|
# The parent's load lines merge into the freshly built map: the forked
|
|
123
141
|
# build child cannot inherit the parent's Coverage counters.
|
|
124
142
|
def with_parent_load_lines(map)
|
|
125
|
-
@load_lines.empty? ? map : map.with_load_lines(@load_lines)
|
|
143
|
+
@load_lines.empty? && @load_methods.empty? ? map : map.with_load_lines(@load_lines, @load_methods)
|
|
126
144
|
end
|
|
127
145
|
|
|
128
146
|
# A whole-suite build that records nothing - no line hit and no method
|
|
129
|
-
# call - in any included file fails the run (exit 3)
|
|
130
|
-
#
|
|
131
|
-
#
|
|
147
|
+
# call - in any included file fails the run (exit 3). The recorder's map
|
|
148
|
+
# also holds the suite's spec files, so the check is per included file,
|
|
149
|
+
# not whole-map: a suite that tests nothing here still records its own
|
|
150
|
+
# spec code. Usually no test in the suite exercises these files, or
|
|
151
|
+
# the setup hid the app from Coverage (Bootsnap's compile cache is the
|
|
152
|
+
# default Rails case). An
|
|
132
153
|
# all-endless-def file records method calls only and passes. Either way
|
|
133
154
|
# a run on a truly empty map would test nothing, and exit 0 would lie.
|
|
134
155
|
# With explicit tests (--test) an empty map is a legitimate outcome
|
|
@@ -137,7 +158,7 @@ module Saboteur
|
|
|
137
158
|
# map too, so a map cached before this check cannot keep a run green,
|
|
138
159
|
# and the guard runs before saving, so an empty map is never cached.
|
|
139
160
|
def refuse_empty_map(map, files)
|
|
140
|
-
return map if files.empty? || !map.
|
|
161
|
+
return map if files.empty? || !map.empty_for?(files)
|
|
141
162
|
|
|
142
163
|
if @config["tests"].empty?
|
|
143
164
|
raise Error, "Coverage recorded nothing in #{files.size} included files during a whole-suite " \
|
|
@@ -175,7 +196,10 @@ module Saboteur
|
|
|
175
196
|
$stderr.reopen($stdout)
|
|
176
197
|
@hooks.run(:after_fork)
|
|
177
198
|
started = Process.clock_gettime(Process::CLOCK_MONOTONIC)
|
|
178
|
-
|
|
199
|
+
# The suite's own files are recorded too: a spec can read a
|
|
200
|
+
# constant directly, and without its line hits no test maps to the
|
|
201
|
+
# read (#319).
|
|
202
|
+
recorder = Selection::Recorder.new(files + @integration.test_files)
|
|
179
203
|
recorder.start
|
|
180
204
|
Kernel.srand(@config["seed"])
|
|
181
205
|
passed = @integration.run_all(recorder)
|
|
@@ -183,9 +207,7 @@ module Saboteur
|
|
|
183
207
|
raise Error, "the unchanged suite failed while building coverage. Fix the suite, then run again " \
|
|
184
208
|
"with seed #{@config['seed']}."
|
|
185
209
|
end
|
|
186
|
-
|
|
187
|
-
writer.write(recorder.result(build_seconds: build_seconds).to_json)
|
|
188
|
-
writer.close
|
|
210
|
+
write_map(writer, recorder, Process.clock_gettime(Process::CLOCK_MONOTONIC) - started)
|
|
189
211
|
rescue Exception => e # rubocop:disable Lint/RescueException
|
|
190
212
|
writer.write(JSON.generate(error: "#{e.class}: #{e.message}"))
|
|
191
213
|
exit!(1)
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
# SPDX-License-Identifier: MIT
|
|
2
|
+
# frozen_string_literal: true
|
|
3
|
+
|
|
4
|
+
module Saboteur
|
|
5
|
+
class Engine
|
|
6
|
+
# The examples an order-dependent spec file needs, in the order RSpec
|
|
7
|
+
# defines them: for each file with a covering example, every example from
|
|
8
|
+
# the start of the file through its last covering example. Later examples
|
|
9
|
+
# cannot change earlier ones, so this is the smallest run that keeps
|
|
10
|
+
# whatever state the earlier examples leave behind.
|
|
11
|
+
class DefinedOrderPrefix
|
|
12
|
+
ID_PATTERN = /\A(?<file>.*)\[(?<path>[\d:]+)\]\z/
|
|
13
|
+
|
|
14
|
+
# every_test: all example ids the map knows, in any order.
|
|
15
|
+
def initialize(every_test)
|
|
16
|
+
@every_test = every_test
|
|
17
|
+
end
|
|
18
|
+
|
|
19
|
+
# The prefix for the covering examples; nil when it adds nothing to
|
|
20
|
+
# what already ran (an id that is not an RSpec id included).
|
|
21
|
+
def for(covering)
|
|
22
|
+
parsed = covering.map { |id| parse(id) }
|
|
23
|
+
return nil if parsed.any?(&:nil?)
|
|
24
|
+
|
|
25
|
+
prefix = parsed.group_by(&:first).flat_map { |file, entries| file_prefix(file, entries.map(&:last).max) }
|
|
26
|
+
prefix == covering ? nil : prefix
|
|
27
|
+
end
|
|
28
|
+
|
|
29
|
+
private
|
|
30
|
+
|
|
31
|
+
def file_prefix(file, last_position)
|
|
32
|
+
in_file = @every_test.filter_map { |id| (entry = parse(id)) && entry.first == file ? entry : nil }
|
|
33
|
+
through = in_file.select { |entry| (entry.last <=> last_position) <= 0 }.sort_by(&:last)
|
|
34
|
+
through.map { |entry| "#{file}[#{entry.last.join(':')}]" }
|
|
35
|
+
end
|
|
36
|
+
|
|
37
|
+
def parse(id)
|
|
38
|
+
match = ID_PATTERN.match(id)
|
|
39
|
+
match && [match[:file], match[:path].split(":").map(&:to_i)]
|
|
40
|
+
end
|
|
41
|
+
end
|
|
42
|
+
end
|
|
43
|
+
end
|
|
@@ -86,12 +86,30 @@ module Saboteur
|
|
|
86
86
|
@discovery_failures = []
|
|
87
87
|
@hazard_suppressions = 0
|
|
88
88
|
@inline_suppressions = []
|
|
89
|
+
@erb_template_skips = []
|
|
90
|
+
@erb_template_note_emitted = false
|
|
89
91
|
end
|
|
90
92
|
|
|
91
93
|
def warn(message)
|
|
92
94
|
@warnings << message
|
|
93
95
|
end
|
|
94
96
|
|
|
97
|
+
# ERB generator templates (Rails generators carry `.rb` files with ERB
|
|
98
|
+
# interpolation, conventionally under `lib/generators/**/templates/`)
|
|
99
|
+
# can never parse as plain Ruby, so they can never hold mutation
|
|
100
|
+
# subjects (#372). One summarized note per run replaces the per-file
|
|
101
|
+
# parse warning every discovery pass used to print; a real syntax
|
|
102
|
+
# error in a plain Ruby file still warns per file.
|
|
103
|
+
def record_erb_template_skip(file, root:)
|
|
104
|
+
relative = file.to_s.delete_prefix("#{root}/")
|
|
105
|
+
@erb_template_skips << relative.freeze unless @erb_template_skips.include?(relative)
|
|
106
|
+
return if @erb_template_note_emitted
|
|
107
|
+
|
|
108
|
+
@erb_template_note_emitted = true
|
|
109
|
+
warn("skipping ERB template files that cannot parse as Ruby " \
|
|
110
|
+
"(for example #{relative}); templates are not mutation subjects")
|
|
111
|
+
end
|
|
112
|
+
|
|
95
113
|
def count_invalid_mutation
|
|
96
114
|
@invalid_mutations += 1
|
|
97
115
|
end
|
|
@@ -15,6 +15,16 @@ module Saboteur
|
|
|
15
15
|
# whose fresh kills the spent budget left unchecked (#296).
|
|
16
16
|
attr_reader :unreached, :truncated, :unverified
|
|
17
17
|
|
|
18
|
+
# The run's peak-memory record for the JSON report (#202): each
|
|
19
|
+
# worker's kernel peak RSS and the supervisor's own, nil when the
|
|
20
|
+
# platform measured neither so the report omits the section. No total:
|
|
21
|
+
# fork copy-on-write makes per-process RSS non-additive.
|
|
22
|
+
def memory_report
|
|
23
|
+
workers = (@pool&.worker_peaks || {}).sort.map { |number, mb| { worker: number, peakMb: mb } }
|
|
24
|
+
supervisor = Execution::PeakMemory.peak_mb
|
|
25
|
+
workers.empty? && supervisor.nil? ? nil : { workers: workers, supervisorPeakMb: supervisor }.compact
|
|
26
|
+
end
|
|
27
|
+
|
|
18
28
|
# stop_after: nil, or a callable that gets one subject's results and
|
|
19
29
|
# returns true to stop before the next subject (--fail-fast). It only
|
|
20
30
|
# works on one process: parallel workers have already started the
|
|
@@ -10,12 +10,13 @@ module Saboteur
|
|
|
10
10
|
private
|
|
11
11
|
|
|
12
12
|
# The subjects a stop left unrun or mid-way: the verdict covers less than the selection.
|
|
13
|
-
def record_early_stop(distribution)
|
|
13
|
+
def record_early_stop(distribution, planned_mutants: nil)
|
|
14
14
|
return if distribution.unreached.empty? && distribution.truncated.empty? && distribution.unverified.empty?
|
|
15
15
|
|
|
16
16
|
truncated = distribution.truncated.map { |subject, omitted| { subject: subject.id, omitted: omitted } }
|
|
17
17
|
@early_stop = EarlyStop.new(reason: stop_reason(distribution), unreached: distribution.unreached.map(&:id),
|
|
18
|
-
truncated: truncated, unverified: distribution.unverified.map(&:id)
|
|
18
|
+
truncated: truncated, unverified: distribution.unverified.map(&:id),
|
|
19
|
+
planned_mutants: planned_mutants)
|
|
19
20
|
end
|
|
20
21
|
|
|
21
22
|
# Mid-subject omissions and skipped checks are budget work: only a
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
# SPDX-License-Identifier: MIT
|
|
2
|
+
# frozen_string_literal: true
|
|
3
|
+
|
|
4
|
+
module Saboteur
|
|
5
|
+
class Engine
|
|
6
|
+
# A threshold or mutant-count gate judges the score, so a path that
|
|
7
|
+
# narrows what the score covers (a partially mapped --since run, #370)
|
|
8
|
+
# must keep its hard stop instead of green-lighting the change.
|
|
9
|
+
module GatedRun
|
|
10
|
+
private
|
|
11
|
+
|
|
12
|
+
def gated?
|
|
13
|
+
config["threshold"] || Gates.new(config).any?
|
|
14
|
+
end
|
|
15
|
+
|
|
16
|
+
# Reports the guess's skipped and subjectless files, then stops or
|
|
17
|
+
# warns on unmapped files per the gate (#370).
|
|
18
|
+
def report_unmapped(guess)
|
|
19
|
+
@diagnostics.inform(guess.subjectless_message(since: config["since"])) if guess.subjectless.any?
|
|
20
|
+
inform_skipped_specs(guess)
|
|
21
|
+
stop_or_warn_unmapped(guess)
|
|
22
|
+
end
|
|
23
|
+
|
|
24
|
+
# A --since run with a partial mapping runs the mapped subset with a
|
|
25
|
+
# loud warning per unmapped file (#370); a full run, a zero guess and
|
|
26
|
+
# a gated run keep the hard stop.
|
|
27
|
+
def stop_or_warn_unmapped(guess)
|
|
28
|
+
return if guess.unmatched.empty?
|
|
29
|
+
raise ConfigError, guess.unmatched_message if !config["since"] || guess.specs.empty? || gated?
|
|
30
|
+
|
|
31
|
+
guess.unmatched.each { |file| @diagnostics.announce(guess.unmatched_warning(file)) }
|
|
32
|
+
end
|
|
33
|
+
end
|
|
34
|
+
end
|
|
35
|
+
end
|
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
# SPDX-License-Identifier: MIT
|
|
2
|
+
# frozen_string_literal: true
|
|
3
|
+
|
|
4
|
+
require "coverage"
|
|
5
|
+
|
|
6
|
+
module Saboteur
|
|
7
|
+
class Engine
|
|
8
|
+
# The boot's coverage snapshot, read in the parent because a forked
|
|
9
|
+
# child does not inherit Coverage counters: the lines and the def
|
|
10
|
+
# calls that ran while the app and spec files loaded. The executable
|
|
11
|
+
# counts from its first line, so this snapshot holds the whole boot.
|
|
12
|
+
# The lines tell ran-at-load mutants apart from uncovered ones; the
|
|
13
|
+
# def calls name def-time macros whose only caller ran here, so their
|
|
14
|
+
# mutants read as ran at load instead of surviving on the pristine
|
|
15
|
+
# closure (#409).
|
|
16
|
+
module LoadCoverage
|
|
17
|
+
# Returns [load_lines, load_methods]: file => line numbers for each.
|
|
18
|
+
def self.capture(files)
|
|
19
|
+
return [{}, {}] unless Coverage.state == :suspended
|
|
20
|
+
|
|
21
|
+
wanted = files.to_set { |file| Selection::CoverageMap.canonical(file) }
|
|
22
|
+
CoverageReads.result(stop: false, clear: false)
|
|
23
|
+
.each_with_object([{}, {}]) do |(path, data), (lines_h, methods_h)|
|
|
24
|
+
file = Selection::CoverageMap.canonical(path)
|
|
25
|
+
next unless wanted.include?(file)
|
|
26
|
+
|
|
27
|
+
lines = data[:lines].each_with_index.filter_map { |count, index| index + 1 if count&.positive? }
|
|
28
|
+
lines -= uncalled_shared_lines(data)
|
|
29
|
+
lines_h[file] = lines unless lines.empty?
|
|
30
|
+
defs = data.fetch(:methods, {}).filter_map { |(_owner, _name, first, *), count| first if count.positive? }
|
|
31
|
+
methods_h[file] = defs.uniq.sort unless defs.empty?
|
|
32
|
+
end
|
|
33
|
+
end
|
|
34
|
+
|
|
35
|
+
# A one-line or endless definition executes its def line even when its
|
|
36
|
+
# body is never called. If any sibling on that physical line is uncalled,
|
|
37
|
+
# discard its ambiguous line evidence rather than crediting every body.
|
|
38
|
+
def self.uncalled_shared_lines(data)
|
|
39
|
+
shared = data.fetch(:methods, {}).select { |(_owner, _name, first, _column, last, *), _| first == last }
|
|
40
|
+
calls = shared.group_by { |(_owner, _name, first, *), _count| first }
|
|
41
|
+
calls.filter_map { |line, entries| line if entries.any? { |_key, count| count.zero? } }
|
|
42
|
+
end
|
|
43
|
+
end
|
|
44
|
+
end
|
|
45
|
+
end
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
# SPDX-License-Identifier: MIT
|
|
2
|
+
# frozen_string_literal: true
|
|
3
|
+
|
|
4
|
+
module Saboteur
|
|
5
|
+
class Engine
|
|
6
|
+
# The --since stop whose changed source files all sit outside the
|
|
7
|
+
# include scope (#483), kept apart from the guess flow it guards.
|
|
8
|
+
module OutOfScopeGuard
|
|
9
|
+
private
|
|
10
|
+
|
|
11
|
+
# Every changed source file sits outside the include scope (#483): the
|
|
12
|
+
# whole-suite fallback would run the slowest possible suite against
|
|
13
|
+
# zero subjects, so stop and name what the scope hid.
|
|
14
|
+
def refuse_out_of_scope_changes(guess)
|
|
15
|
+
return unless guess.changed.zero? && guess.out_of_scope.any?
|
|
16
|
+
|
|
17
|
+
raise ConfigError, guess.out_of_scope_message
|
|
18
|
+
end
|
|
19
|
+
end
|
|
20
|
+
end
|
|
21
|
+
end
|
|
@@ -0,0 +1,107 @@
|
|
|
1
|
+
# SPDX-License-Identifier: MIT
|
|
2
|
+
# frozen_string_literal: true
|
|
3
|
+
|
|
4
|
+
module Saboteur
|
|
5
|
+
class Engine
|
|
6
|
+
# The two run pipelines: in-process (boot, coverage, workers) and the
|
|
7
|
+
# --test-command external mode (#140).
|
|
8
|
+
module Pipelines
|
|
9
|
+
# The in-process pipeline: boot the target, mutate under coverage, and
|
|
10
|
+
# clean the workers up however the run ends.
|
|
11
|
+
def run_pipeline(ignore_list, started)
|
|
12
|
+
return call_external(ignore_list, started) if external_command?
|
|
13
|
+
|
|
14
|
+
boot = Boot.new(config, hooks)
|
|
15
|
+
begin
|
|
16
|
+
integration, files, load_lines, load_methods = boot_target(boot)
|
|
17
|
+
@rails = boot.rails
|
|
18
|
+
subjects, scope = select_subjects(files, test_files: integration.test_files)
|
|
19
|
+
results = if subjects.empty?
|
|
20
|
+
[]
|
|
21
|
+
else
|
|
22
|
+
mutate(subjects, files, integration, scope, ignore_list,
|
|
23
|
+
load_lines:, load_methods:)
|
|
24
|
+
end
|
|
25
|
+
@diagnostics.warnings.concat(ignore_list.apply(results, unscoped_digests: @unscoped_digests))
|
|
26
|
+
build_report(results, subjects, scope, started)
|
|
27
|
+
ensure
|
|
28
|
+
# Cleanup runs however the call ends - a partway boot failure, no
|
|
29
|
+
# subjects, no runnable work - so a host embedding Engine never
|
|
30
|
+
# holds worker copies until process exit. Idempotent; the Rails
|
|
31
|
+
# profile's at_exit handler stays the fallback for abrupt exit.
|
|
32
|
+
begin
|
|
33
|
+
@coverage_guard&.resume # an embedded host gets its coverage back (#254)
|
|
34
|
+
ensure
|
|
35
|
+
safely_cleanup_workers(boot.rails)
|
|
36
|
+
end
|
|
37
|
+
end
|
|
38
|
+
end
|
|
39
|
+
|
|
40
|
+
# The --test-command pipeline (#140): no boot, no coverage, no
|
|
41
|
+
# baseline - every mutant runs the command on the mutated tree,
|
|
42
|
+
# serially, with no sandbox. The command is arbitrary user code.
|
|
43
|
+
def call_external(ignore_list, started)
|
|
44
|
+
@diagnostics.inform("external test command: #{config['test_command']} " \
|
|
45
|
+
"(serial, no coverage, unsandboxed, mutants applied in a disposable copy)")
|
|
46
|
+
files = config.included_files
|
|
47
|
+
subjects, scope = select_subjects(files)
|
|
48
|
+
results = subjects.empty? ? [] : mutate_external(subjects, scope, ignore_list)
|
|
49
|
+
@diagnostics.warnings.concat(ignore_list.apply(results, unscoped_digests: @unscoped_digests))
|
|
50
|
+
build_report(results, subjects, scope, started)
|
|
51
|
+
end
|
|
52
|
+
|
|
53
|
+
def mutate_external(subjects, scope, ignore_list)
|
|
54
|
+
runner = external_runner
|
|
55
|
+
# One neutral run before any mutant exists: a command that fails on
|
|
56
|
+
# the unchanged tree can never claim kills (#305). It answers to the
|
|
57
|
+
# run budget with an ABSOLUTE deadline, admitted after the tree copy:
|
|
58
|
+
# a budget already spent is a budget stop, not a command error.
|
|
59
|
+
begin
|
|
60
|
+
runner.preflight!(deadline: preflight_deadline, budget_deadline: budget_deadline)
|
|
61
|
+
rescue BudgetPhaseTimeout => e
|
|
62
|
+
return budget_phase_stop(e, subjects)
|
|
63
|
+
end
|
|
64
|
+
map = Selection::ExternalTestMap.new
|
|
65
|
+
scheduler = Scheduler.new(config:, runner:, map:,
|
|
66
|
+
reevaluator: Execution::Reevaluator.new(config["reeval"],
|
|
67
|
+
whole_file: config["whole_file"]),
|
|
68
|
+
diagnostics: @diagnostics, deadline: budget_deadline, verify_loaded: false)
|
|
69
|
+
lists = candidate_lists(subjects, scope, map)
|
|
70
|
+
work = subjects.zip(lists).reject { |_, list| list.empty? }
|
|
71
|
+
return [] if work.empty?
|
|
72
|
+
|
|
73
|
+
runnable_subjects, runnable_lists = work.transpose
|
|
74
|
+
begin
|
|
75
|
+
warn_unless_mutants_visible(runner, runnable_subjects.first)
|
|
76
|
+
rescue BudgetPhaseTimeout => e
|
|
77
|
+
return budget_phase_stop(e, subjects)
|
|
78
|
+
end
|
|
79
|
+
run_candidates(runnable_subjects, runnable_lists, scheduler, nil, ignore_list)
|
|
80
|
+
ensure
|
|
81
|
+
runner.close
|
|
82
|
+
end
|
|
83
|
+
|
|
84
|
+
# Warn loudly when the command cannot see mutants at all: an
|
|
85
|
+
# anchored or never-loaded file reads as 100 % survivors (#441).
|
|
86
|
+
def warn_unless_mutants_visible(runner, subject)
|
|
87
|
+
warning = runner.visibility_check(subject: subject, deadline: preflight_deadline,
|
|
88
|
+
budget_deadline: budget_deadline)
|
|
89
|
+
@diagnostics.warn(warning) if warning
|
|
90
|
+
end
|
|
91
|
+
|
|
92
|
+
def external_command? = !config["test_command"].to_s.empty?
|
|
93
|
+
|
|
94
|
+
# The preflight's absolute monotonic deadline: its own
|
|
95
|
+
# baseline_timeout slice, or the run budget's end, whichever is
|
|
96
|
+
# sooner. Absolute, so the tree copy before admission cannot eat the
|
|
97
|
+
# budget while the command still gets a full timeout (#305 Architect).
|
|
98
|
+
def preflight_deadline
|
|
99
|
+
own = Process.clock_gettime(Process::CLOCK_MONOTONIC) + config["baseline_timeout"]
|
|
100
|
+
deadline = budget_deadline
|
|
101
|
+
deadline ? [own, deadline].min : own
|
|
102
|
+
end
|
|
103
|
+
|
|
104
|
+
def external_runner = Execution::ExternalRunner.new(command: config["test_command"], root: @root)
|
|
105
|
+
end
|
|
106
|
+
end
|
|
107
|
+
end
|
|
@@ -15,25 +15,36 @@ module Saboteur
|
|
|
15
15
|
# same: the method's source, the whole file the method lives in (so a
|
|
16
16
|
# constant or helper changed elsewhere in that file is seen), the mutant,
|
|
17
17
|
# the set of covering tests, the content of those tests' files, the
|
|
18
|
-
# lockfile, the run's inputs (config, saboteur version)
|
|
19
|
-
#
|
|
18
|
+
# lockfile, the run's inputs (config, saboteur version), and every source
|
|
19
|
+
# file a covering test touched (a method it calls, a constant defined
|
|
20
|
+
# elsewhere - #197). Only killed, survived and timeout are stored;
|
|
20
21
|
# errors and skips always run again.
|
|
21
22
|
#
|
|
22
|
-
#
|
|
23
|
-
#
|
|
24
|
-
#
|
|
23
|
+
# When the dependency set is unknown - no coverage map, or an empty one -
|
|
24
|
+
# every source file is watched instead: reuse turns conservative, never
|
|
25
|
+
# stale.
|
|
25
26
|
class ResultCache
|
|
26
27
|
FORMAT = 1
|
|
27
28
|
STORED = %i[killed survived timeout].freeze
|
|
28
29
|
LOCKFILES = %w[Gemfile.lock gems.locked].freeze
|
|
30
|
+
# Dependency declarations themselves, so the no-lockfile library
|
|
31
|
+
# shape (bundler's own convention: libraries gitignore the lockfile)
|
|
32
|
+
# still invalidates on a dependency edit (#408).
|
|
33
|
+
MANIFESTS = %w[Gemfile gems.rb].freeze
|
|
34
|
+
|
|
35
|
+
SOURCE_EXCLUDES = %w[.git vendor node_modules].freeze
|
|
29
36
|
|
|
30
37
|
# suite_digest: SuiteFiles#digest, so an edit to a spec helper or
|
|
31
38
|
# support file invalidates every verdict.
|
|
32
|
-
def initialize(path:, root:, inputs:, suite_digest: "")
|
|
39
|
+
def initialize(path:, root:, inputs:, suite_digest: "", coverage: nil, fallback_files: [])
|
|
33
40
|
@path = path
|
|
34
41
|
@root = root
|
|
35
42
|
@inputs = inputs
|
|
36
43
|
@suite_digest = suite_digest
|
|
44
|
+
# Engine always passes the built map; a missing one behaves as an
|
|
45
|
+
# empty one, which watches every fallback and child-loaded source (#197).
|
|
46
|
+
@coverage = coverage || Selection::CoverageMap.new(lines: {}, tests: [])
|
|
47
|
+
@fallback_files = fallback_files.map { |file| File.expand_path(file, root) }.sort
|
|
37
48
|
@entries = load
|
|
38
49
|
@file_digests = {}
|
|
39
50
|
end
|
|
@@ -65,6 +76,16 @@ module Saboteur
|
|
|
65
76
|
|
|
66
77
|
private
|
|
67
78
|
|
|
79
|
+
# The current project source files, relative and sorted. A file added
|
|
80
|
+
# after the map was built changes the set and invalidates every stored
|
|
81
|
+
# verdict, whether or not any watched file references it yet.
|
|
82
|
+
def source_set
|
|
83
|
+
@source_set ||= Dir.glob("**/*.rb", base: @root)
|
|
84
|
+
.reject { |rel| SOURCE_EXCLUDES.include?(rel.split("/").first) }
|
|
85
|
+
.sort
|
|
86
|
+
.join("\0")
|
|
87
|
+
end
|
|
88
|
+
|
|
68
89
|
def load
|
|
69
90
|
data = JSON.parse(Text.utf8_strict(File.read(@path)))
|
|
70
91
|
data["format"] == FORMAT ? data["entries"] : {}
|
|
@@ -78,12 +99,33 @@ module Saboteur
|
|
|
78
99
|
|
|
79
100
|
def fingerprint(subject, tests)
|
|
80
101
|
digest = Digest::SHA256.new
|
|
81
|
-
digest << VERSION
|
|
102
|
+
digest << [VERSION, @inputs, @suite_digest, source_set, subject.source].join("\0") << "\0"
|
|
82
103
|
# A hex digest has a fixed width, so it needs no separator after it.
|
|
83
104
|
digest << Digest::SHA256.hexdigest(subject.file_source)
|
|
105
|
+
digest_tests(digest, tests)
|
|
106
|
+
hash_dependency_inputs(digest)
|
|
107
|
+
digest.hexdigest
|
|
108
|
+
end
|
|
109
|
+
|
|
110
|
+
def digest_tests(digest, tests)
|
|
84
111
|
tests.sort.each { |test| digest << test << "\0" << file_digest(test_file(test)) << "\0" }
|
|
112
|
+
dependencies(tests).each { |dep| digest << dep << "\0" << file_digest(dep) << "\0" }
|
|
85
113
|
LOCKFILES.each { |name| digest << file_digest(File.join(@root, name)) << "\0" }
|
|
86
|
-
|
|
114
|
+
end
|
|
115
|
+
|
|
116
|
+
# Files a change in which can flip the verdict (#197): every source
|
|
117
|
+
# file a covering test touched, from the run's coverage map, plus the
|
|
118
|
+
# loaded sources the map never observed (#344, #353).
|
|
119
|
+
def dependencies(tests)
|
|
120
|
+
(@coverage.files_for(tests) | @coverage.boot_files | unobserved_files).sort
|
|
121
|
+
end
|
|
122
|
+
|
|
123
|
+
# Loaded sources the map never observed (no line, method, or load
|
|
124
|
+
# evidence): coverage cannot say whether they matter, so a change in
|
|
125
|
+
# any of them must invalidate (#344).
|
|
126
|
+
def unobserved_files
|
|
127
|
+
@unobserved_files ||=
|
|
128
|
+
(@fallback_files | @coverage.loaded_files).select { |file| @coverage.empty_for?([file]) }
|
|
87
129
|
end
|
|
88
130
|
|
|
89
131
|
# RSpec ids name their file: "./spec/a_spec.rb[1:2]".
|
|
@@ -91,6 +133,21 @@ module Saboteur
|
|
|
91
133
|
File.expand_path(test[/\A(.+?\.rb)(?:\[|:|\z)/, 1] || test, @root)
|
|
92
134
|
end
|
|
93
135
|
|
|
136
|
+
def hash_dependency_inputs(digest)
|
|
137
|
+
dependency_files.each { |path| digest << path.delete_prefix("#{@root}/") << "\0" << file_digest(path) << "\0" }
|
|
138
|
+
end
|
|
139
|
+
|
|
140
|
+
# Manifests and gemspecs anywhere below the root, since a monorepo
|
|
141
|
+
# sub-gem carries its own dependency declarations (#561); keyed
|
|
142
|
+
# root-relative so same-named sub-gem files cannot collide. Vendored
|
|
143
|
+
# installs and dependencies' own checkouts hold gemspecs the project
|
|
144
|
+
# does not declare, so the scan mirrors RunFingerprint's exclusions.
|
|
145
|
+
def dependency_files
|
|
146
|
+
@dependency_files ||= Dir[File.join(@root, "**", "{#{MANIFESTS.join(',')},*.gemspec}")].reject do |path|
|
|
147
|
+
path.delete_prefix("#{@root}/").split("/").any? { |part| RunFingerprint::EXCLUDED_DIRECTORIES.include?(part) }
|
|
148
|
+
end
|
|
149
|
+
end
|
|
150
|
+
|
|
94
151
|
def file_digest(path)
|
|
95
152
|
@file_digests[path] ||= File.file?(path) ? Digest::SHA256.file(path).hexdigest : "missing"
|
|
96
153
|
end
|