saboteur 0.6.0 → 0.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +1024 -0
- data/README.md +199 -952
- data/exe/saboteur +34 -7
- data/lib/saboteur/baseline/workflow.rb +2 -1
- data/lib/saboteur/baseline_health.rb +40 -6
- data/lib/saboteur/cli/exit_status.rb +80 -4
- data/lib/saboteur/cli/help_options.rb +17 -0
- data/lib/saboteur/cli/history_command.rb +135 -0
- data/lib/saboteur/cli/presets.rb +40 -0
- data/lib/saboteur/cli/run_budget_options.rb +37 -0
- data/lib/saboteur/cli/run_reporting.rb +74 -5
- data/lib/saboteur/cli/selection_options.rb +31 -0
- data/lib/saboteur/cli/shard_options.rb +17 -0
- data/lib/saboteur/cli/source_set.rb +53 -0
- data/lib/saboteur/cli/survivors_command.rb +157 -0
- data/lib/saboteur/cli.rb +136 -98
- data/lib/saboteur/config.rb +163 -15
- data/lib/saboteur/config_file.rb +8 -0
- data/lib/saboteur/constant_reads.rb +182 -0
- data/lib/saboteur/constant_subject.rb +70 -0
- data/lib/saboteur/constant_subject_finder.rb +281 -0
- data/lib/saboteur/coverage_reads.rb +29 -0
- data/lib/saboteur/crash_report.rb +154 -0
- data/lib/saboteur/custom_map.rb +3 -1
- data/lib/saboteur/dirty_tree_warning.rb +128 -0
- data/lib/saboteur/doctor.rb +49 -1
- data/lib/saboteur/engine/baseline_guard.rb +136 -14
- data/lib/saboteur/engine/boot.rb +16 -1
- data/lib/saboteur/engine/canary.rb +41 -10
- data/lib/saboteur/engine/candidate_selection.rb +28 -1
- data/lib/saboteur/engine/candidates.rb +43 -3
- data/lib/saboteur/engine/coverage_guard.rb +29 -0
- data/lib/saboteur/engine/coverage_store.rb +79 -15
- data/lib/saboteur/engine/defined_order_prefix.rb +43 -0
- data/lib/saboteur/engine/diagnostics.rb +41 -2
- data/lib/saboteur/engine/diff_scope.rb +24 -5
- data/lib/saboteur/engine/distribution.rb +134 -8
- data/lib/saboteur/engine/early_stop_recording.rb +47 -0
- data/lib/saboteur/engine/gated_run.rb +35 -0
- data/lib/saboteur/engine/load_coverage.rb +45 -0
- data/lib/saboteur/engine/out_of_scope_guard.rb +21 -0
- data/lib/saboteur/engine/pipelines.rb +107 -0
- data/lib/saboteur/engine/result_cache.rb +67 -9
- data/lib/saboteur/engine/run_budget.rb +90 -0
- data/lib/saboteur/engine/run_fingerprint.rb +4 -0
- data/lib/saboteur/engine/sandbox_policy.rb +56 -11
- data/lib/saboteur/engine/scheduler.rb +222 -20
- data/lib/saboteur/engine/selection_announcement.rb +54 -0
- data/lib/saboteur/engine/shard.rb +69 -0
- data/lib/saboteur/engine/subject_selection.rb +373 -19
- data/lib/saboteur/engine/suite_files.rb +3 -1
- data/lib/saboteur/engine/test_guess.rb +438 -25
- data/lib/saboteur/engine/thread_warnings.rb +53 -0
- data/lib/saboteur/engine/unchanged_syntax.rb +17 -2
- data/lib/saboteur/engine/unscoped_suite_warning.rb +41 -0
- data/lib/saboteur/engine/verdict_check.rb +54 -18
- data/lib/saboteur/engine.rb +202 -158
- data/lib/saboteur/execution/external_runner.rb +471 -0
- data/lib/saboteur/execution/marker_sweep.rb +79 -0
- data/lib/saboteur/execution/peak_memory.rb +28 -0
- data/lib/saboteur/execution/process_table.rb +12 -0
- data/lib/saboteur/execution/process_tree.rb +65 -5
- data/lib/saboteur/execution/protocol.rb +2 -1
- data/lib/saboteur/execution/reevaluator.rb +68 -12
- data/lib/saboteur/execution/runner.rb +225 -24
- data/lib/saboteur/execution/sandbox.rb +21 -3
- data/lib/saboteur/execution/seatbelt.rb +226 -0
- data/lib/saboteur/execution/supervisor.rb +45 -13
- data/lib/saboteur/execution/worker_pool.rb +110 -8
- data/lib/saboteur/frameworks/sqlite_memory_database.rb +49 -0
- data/lib/saboteur/gates.rb +43 -2
- data/lib/saboteur/help.rb +108 -13
- data/lib/saboteur/ignore_list.rb +40 -7
- data/lib/saboteur/install.rb +4 -3
- data/lib/saboteur/integration/base.rb +7 -2
- data/lib/saboteur/integration/rspec.rb +175 -9
- data/lib/saboteur/mutation_source/argument_removal.rb +1 -1
- data/lib/saboteur/mutation_source/builtin.rb +44 -14
- data/lib/saboteur/mutation_source/chain_removal.rb +1 -1
- data/lib/saboteur/mutation_source/condition_negation.rb +1 -1
- data/lib/saboteur/mutation_source/constant_literals.rb +66 -0
- data/lib/saboteur/mutation_source/element_removal.rb +73 -0
- data/lib/saboteur/mutation_source/endpoint_replacement.rb +1 -1
- data/lib/saboteur/mutation_source/entry_edits.rb +44 -0
- data/lib/saboteur/mutation_source/filter_replacement.rb +1 -1
- data/lib/saboteur/mutation_source/hazard_guard.rb +129 -3
- data/lib/saboteur/mutation_source/interpolation_removal.rb +41 -0
- data/lib/saboteur/mutation_source/literal_replacement.rb +1 -1
- data/lib/saboteur/mutation_source/method_swap.rb +193 -0
- data/lib/saboteur/mutation_source/mutation.rb +5 -3
- data/lib/saboteur/mutation_source/operand_replacement.rb +54 -0
- data/lib/saboteur/mutation_source/operator_assignment.rb +1 -1
- data/lib/saboteur/mutation_source/operator_examples.rb +6 -1
- data/lib/saboteur/mutation_source/operator_registry.rb +7 -2
- data/lib/saboteur/mutation_source/operator_visit.rb +39 -0
- data/lib/saboteur/mutation_source/pair_removal.rb +13 -31
- data/lib/saboteur/mutation_source/range_replacement.rb +1 -1
- data/lib/saboteur/mutation_source/regex_replacement.rb +93 -0
- data/lib/saboteur/preview.rb +122 -13
- data/lib/saboteur/refinement_activations.rb +60 -0
- data/lib/saboteur/reporter/agent.rb +9 -1
- data/lib/saboteur/reporter/agent_commands.rb +47 -15
- data/lib/saboteur/reporter/agent_incomplete.rb +38 -2
- data/lib/saboteur/reporter/agent_summary.rb +4 -3
- data/lib/saboteur/reporter/findings.rb +5 -2
- data/lib/saboteur/reporter/history_policy.rb +11 -0
- data/lib/saboteur/reporter/history_reader.rb +47 -0
- data/lib/saboteur/reporter/html.rb +26 -4
- data/lib/saboteur/reporter/mutation_text.rb +6 -0
- data/lib/saboteur/reporter/next_steps.rb +14 -4
- data/lib/saboteur/reporter/score_history.rb +5 -1
- data/lib/saboteur/reporter/stryker.rb +40 -7
- data/lib/saboteur/reporter/survivor_classification.rb +101 -0
- data/lib/saboteur/reporter/survivor_hints.rb +3 -0
- data/lib/saboteur/reporter/survivor_list.rb +15 -9
- data/lib/saboteur/reporter/terminal.rb +74 -17
- data/lib/saboteur/result.rb +38 -7
- data/lib/saboteur/selection/coverage_map.rb +107 -11
- data/lib/saboteur/selection/dependency_shield.rb +80 -0
- data/lib/saboteur/selection/external_test_map.rb +27 -0
- data/lib/saboteur/selection/git_diff.rb +105 -13
- data/lib/saboteur/selection/layout_only_lines.rb +80 -0
- data/lib/saboteur/selection/recorder.rb +22 -4
- data/lib/saboteur/selection/source_scope.rb +22 -0
- data/lib/saboteur/selection/untracked_changes.rb +126 -0
- data/lib/saboteur/subject.rb +53 -7
- data/lib/saboteur/subject_finder.rb +173 -29
- data/lib/saboteur/text.rb +42 -0
- data/lib/saboteur/version.rb +1 -1
- data/lib/saboteur/working_tree.rb +38 -15
- data/lib/saboteur.rb +29 -1
- metadata +50 -2
|
@@ -15,19 +15,32 @@ module Saboteur
|
|
|
15
15
|
LOCKFILES = %w[Gemfile.lock gems.locked].freeze
|
|
16
16
|
LOG = "coverage-build.log"
|
|
17
17
|
|
|
18
|
-
def initialize(config:, integration:, hooks:, supervisor:, diagnostics:, load_lines: {}
|
|
18
|
+
def initialize(config:, integration:, hooks:, supervisor:, diagnostics:, load_lines: {}, load_methods: {},
|
|
19
|
+
timeout_cap: nil)
|
|
19
20
|
@config = config
|
|
20
21
|
@integration = integration
|
|
21
22
|
@hooks = hooks
|
|
22
23
|
@supervisor = supervisor
|
|
23
24
|
@diagnostics = diagnostics
|
|
25
|
+
@timeout_cap = timeout_cap
|
|
24
26
|
@load_lines = load_lines
|
|
27
|
+
@load_methods = load_methods
|
|
25
28
|
@root = File.realpath(config.root)
|
|
29
|
+
@root_prefixes = Selection::CoverageMap.root_prefixes(config.root)
|
|
30
|
+
end
|
|
31
|
+
|
|
32
|
+
# True when fetch would reuse the cached map instead of running the
|
|
33
|
+
# suite pass that builds it: lets the run announce the real number of
|
|
34
|
+
# full passes up front (#260). Loads the map with the same validation
|
|
35
|
+
# fetch uses and retains it, so a corrupt or truncated cache announces
|
|
36
|
+
# a build pass, not a reuse it cannot keep.
|
|
37
|
+
def reusable?(files)
|
|
38
|
+
!cached_map(files).nil?
|
|
26
39
|
end
|
|
27
40
|
|
|
28
41
|
def fetch(files)
|
|
29
|
-
path =
|
|
30
|
-
cached =
|
|
42
|
+
path = cache_path(files)
|
|
43
|
+
cached = cached_map(files)
|
|
31
44
|
return refuse_empty_map(cached, files) if cached
|
|
32
45
|
|
|
33
46
|
map = refuse_empty_map(build(files), files)
|
|
@@ -39,6 +52,31 @@ module Saboteur
|
|
|
39
52
|
|
|
40
53
|
private
|
|
41
54
|
|
|
55
|
+
def write_map(writer, recorder, build_seconds)
|
|
56
|
+
writer.write(recorder.result(build_seconds: build_seconds, loaded_files: loaded_project_files).to_json)
|
|
57
|
+
writer.close
|
|
58
|
+
end
|
|
59
|
+
|
|
60
|
+
# Every project source this child loaded, even a lazy require outside the
|
|
61
|
+
# include list that the recorder drops: --incremental must watch it, or a
|
|
62
|
+
# change there keeps stale verdicts (#353).
|
|
63
|
+
def loaded_project_files
|
|
64
|
+
$LOADED_FEATURES.filter_map do |file|
|
|
65
|
+
path = Text.utf8_strict(Selection::CoverageMap.canonical(file))
|
|
66
|
+
path if path.end_with?(".rb") && @root_prefixes.any? { |prefix| path.start_with?(prefix) }
|
|
67
|
+
end
|
|
68
|
+
end
|
|
69
|
+
|
|
70
|
+
def cache_path(files)
|
|
71
|
+
File.join(@config.path("cache_dir"), "coverage-#{cache_key(files)}.json")
|
|
72
|
+
end
|
|
73
|
+
|
|
74
|
+
def cached_map(files)
|
|
75
|
+
return nil unless @config["cache"]
|
|
76
|
+
|
|
77
|
+
@cached_map ||= Selection::CoverageMap.load(cache_path(files))
|
|
78
|
+
end
|
|
79
|
+
|
|
42
80
|
def cache_key(files)
|
|
43
81
|
inputs = JSON.generate(CACHE_INPUTS.to_h { |key| [key, @config[key]] })
|
|
44
82
|
all = files + @integration.test_files + SuiteFiles.new(@config).files + lockfiles
|
|
@@ -49,6 +87,12 @@ module Saboteur
|
|
|
49
87
|
LOCKFILES.map { |file| File.join(@root, file) }.select { |file| File.file?(file) }
|
|
50
88
|
end
|
|
51
89
|
|
|
90
|
+
# An admitted build still gets only the time left in the run budget
|
|
91
|
+
# (see BaselineGuard#phase_timeout).
|
|
92
|
+
def build_timeout
|
|
93
|
+
@timeout_cap ? @timeout_cap.call(@config["coverage_timeout"]) : @config["coverage_timeout"]
|
|
94
|
+
end
|
|
95
|
+
|
|
52
96
|
def build(files)
|
|
53
97
|
unless Coverage.respond_to?(:state) && Coverage.state == :suspended
|
|
54
98
|
raise Error, "Coverage is #{Coverage.state}; saboteur must set it up before the app loads. " \
|
|
@@ -61,13 +105,16 @@ module Saboteur
|
|
|
61
105
|
# reduce later copying of shared pages.
|
|
62
106
|
Execution::ForkReadiness.prepare_heap
|
|
63
107
|
heartbeat = @diagnostics.progress_heartbeat("building coverage map").watch(log_path)
|
|
64
|
-
|
|
108
|
+
admitted = build_timeout
|
|
109
|
+
result = @supervisor.run(timeout: admitted, on_tick: heartbeat.method(:tick),
|
|
110
|
+
label: "coverage child died during the map build") do |writer|
|
|
65
111
|
record(writer, files)
|
|
66
112
|
end
|
|
67
|
-
map_from(result)
|
|
113
|
+
map_from(result, admitted)
|
|
68
114
|
end
|
|
69
115
|
|
|
70
|
-
def map_from(result)
|
|
116
|
+
def map_from(result, admitted = @config["coverage_timeout"])
|
|
117
|
+
raise_budget_stop(admitted) if budget_stopped?(result, admitted)
|
|
71
118
|
raise Error, "coverage map build stopped: #{result.detail}" if result.stopped_by
|
|
72
119
|
raise Error, "coverage map build failed: #{died(result.exit_status)}" if result.payload.empty?
|
|
73
120
|
|
|
@@ -77,16 +124,32 @@ module Saboteur
|
|
|
77
124
|
with_parent_load_lines(Selection::CoverageMap.from_json(result.payload))
|
|
78
125
|
end
|
|
79
126
|
|
|
127
|
+
# A build that hit its deadline only because --max-duration shortened
|
|
128
|
+
# the admission below the configured phase timeout: the budget ended
|
|
129
|
+
# the run, the suite is not broken (see BaselineGuard#budget_stopped?).
|
|
130
|
+
def budget_stopped?(result, admitted)
|
|
131
|
+
result.stopped_by == :timeout && admitted < @config["coverage_timeout"]
|
|
132
|
+
end
|
|
133
|
+
|
|
134
|
+
def raise_budget_stop(admitted)
|
|
135
|
+
raise BudgetPhaseTimeout, "the --max-duration budget stopped the coverage map build " \
|
|
136
|
+
"(admitted #{admitted.round(1)}s of the #{@config['coverage_timeout']}s " \
|
|
137
|
+
"phase timeout)"
|
|
138
|
+
end
|
|
139
|
+
|
|
80
140
|
# The parent's load lines merge into the freshly built map: the forked
|
|
81
141
|
# build child cannot inherit the parent's Coverage counters.
|
|
82
142
|
def with_parent_load_lines(map)
|
|
83
|
-
@load_lines.empty? ? map : map.with_load_lines(@load_lines)
|
|
143
|
+
@load_lines.empty? && @load_methods.empty? ? map : map.with_load_lines(@load_lines, @load_methods)
|
|
84
144
|
end
|
|
85
145
|
|
|
86
146
|
# A whole-suite build that records nothing - no line hit and no method
|
|
87
|
-
# call - in any included file fails the run (exit 3)
|
|
88
|
-
#
|
|
89
|
-
#
|
|
147
|
+
# call - in any included file fails the run (exit 3). The recorder's map
|
|
148
|
+
# also holds the suite's spec files, so the check is per included file,
|
|
149
|
+
# not whole-map: a suite that tests nothing here still records its own
|
|
150
|
+
# spec code. Usually no test in the suite exercises these files, or
|
|
151
|
+
# the setup hid the app from Coverage (Bootsnap's compile cache is the
|
|
152
|
+
# default Rails case). An
|
|
90
153
|
# all-endless-def file records method calls only and passes. Either way
|
|
91
154
|
# a run on a truly empty map would test nothing, and exit 0 would lie.
|
|
92
155
|
# With explicit tests (--test) an empty map is a legitimate outcome
|
|
@@ -95,7 +158,7 @@ module Saboteur
|
|
|
95
158
|
# map too, so a map cached before this check cannot keep a run green,
|
|
96
159
|
# and the guard runs before saving, so an empty map is never cached.
|
|
97
160
|
def refuse_empty_map(map, files)
|
|
98
|
-
return map if files.empty? || !map.
|
|
161
|
+
return map if files.empty? || !map.empty_for?(files)
|
|
99
162
|
|
|
100
163
|
if @config["tests"].empty?
|
|
101
164
|
raise Error, "Coverage recorded nothing in #{files.size} included files during a whole-suite " \
|
|
@@ -133,7 +196,10 @@ module Saboteur
|
|
|
133
196
|
$stderr.reopen($stdout)
|
|
134
197
|
@hooks.run(:after_fork)
|
|
135
198
|
started = Process.clock_gettime(Process::CLOCK_MONOTONIC)
|
|
136
|
-
|
|
199
|
+
# The suite's own files are recorded too: a spec can read a
|
|
200
|
+
# constant directly, and without its line hits no test maps to the
|
|
201
|
+
# read (#319).
|
|
202
|
+
recorder = Selection::Recorder.new(files + @integration.test_files)
|
|
137
203
|
recorder.start
|
|
138
204
|
Kernel.srand(@config["seed"])
|
|
139
205
|
passed = @integration.run_all(recorder)
|
|
@@ -141,9 +207,7 @@ module Saboteur
|
|
|
141
207
|
raise Error, "the unchanged suite failed while building coverage. Fix the suite, then run again " \
|
|
142
208
|
"with seed #{@config['seed']}."
|
|
143
209
|
end
|
|
144
|
-
|
|
145
|
-
writer.write(recorder.result(build_seconds: build_seconds).to_json)
|
|
146
|
-
writer.close
|
|
210
|
+
write_map(writer, recorder, Process.clock_gettime(Process::CLOCK_MONOTONIC) - started)
|
|
147
211
|
rescue Exception => e # rubocop:disable Lint/RescueException
|
|
148
212
|
writer.write(JSON.generate(error: "#{e.class}: #{e.message}"))
|
|
149
213
|
exit!(1)
|
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
# SPDX-License-Identifier: MIT
|
|
2
|
+
# frozen_string_literal: true
|
|
3
|
+
|
|
4
|
+
module Saboteur
|
|
5
|
+
class Engine
|
|
6
|
+
# The examples an order-dependent spec file needs, in the order RSpec
|
|
7
|
+
# defines them: for each file with a covering example, every example from
|
|
8
|
+
# the start of the file through its last covering example. Later examples
|
|
9
|
+
# cannot change earlier ones, so this is the smallest run that keeps
|
|
10
|
+
# whatever state the earlier examples leave behind.
|
|
11
|
+
class DefinedOrderPrefix
|
|
12
|
+
ID_PATTERN = /\A(?<file>.*)\[(?<path>[\d:]+)\]\z/
|
|
13
|
+
|
|
14
|
+
# every_test: all example ids the map knows, in any order.
|
|
15
|
+
def initialize(every_test)
|
|
16
|
+
@every_test = every_test
|
|
17
|
+
end
|
|
18
|
+
|
|
19
|
+
# The prefix for the covering examples; nil when it adds nothing to
|
|
20
|
+
# what already ran (an id that is not an RSpec id included).
|
|
21
|
+
def for(covering)
|
|
22
|
+
parsed = covering.map { |id| parse(id) }
|
|
23
|
+
return nil if parsed.any?(&:nil?)
|
|
24
|
+
|
|
25
|
+
prefix = parsed.group_by(&:first).flat_map { |file, entries| file_prefix(file, entries.map(&:last).max) }
|
|
26
|
+
prefix == covering ? nil : prefix
|
|
27
|
+
end
|
|
28
|
+
|
|
29
|
+
private
|
|
30
|
+
|
|
31
|
+
def file_prefix(file, last_position)
|
|
32
|
+
in_file = @every_test.filter_map { |id| (entry = parse(id)) && entry.first == file ? entry : nil }
|
|
33
|
+
through = in_file.select { |entry| (entry.last <=> last_position) <= 0 }.sort_by(&:last)
|
|
34
|
+
through.map { |entry| "#{file}[#{entry.last.join(':')}]" }
|
|
35
|
+
end
|
|
36
|
+
|
|
37
|
+
def parse(id)
|
|
38
|
+
match = ID_PATTERN.match(id)
|
|
39
|
+
match && [match[:file], match[:path].split(":").map(&:to_i)]
|
|
40
|
+
end
|
|
41
|
+
end
|
|
42
|
+
end
|
|
43
|
+
end
|
|
@@ -76,8 +76,9 @@ module Saboteur
|
|
|
76
76
|
attr_reader :warnings, :invalid_mutations, :generation_events, :omissions, :discovery_failures,
|
|
77
77
|
:hazard_suppressions, :inline_suppressions
|
|
78
78
|
|
|
79
|
-
def initialize(err)
|
|
79
|
+
def initialize(err, verbose: false)
|
|
80
80
|
@err = err
|
|
81
|
+
@verbose = verbose
|
|
81
82
|
@warnings = []
|
|
82
83
|
@invalid_mutations = 0
|
|
83
84
|
@generation_events = []
|
|
@@ -85,12 +86,30 @@ module Saboteur
|
|
|
85
86
|
@discovery_failures = []
|
|
86
87
|
@hazard_suppressions = 0
|
|
87
88
|
@inline_suppressions = []
|
|
89
|
+
@erb_template_skips = []
|
|
90
|
+
@erb_template_note_emitted = false
|
|
88
91
|
end
|
|
89
92
|
|
|
90
93
|
def warn(message)
|
|
91
94
|
@warnings << message
|
|
92
95
|
end
|
|
93
96
|
|
|
97
|
+
# ERB generator templates (Rails generators carry `.rb` files with ERB
|
|
98
|
+
# interpolation, conventionally under `lib/generators/**/templates/`)
|
|
99
|
+
# can never parse as plain Ruby, so they can never hold mutation
|
|
100
|
+
# subjects (#372). One summarized note per run replaces the per-file
|
|
101
|
+
# parse warning every discovery pass used to print; a real syntax
|
|
102
|
+
# error in a plain Ruby file still warns per file.
|
|
103
|
+
def record_erb_template_skip(file, root:)
|
|
104
|
+
relative = file.to_s.delete_prefix("#{root}/")
|
|
105
|
+
@erb_template_skips << relative.freeze unless @erb_template_skips.include?(relative)
|
|
106
|
+
return if @erb_template_note_emitted
|
|
107
|
+
|
|
108
|
+
@erb_template_note_emitted = true
|
|
109
|
+
warn("skipping ERB template files that cannot parse as Ruby " \
|
|
110
|
+
"(for example #{relative}); templates are not mutation subjects")
|
|
111
|
+
end
|
|
112
|
+
|
|
94
113
|
def count_invalid_mutation
|
|
95
114
|
@invalid_mutations += 1
|
|
96
115
|
end
|
|
@@ -102,8 +121,11 @@ module Saboteur
|
|
|
102
121
|
@generation_events << GenerationEvent.new(*values)
|
|
103
122
|
end
|
|
104
123
|
|
|
124
|
+
# The selection and the test inference each walk the same changed
|
|
125
|
+
# files in one run; a failure either of them finds is recorded once.
|
|
105
126
|
def record_discovery_failure(file, reason:)
|
|
106
|
-
|
|
127
|
+
failure = DiscoveryFailure.new(file: file.to_s.dup.freeze, reason: reason.to_s.dup.freeze)
|
|
128
|
+
@discovery_failures << failure unless @discovery_failures.include?(failure)
|
|
107
129
|
end
|
|
108
130
|
|
|
109
131
|
# A mutant the hazard guard refused to generate (issue #102): counted
|
|
@@ -129,6 +151,23 @@ module Saboteur
|
|
|
129
151
|
@err.puts("saboteur: #{message}")
|
|
130
152
|
end
|
|
131
153
|
|
|
154
|
+
# --verbose: one line per finished subject with its result marks, so a
|
|
155
|
+
# survivor is attributable to a method as soon as the subject
|
|
156
|
+
# completes, before the whole run ends (#262).
|
|
157
|
+
def subject_progress(subject, results)
|
|
158
|
+
return unless @verbose
|
|
159
|
+
return if results.empty?
|
|
160
|
+
|
|
161
|
+
marks = results.map { |result| MARKS.fetch(result.status, "?") }.join
|
|
162
|
+
inform("#{subject.id} #{marks}")
|
|
163
|
+
end
|
|
164
|
+
|
|
165
|
+
# A message that is both a report warning and a stderr note.
|
|
166
|
+
def announce(message)
|
|
167
|
+
warn(message)
|
|
168
|
+
inform(message)
|
|
169
|
+
end
|
|
170
|
+
|
|
132
171
|
# A heartbeat for a long phase: the supervisor's wait loop calls tick
|
|
133
172
|
# in the parent after the fork, so no extra thread is ever alive at
|
|
134
173
|
# fork time. Prints "<label>: N s elapsed" about every interval
|
|
@@ -1,6 +1,8 @@
|
|
|
1
1
|
# SPDX-License-Identifier: MIT
|
|
2
2
|
# frozen_string_literal: true
|
|
3
3
|
|
|
4
|
+
require_relative "../text"
|
|
5
|
+
|
|
4
6
|
module Saboteur
|
|
5
7
|
class Engine
|
|
6
8
|
# Review-shaped diff mode (--since). A reviewer asks "do the tests check
|
|
@@ -14,7 +16,7 @@ module Saboteur
|
|
|
14
16
|
end
|
|
15
17
|
|
|
16
18
|
def restrict(subject, candidates)
|
|
17
|
-
changed =
|
|
19
|
+
changed = lines_for(subject)
|
|
18
20
|
candidates.select do |mutation, _|
|
|
19
21
|
mutation.coarse || mutation.edits.any? { |edit| edited_lines(subject, edit).any? { changed.include?(_1) } }
|
|
20
22
|
end
|
|
@@ -24,22 +26,39 @@ module Saboteur
|
|
|
24
26
|
# several lines is mutated as one piece and reported at its first line,
|
|
25
27
|
# which can sit above the line that changed; this names the changed line.
|
|
26
28
|
def changed_lines_for(subject, mutation)
|
|
27
|
-
changed =
|
|
29
|
+
changed = lines_for(subject)
|
|
28
30
|
mutation.edits.flat_map { |edit| edited_lines(subject, edit).to_a }.uniq.select { changed.include?(_1) }.sort
|
|
29
31
|
end
|
|
30
32
|
|
|
31
33
|
# Distance to the nearest changed line in this file. Unknown outside
|
|
32
|
-
# the diff is nil, not a made-up nearby location.
|
|
34
|
+
# the diff is nil, not a made-up nearby location. A range answers
|
|
35
|
+
# geometrically - 0 when the edit overlaps, else the gap to the nearer
|
|
36
|
+
# endpoint - so a whole-file range never materializes one entry per
|
|
37
|
+
# line.
|
|
33
38
|
def distance_for(subject, mutation)
|
|
34
|
-
changed =
|
|
39
|
+
changed = lines_for(subject)
|
|
35
40
|
mutation.edits.map do |edit|
|
|
36
41
|
first, last = edited_lines(subject, edit).minmax
|
|
37
|
-
changed.
|
|
42
|
+
if changed.is_a?(Range)
|
|
43
|
+
if last < changed.begin then changed.begin - last
|
|
44
|
+
elsif first > changed.end then first - changed.end
|
|
45
|
+
else 0
|
|
46
|
+
end
|
|
47
|
+
else
|
|
48
|
+
changed.map { |line| (line - line.clamp(first, last)).abs }.min
|
|
49
|
+
end
|
|
38
50
|
end.min
|
|
39
51
|
end
|
|
40
52
|
|
|
41
53
|
private
|
|
42
54
|
|
|
55
|
+
# subject.file carries the locale tag under a non-UTF-8 locale
|
|
56
|
+
# while the map keys are UTF-8; retag losslessly or a non-ASCII
|
|
57
|
+
# path never matches and every fine mutant is silently dropped.
|
|
58
|
+
def lines_for(subject)
|
|
59
|
+
@changed_lines.fetch(Text.utf8_identity(subject.file), Set.new)
|
|
60
|
+
end
|
|
61
|
+
|
|
43
62
|
def edited_lines(subject, edit)
|
|
44
63
|
first = subject.line_of(edit.start_byte)
|
|
45
64
|
last = subject.line_of([edit.end_byte - 1, edit.start_byte].max)
|
|
@@ -9,23 +9,41 @@ module Saboteur
|
|
|
9
9
|
# candidate list, so results look the same either way.
|
|
10
10
|
class Distribution
|
|
11
11
|
# After run: the subjects a stop_after callable left unrun (always empty
|
|
12
|
-
# without one)
|
|
13
|
-
|
|
12
|
+
# without one), and the subjects that started but stopped mid-list as
|
|
13
|
+
# [subject, omitted_candidate_count] pairs (a spent --max-duration
|
|
14
|
+
# breaking a subject's scheduler loop). unverified lists the subjects
|
|
15
|
+
# whose fresh kills the spent budget left unchecked (#296).
|
|
16
|
+
attr_reader :unreached, :truncated, :unverified
|
|
17
|
+
|
|
18
|
+
# The run's peak-memory record for the JSON report (#202): each
|
|
19
|
+
# worker's kernel peak RSS and the supervisor's own, nil when the
|
|
20
|
+
# platform measured neither so the report omits the section. No total:
|
|
21
|
+
# fork copy-on-write makes per-process RSS non-additive.
|
|
22
|
+
def memory_report
|
|
23
|
+
workers = (@pool&.worker_peaks || {}).sort.map { |number, mb| { worker: number, peakMb: mb } }
|
|
24
|
+
supervisor = Execution::PeakMemory.peak_mb
|
|
25
|
+
workers.empty? && supervisor.nil? ? nil : { workers: workers, supervisorPeakMb: supervisor }.compact
|
|
26
|
+
end
|
|
14
27
|
|
|
15
28
|
# stop_after: nil, or a callable that gets one subject's results and
|
|
16
29
|
# returns true to stop before the next subject (--fail-fast). It only
|
|
17
30
|
# works on one process: parallel workers have already started the
|
|
18
31
|
# other subjects.
|
|
19
32
|
def initialize(jobs:, scheduler:, diagnostics:, cache:, on_start: nil, after_subject: nil,
|
|
20
|
-
stop_after: nil
|
|
33
|
+
stop_after: nil, halt_check: nil,
|
|
34
|
+
drain_timeout: 30) # seconds; two minimum-strength mutant deadlines at default settings
|
|
21
35
|
@jobs = jobs
|
|
22
36
|
@stop_after = stop_after
|
|
37
|
+
@halt_check = halt_check
|
|
23
38
|
@unreached = []
|
|
39
|
+
@truncated = []
|
|
40
|
+
@unverified = []
|
|
24
41
|
@on_start = on_start
|
|
25
42
|
@scheduler = scheduler
|
|
26
43
|
@diagnostics = diagnostics
|
|
27
44
|
@cache = cache
|
|
28
45
|
@after_subject = after_subject
|
|
46
|
+
@drain_timeout = drain_timeout
|
|
29
47
|
end
|
|
30
48
|
|
|
31
49
|
# lists: each subject's [[mutation, text]] candidates, in subject order.
|
|
@@ -35,23 +53,116 @@ module Saboteur
|
|
|
35
53
|
# The watcher runs as each subject's row arrives, before the pool
|
|
36
54
|
# hands out more work, so a tripped canary stops the queue (jobs>1
|
|
37
55
|
# used to check only after the whole batch returned, issue #102).
|
|
38
|
-
|
|
39
|
-
|
|
56
|
+
# A tool error no longer raises on sight: one wiped log dir hits
|
|
57
|
+
# every worker, and a bystander whose current job ran no mutant
|
|
58
|
+
# reports "an unknown mutant", which could land before the report
|
|
59
|
+
# of the worker whose mutant did the damage (macOS CI on #285).
|
|
60
|
+
# The pool is halted instead - no new work, and the drain is
|
|
61
|
+
# bounded by drain_timeout so a wedged worker cannot stall the
|
|
62
|
+
# abort - and the abort raises the report with the best
|
|
63
|
+
# attribution.
|
|
64
|
+
# The abort reports and the pool are run state: a Distribution
|
|
65
|
+
# runs once. @pool is only read from arrivals, which fire inside
|
|
66
|
+
# @pool.map after the assignment.
|
|
67
|
+
# A budget already spent by boot, baselines or the coverage map
|
|
68
|
+
# launches nothing: every subject comes back unreached (#245).
|
|
69
|
+
if @halt_check&.call
|
|
70
|
+
@unreached = subjects
|
|
71
|
+
return []
|
|
40
72
|
end
|
|
41
|
-
|
|
42
|
-
|
|
73
|
+
|
|
74
|
+
@abort_reports = []
|
|
75
|
+
arrival = arrival_for(subjects, lists)
|
|
76
|
+
@pool = Execution::WorkerPool.new(jobs: @jobs, on_start: @on_start, on_result: arrival,
|
|
77
|
+
drain_timeout: @drain_timeout)
|
|
78
|
+
rows = @pool.map(subjects.each_index.to_a, weights: lists.map(&:size)) do |i|
|
|
43
79
|
work(subjects[i], lists[i])
|
|
44
80
|
end
|
|
45
|
-
|
|
81
|
+
raise_preferred_tool_error(@abort_reports) if @abort_reports.any?
|
|
82
|
+
record_unreached(rows, subjects, lists)
|
|
83
|
+
rows.each_with_index.flat_map do |row, i|
|
|
84
|
+
next [] if row.nil?
|
|
85
|
+
|
|
86
|
+
raise_if_tool_error(row)
|
|
87
|
+
rebuild(row, subjects[i], lists[i])
|
|
88
|
+
end
|
|
46
89
|
end
|
|
47
90
|
|
|
48
91
|
private
|
|
49
92
|
|
|
93
|
+
# @pool is only read here from arrivals, which fire inside @pool.map
|
|
94
|
+
# after the assignment. A spent --max-duration budget stops the queue
|
|
95
|
+
# without the safety-abort drain: in-flight subjects finish and
|
|
96
|
+
# report, and only never-started ones come back as nil holes, so
|
|
97
|
+
# unreached never mislabels an interrupted subject as unrun (#245).
|
|
98
|
+
def arrival_for(subjects, lists)
|
|
99
|
+
lambda do |i, row|
|
|
100
|
+
if row["tool_error"]
|
|
101
|
+
collect_abort_report(row)
|
|
102
|
+
else
|
|
103
|
+
check_subject(subjects, lists, i, row)
|
|
104
|
+
@pool.halt!(drain: false) if @halt_check&.call
|
|
105
|
+
end
|
|
106
|
+
end
|
|
107
|
+
end
|
|
108
|
+
|
|
109
|
+
# nil rows never started; a short results list is a subject whose
|
|
110
|
+
# scheduler stopped mid-list at the budget deadline. Worker-level error
|
|
111
|
+
# rows synthesize an error per candidate, so they never look truncated.
|
|
112
|
+
def unverified_result?(result)
|
|
113
|
+
result.is_a?(MutantResult) && result.status == :unverified
|
|
114
|
+
end
|
|
115
|
+
|
|
116
|
+
def record_unreached(rows, subjects, lists)
|
|
117
|
+
@unreached = []
|
|
118
|
+
@truncated = []
|
|
119
|
+
@unverified = []
|
|
120
|
+
rows.each_with_index do |row, i|
|
|
121
|
+
if row.nil?
|
|
122
|
+
@unreached << subjects[i]
|
|
123
|
+
next
|
|
124
|
+
end
|
|
125
|
+
next unless row["results"] && !row["error"]
|
|
126
|
+
|
|
127
|
+
@truncated << [subjects[i], lists[i].size - row["results"].size] if row["results"].size < lists[i].size
|
|
128
|
+
@unverified << subjects[i] if row["results"].any? { |result| result["status"] == "unverified" }
|
|
129
|
+
end
|
|
130
|
+
end
|
|
131
|
+
|
|
132
|
+
# @pool is always assigned when an arrival fires: arrivals only fire
|
|
133
|
+
# from inside @pool.map, which runs after the assignment.
|
|
134
|
+
def collect_abort_report(row)
|
|
135
|
+
@abort_reports << row
|
|
136
|
+
@pool.halt!
|
|
137
|
+
end
|
|
138
|
+
|
|
139
|
+
# A canary trip from the end-of-subject check folds into the same
|
|
140
|
+
# record + halt + preferred-report path as a worker's own report:
|
|
141
|
+
# escaping the pool here would override a collected report that
|
|
142
|
+
# names the real culprit (#285 review).
|
|
143
|
+
def check_subject(subjects, lists, index, row)
|
|
144
|
+
@after_subject&.call(subjects[index], decode(row, subjects[index], lists[index]))
|
|
145
|
+
rescue Error => e
|
|
146
|
+
collect_abort_report(Execution::WorkerPool.tool_error_row(e))
|
|
147
|
+
end
|
|
148
|
+
|
|
50
149
|
def run_serially(subjects, lists)
|
|
51
150
|
results = []
|
|
52
151
|
subjects.zip(lists).each_with_index do |(subject, list), index|
|
|
152
|
+
# Same pre-dispatch budget check as the parallel first wave: an
|
|
153
|
+
# already-spent baseline starts no further subjects (#245).
|
|
154
|
+
if @halt_check&.call
|
|
155
|
+
@unreached = subjects.drop(index)
|
|
156
|
+
break
|
|
157
|
+
end
|
|
158
|
+
|
|
53
159
|
subject_results = @scheduler.call(subject, list)
|
|
54
160
|
results.concat(subject_results)
|
|
161
|
+
# A scheduler that stopped mid-list at the budget deadline: the
|
|
162
|
+
# completed results stay, the omitted candidates are accounted.
|
|
163
|
+
omitted = list.size - subject_results.size
|
|
164
|
+
@truncated << [subject, omitted] if omitted.positive?
|
|
165
|
+
@unverified << subject if subject_results.any? { |result| unverified_result?(result) }
|
|
55
166
|
@after_subject&.call(subject, subject_results)
|
|
56
167
|
next unless @stop_after&.call(subject_results)
|
|
57
168
|
|
|
@@ -91,6 +202,21 @@ module Saboteur
|
|
|
91
202
|
|
|
92
203
|
# Turns a worker row back into results, without diagnostics or caching:
|
|
93
204
|
# used both at result arrival (the canary) and at rebuild time.
|
|
205
|
+
# A tool error from the worker (a vanished cache dir, ...) aborts
|
|
206
|
+
# the whole run with exit 3; it is not a per-mutant outcome (#283).
|
|
207
|
+
def raise_if_tool_error(row)
|
|
208
|
+
raise Error, row["tool_error"] if row["tool_error"]
|
|
209
|
+
end
|
|
210
|
+
|
|
211
|
+
# The abort names a culprit when any report can, ranked by
|
|
212
|
+
# attribution quality (#285 review): a report carrying its mutant's
|
|
213
|
+
# per-mutant digest first, then a canary trip (the hazard report
|
|
214
|
+
# still beats a bare report, #283), then the first bare report.
|
|
215
|
+
def raise_preferred_tool_error(rows)
|
|
216
|
+
row = rows.find { |r| r["culprit"] } || rows.find { |r| r["canary"] } || rows.first
|
|
217
|
+
raise Error, row["tool_error"]
|
|
218
|
+
end
|
|
219
|
+
|
|
94
220
|
def decode(row, subject, list)
|
|
95
221
|
return failed(row["error"], subject, list) if row["error"]
|
|
96
222
|
|
|
@@ -0,0 +1,47 @@
|
|
|
1
|
+
# SPDX-License-Identifier: MIT
|
|
2
|
+
# frozen_string_literal: true
|
|
3
|
+
|
|
4
|
+
module Saboteur
|
|
5
|
+
class Engine
|
|
6
|
+
# Records the subjects an early stop left behind - whole subjects that
|
|
7
|
+
# never started and candidates skipped mid-way through a subject - so
|
|
8
|
+
# the report says the verdict covers less than the selection.
|
|
9
|
+
module EarlyStopRecording
|
|
10
|
+
private
|
|
11
|
+
|
|
12
|
+
# The subjects a stop left unrun or mid-way: the verdict covers less than the selection.
|
|
13
|
+
def record_early_stop(distribution, planned_mutants: nil)
|
|
14
|
+
return if distribution.unreached.empty? && distribution.truncated.empty? && distribution.unverified.empty?
|
|
15
|
+
|
|
16
|
+
truncated = distribution.truncated.map { |subject, omitted| { subject: subject.id, omitted: omitted } }
|
|
17
|
+
@early_stop = EarlyStop.new(reason: stop_reason(distribution), unreached: distribution.unreached.map(&:id),
|
|
18
|
+
truncated: truncated, unverified: distribution.unverified.map(&:id),
|
|
19
|
+
planned_mutants: planned_mutants)
|
|
20
|
+
end
|
|
21
|
+
|
|
22
|
+
# Mid-subject omissions and skipped checks are budget work: only a
|
|
23
|
+
# stop that left whole subjects unreached can be fail_fast's.
|
|
24
|
+
def stop_reason(distribution)
|
|
25
|
+
return "max_duration" if distribution.truncated.any? || distribution.unverified.any?
|
|
26
|
+
|
|
27
|
+
run_early_stop_reason
|
|
28
|
+
end
|
|
29
|
+
|
|
30
|
+
# max_duration was announced before fail_fast could stop the run: a
|
|
31
|
+
# budget stop overrides a failure stop announced later.
|
|
32
|
+
def run_early_stop_reason
|
|
33
|
+
@budget_announced ? "max_duration" : "fail_fast"
|
|
34
|
+
end
|
|
35
|
+
|
|
36
|
+
# A baseline or coverage phase stopped because --max-duration ate its
|
|
37
|
+
# allowance: budget exhaustion, not a broken suite. The run reports
|
|
38
|
+
# partial with every selected subject unreached (exit 4) (#296).
|
|
39
|
+
def budget_phase_stop(error, subjects)
|
|
40
|
+
@diagnostics.warn(error.message)
|
|
41
|
+
@early_stop = EarlyStop.new(reason: "max_duration", unreached: subjects.map(&:id), truncated: [],
|
|
42
|
+
unverified: [])
|
|
43
|
+
[]
|
|
44
|
+
end
|
|
45
|
+
end
|
|
46
|
+
end
|
|
47
|
+
end
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
# SPDX-License-Identifier: MIT
|
|
2
|
+
# frozen_string_literal: true
|
|
3
|
+
|
|
4
|
+
module Saboteur
|
|
5
|
+
class Engine
|
|
6
|
+
# A threshold or mutant-count gate judges the score, so a path that
|
|
7
|
+
# narrows what the score covers (a partially mapped --since run, #370)
|
|
8
|
+
# must keep its hard stop instead of green-lighting the change.
|
|
9
|
+
module GatedRun
|
|
10
|
+
private
|
|
11
|
+
|
|
12
|
+
def gated?
|
|
13
|
+
config["threshold"] || Gates.new(config).any?
|
|
14
|
+
end
|
|
15
|
+
|
|
16
|
+
# Reports the guess's skipped and subjectless files, then stops or
|
|
17
|
+
# warns on unmapped files per the gate (#370).
|
|
18
|
+
def report_unmapped(guess)
|
|
19
|
+
@diagnostics.inform(guess.subjectless_message(since: config["since"])) if guess.subjectless.any?
|
|
20
|
+
inform_skipped_specs(guess)
|
|
21
|
+
stop_or_warn_unmapped(guess)
|
|
22
|
+
end
|
|
23
|
+
|
|
24
|
+
# A --since run with a partial mapping runs the mapped subset with a
|
|
25
|
+
# loud warning per unmapped file (#370); a full run, a zero guess and
|
|
26
|
+
# a gated run keep the hard stop.
|
|
27
|
+
def stop_or_warn_unmapped(guess)
|
|
28
|
+
return if guess.unmatched.empty?
|
|
29
|
+
raise ConfigError, guess.unmatched_message if !config["since"] || guess.specs.empty? || gated?
|
|
30
|
+
|
|
31
|
+
guess.unmatched.each { |file| @diagnostics.announce(guess.unmatched_warning(file)) }
|
|
32
|
+
end
|
|
33
|
+
end
|
|
34
|
+
end
|
|
35
|
+
end
|
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
# SPDX-License-Identifier: MIT
|
|
2
|
+
# frozen_string_literal: true
|
|
3
|
+
|
|
4
|
+
require "coverage"
|
|
5
|
+
|
|
6
|
+
module Saboteur
|
|
7
|
+
class Engine
|
|
8
|
+
# The boot's coverage snapshot, read in the parent because a forked
|
|
9
|
+
# child does not inherit Coverage counters: the lines and the def
|
|
10
|
+
# calls that ran while the app and spec files loaded. The executable
|
|
11
|
+
# counts from its first line, so this snapshot holds the whole boot.
|
|
12
|
+
# The lines tell ran-at-load mutants apart from uncovered ones; the
|
|
13
|
+
# def calls name def-time macros whose only caller ran here, so their
|
|
14
|
+
# mutants read as ran at load instead of surviving on the pristine
|
|
15
|
+
# closure (#409).
|
|
16
|
+
module LoadCoverage
|
|
17
|
+
# Returns [load_lines, load_methods]: file => line numbers for each.
|
|
18
|
+
def self.capture(files)
|
|
19
|
+
return [{}, {}] unless Coverage.state == :suspended
|
|
20
|
+
|
|
21
|
+
wanted = files.to_set { |file| Selection::CoverageMap.canonical(file) }
|
|
22
|
+
CoverageReads.result(stop: false, clear: false)
|
|
23
|
+
.each_with_object([{}, {}]) do |(path, data), (lines_h, methods_h)|
|
|
24
|
+
file = Selection::CoverageMap.canonical(path)
|
|
25
|
+
next unless wanted.include?(file)
|
|
26
|
+
|
|
27
|
+
lines = data[:lines].each_with_index.filter_map { |count, index| index + 1 if count&.positive? }
|
|
28
|
+
lines -= uncalled_shared_lines(data)
|
|
29
|
+
lines_h[file] = lines unless lines.empty?
|
|
30
|
+
defs = data.fetch(:methods, {}).filter_map { |(_owner, _name, first, *), count| first if count.positive? }
|
|
31
|
+
methods_h[file] = defs.uniq.sort unless defs.empty?
|
|
32
|
+
end
|
|
33
|
+
end
|
|
34
|
+
|
|
35
|
+
# A one-line or endless definition executes its def line even when its
|
|
36
|
+
# body is never called. If any sibling on that physical line is uncalled,
|
|
37
|
+
# discard its ambiguous line evidence rather than crediting every body.
|
|
38
|
+
def self.uncalled_shared_lines(data)
|
|
39
|
+
shared = data.fetch(:methods, {}).select { |(_owner, _name, first, _column, last, *), _| first == last }
|
|
40
|
+
calls = shared.group_by { |(_owner, _name, first, *), _count| first }
|
|
41
|
+
calls.filter_map { |line, entries| line if entries.any? { |_key, count| count.zero? } }
|
|
42
|
+
end
|
|
43
|
+
end
|
|
44
|
+
end
|
|
45
|
+
end
|