saboteur 0.8.0 → 0.9.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +215 -0
- data/README.md +44 -6
- data/exe/saboteur +6 -2
- data/lib/saboteur/attribute_finder.rb +116 -0
- data/lib/saboteur/baseline/run_evidence.rb +2 -1
- data/lib/saboteur/cli/agent_provenance.rb +59 -0
- data/lib/saboteur/cli/dirty_tree_notice.rb +28 -0
- data/lib/saboteur/cli/doctor_command.rb +41 -0
- data/lib/saboteur/cli/error_cause.rb +40 -0
- data/lib/saboteur/cli/execution_options.rb +46 -0
- data/lib/saboteur/cli/exit_status.rb +12 -14
- data/lib/saboteur/cli/explicit_file_options.rb +21 -0
- data/lib/saboteur/cli/explicit_sources.rb +46 -0
- data/lib/saboteur/cli/failure_output.rb +35 -0
- data/lib/saboteur/cli/help_command.rb +23 -0
- data/lib/saboteur/cli/matrix_option.rb +17 -0
- data/lib/saboteur/cli/output_options.rb +27 -0
- data/lib/saboteur/cli/presets.rb +41 -1
- data/lib/saboteur/cli/preview_command.rb +26 -0
- data/lib/saboteur/cli/run_budget_options.rb +6 -1
- data/lib/saboteur/cli/run_command.rb +33 -0
- data/lib/saboteur/cli/run_output.rb +38 -0
- data/lib/saboteur/cli/run_reporting.rb +6 -5
- data/lib/saboteur/cli/run_validation.rb +54 -0
- data/lib/saboteur/cli/source_set.rb +0 -18
- data/lib/saboteur/cli.rb +29 -305
- data/lib/saboteur/config.rb +76 -7
- data/lib/saboteur/config_file.rb +12 -8
- data/lib/saboteur/constant_subject_finder.rb +3 -0
- data/lib/saboteur/custom_map.rb +24 -5
- data/lib/saboteur/deep_source.rb +49 -0
- data/lib/saboteur/dirty_tree_warning.rb +6 -4
- data/lib/saboteur/doctor.rb +10 -0
- data/lib/saboteur/engine/baseline_guard.rb +118 -15
- data/lib/saboteur/engine/baseline_parts.rb +51 -0
- data/lib/saboteur/engine/boot.rb +11 -3
- data/lib/saboteur/engine/bytecode_equivalence.rb +17 -1
- data/lib/saboteur/engine/candidate_selection.rb +16 -5
- data/lib/saboteur/engine/candidates.rb +4 -1
- data/lib/saboteur/engine/coverage_inputs_snapshot.rb +146 -0
- data/lib/saboteur/engine/coverage_miss_note.rb +86 -0
- data/lib/saboteur/engine/coverage_partial.rb +48 -0
- data/lib/saboteur/engine/coverage_parts.rb +50 -0
- data/lib/saboteur/engine/coverage_shadow.rb +92 -0
- data/lib/saboteur/engine/coverage_store.rb +296 -21
- data/lib/saboteur/engine/diagnostics.rb +24 -7
- data/lib/saboteur/engine/distribution.rb +2 -2
- data/lib/saboteur/engine/gated_run.rb +4 -1
- data/lib/saboteur/engine/load_coverage.rb +72 -8
- data/lib/saboteur/engine/partial_invalidation.rb +115 -0
- data/lib/saboteur/engine/partial_merge.rb +40 -0
- data/lib/saboteur/engine/pipelines.rb +11 -3
- data/lib/saboteur/engine/reference_specs.rb +71 -0
- data/lib/saboteur/engine/result_cache.rb +68 -5
- data/lib/saboteur/engine/resume_manifest.rb +110 -0
- data/lib/saboteur/engine/run_budget.rb +10 -7
- data/lib/saboteur/engine/run_fingerprint.rb +2 -1
- data/lib/saboteur/engine/sandbox_policy.rb +26 -5
- data/lib/saboteur/engine/scheduler.rb +33 -25
- data/lib/saboteur/engine/selection_announcement.rb +9 -3
- data/lib/saboteur/engine/shadow_comparison.rb +121 -0
- data/lib/saboteur/engine/shadow_run.rb +38 -0
- data/lib/saboteur/engine/subject_selection.rb +144 -64
- data/lib/saboteur/engine/suite_partition.rb +22 -0
- data/lib/saboteur/engine/test_guess.rb +186 -32
- data/lib/saboteur/engine/unchanged_syntax.rb +10 -2
- data/lib/saboteur/engine.rb +44 -11
- data/lib/saboteur/environment_hints.rb +55 -0
- data/lib/saboteur/execution/bundle_gemfile_pin.rb +72 -0
- data/lib/saboteur/execution/external_runner.rb +123 -1
- data/lib/saboteur/execution/fiddle_loader.rb +34 -0
- data/lib/saboteur/execution/gemfile_path_sources.rb +166 -0
- data/lib/saboteur/execution/marker_sweep.rb +2 -1
- data/lib/saboteur/execution/process_table.rb +7 -3
- data/lib/saboteur/execution/reevaluator.rb +7 -0
- data/lib/saboteur/execution/sandbox.rb +39 -20
- data/lib/saboteur/execution/seatbelt.rb +10 -3
- data/lib/saboteur/execution/supervisor.rb +42 -8
- data/lib/saboteur/execution/supervisor_batch.rb +105 -0
- data/lib/saboteur/execution/test_command_shape.rb +149 -0
- data/lib/saboteur/execution/worker_pool.rb +48 -4
- data/lib/saboteur/frameworks/rails.rb +2 -5
- data/lib/saboteur/frameworks/sqlite_memory_database.rb +32 -2
- data/lib/saboteur/gates.rb +121 -28
- data/lib/saboteur/install.rb +6 -1
- data/lib/saboteur/integration/base.rb +19 -2
- data/lib/saboteur/integration/rspec.rb +145 -18
- data/lib/saboteur/line_hits.rb +81 -0
- data/lib/saboteur/method_calls.rb +91 -0
- data/lib/saboteur/mutation_source/argument_swap.rb +54 -0
- data/lib/saboteur/mutation_source/builtin.rb +33 -21
- data/lib/saboteur/mutation_source/condition_negation.rb +3 -12
- data/lib/saboteur/mutation_source/hazard_guard.rb +94 -13
- data/lib/saboteur/mutation_source/jump_value.rb +59 -0
- data/lib/saboteur/mutation_source/literal_replacement.rb +6 -23
- data/lib/saboteur/mutation_source/operator_examples.rb +6 -0
- data/lib/saboteur/mutation_source/operator_registry.rb +6 -0
- data/lib/saboteur/mutation_source/operator_visit.rb +7 -1
- data/lib/saboteur/mutation_source/rescue_removal.rb +69 -0
- data/lib/saboteur/mutation_source/unary_removal.rb +47 -0
- data/lib/saboteur/mutation_source/when_removal.rb +51 -0
- data/lib/saboteur/preview.rb +13 -0
- data/lib/saboteur/refinement_activations.rb +22 -10
- data/lib/saboteur/reporter/agent.rb +2 -2
- data/lib/saboteur/reporter/agent_incomplete.rb +31 -5
- data/lib/saboteur/reporter/agent_summary.rb +16 -10
- data/lib/saboteur/reporter/review_comments.rb +151 -0
- data/lib/saboteur/reporter/safe_out.rb +32 -0
- data/lib/saboteur/reporter/stryker.rb +8 -6
- data/lib/saboteur/reporter/survivor_classification.rb +41 -3
- data/lib/saboteur/reporter/survivor_hints.rb +5 -0
- data/lib/saboteur/reporter/terminal.rb +45 -5
- data/lib/saboteur/result.rb +14 -4
- data/lib/saboteur/selection/coverage_map.rb +150 -9
- data/lib/saboteur/selection/external_test_map.rb +4 -0
- data/lib/saboteur/selection/git_diff.rb +24 -2
- data/lib/saboteur/selection/layout_only_lines.rb +45 -1
- data/lib/saboteur/selection/recorder.rb +52 -22
- data/lib/saboteur/selection/trace_recorder.rb +83 -0
- data/lib/saboteur/selection/untracked_changes.rb +1 -2
- data/lib/saboteur/subject.rb +16 -0
- data/lib/saboteur/subject_finder.rb +91 -6
- data/lib/saboteur/text.rb +31 -0
- data/lib/saboteur/version.rb +1 -1
- data/lib/saboteur/working_tree.rb +7 -2
- data/lib/saboteur.rb +9 -0
- metadata +48 -16
|
@@ -11,13 +11,29 @@ module Saboteur
|
|
|
11
11
|
# unchanged, otherwise built by running the whole suite once in a
|
|
12
12
|
# supervised child and saved.
|
|
13
13
|
class CoverageStore
|
|
14
|
-
|
|
14
|
+
include CoverageShadow
|
|
15
|
+
include CoveragePartial
|
|
16
|
+
include CoverageParts
|
|
17
|
+
|
|
18
|
+
CACHE_INPUTS = %w[integration require rspec_args rails tests coverage_recorder].freeze
|
|
15
19
|
LOCKFILES = %w[Gemfile.lock gems.locked].freeze
|
|
16
20
|
LOG = "coverage-build.log"
|
|
21
|
+
# RSpec rewrites this on every run (example_status_persistence_file_path),
|
|
22
|
+
# so hashing it would make every second run look edited.
|
|
23
|
+
GENERATED_OUTPUT = "spec/examples.txt"
|
|
24
|
+
# A snapshot entry that is never a file, so a snapshot holding it is never
|
|
25
|
+
# reused partially. It stands for a test file outside the project root,
|
|
26
|
+
# whose own inputs the snapshot cannot see.
|
|
27
|
+
OUTSIDE_TESTS = "(test file outside the project root)"
|
|
28
|
+
# What a failed child's log shows in the error: its last lines, bounded.
|
|
29
|
+
TAIL_LINES = 60
|
|
30
|
+
TAIL_BYTES = 65_536
|
|
31
|
+
TAIL_LINE_WIDTH = 300
|
|
17
32
|
|
|
18
33
|
def initialize(config:, integration:, hooks:, supervisor:, diagnostics:, load_lines: {}, load_methods: {},
|
|
19
|
-
timeout_cap: nil)
|
|
34
|
+
timeout_cap: nil, parts: 1)
|
|
20
35
|
@config = config
|
|
36
|
+
@parts = parts
|
|
21
37
|
@integration = integration
|
|
22
38
|
@hooks = hooks
|
|
23
39
|
@supervisor = supervisor
|
|
@@ -40,11 +56,18 @@ module Saboteur
|
|
|
40
56
|
|
|
41
57
|
def fetch(files)
|
|
42
58
|
path = cache_path(files)
|
|
59
|
+
preloaded = LoadCoverage.preloaded_uninstrumented(files)
|
|
43
60
|
cached = cached_map(files)
|
|
44
|
-
return refuse_empty_map(cached, files) if cached
|
|
61
|
+
return refuse_empty_map(cached.with_preloaded(preloaded), files, preloaded) if cached
|
|
45
62
|
|
|
46
|
-
|
|
63
|
+
note = miss_note(files)
|
|
64
|
+
@diagnostics.inform(note.message(scoped: !@config["tests"].empty?)) if @config["cache"]
|
|
65
|
+
started = Process.clock_gettime(Process::CLOCK_MONOTONIC)
|
|
66
|
+
map = fresh_map(files, note, preloaded)
|
|
67
|
+
build_seconds = Process.clock_gettime(Process::CLOCK_MONOTONIC) - started
|
|
47
68
|
map.save(path)
|
|
69
|
+
shadow(files, map, note, build_seconds) if shadow?
|
|
70
|
+
note.save
|
|
48
71
|
File.write(File.join(@config.path("cache_dir"), "preview.json"),
|
|
49
72
|
JSON.generate(path: path, fingerprint: RunFingerprint.new(config: @config).snapshot.digest))
|
|
50
73
|
map
|
|
@@ -52,6 +75,12 @@ module Saboteur
|
|
|
52
75
|
|
|
53
76
|
private
|
|
54
77
|
|
|
78
|
+
# The partial map when it is on and safe, else the full build.
|
|
79
|
+
def fresh_map(files, note, preloaded)
|
|
80
|
+
(partial_map(files, note, preloaded) if partial_use?) ||
|
|
81
|
+
refuse_empty_map(build(files).with_preloaded(preloaded), files, preloaded)
|
|
82
|
+
end
|
|
83
|
+
|
|
55
84
|
def write_map(writer, recorder, build_seconds)
|
|
56
85
|
writer.write(recorder.result(build_seconds: build_seconds, loaded_files: loaded_project_files).to_json)
|
|
57
86
|
writer.close
|
|
@@ -78,9 +107,63 @@ module Saboteur
|
|
|
78
107
|
end
|
|
79
108
|
|
|
80
109
|
def cache_key(files)
|
|
81
|
-
|
|
82
|
-
|
|
83
|
-
|
|
110
|
+
Selection::CoverageMap.cache_key(files: key_files(files), extra: key_inputs)
|
|
111
|
+
end
|
|
112
|
+
|
|
113
|
+
def key_inputs
|
|
114
|
+
JSON.generate(CACHE_INPUTS.to_h { |key| [key, @config[key]] })
|
|
115
|
+
end
|
|
116
|
+
|
|
117
|
+
def recorder_class
|
|
118
|
+
@config["coverage_recorder"] == "trace" ? Selection::TraceRecorder : Selection::Recorder
|
|
119
|
+
end
|
|
120
|
+
|
|
121
|
+
def key_files(files)
|
|
122
|
+
(files + @integration.test_files + SuiteFiles.new(@config).files + lockfiles).uniq
|
|
123
|
+
end
|
|
124
|
+
|
|
125
|
+
def miss_note(files)
|
|
126
|
+
CoverageMissNote.new(cache_dir: @config.path("cache_dir"), root: @root, files: snapshot_files(files),
|
|
127
|
+
settings: key_inputs, map_key: cache_key(files), seed: @config["seed"],
|
|
128
|
+
clean: ->(path) { plain_line(Text.utf8(path.to_s)) })
|
|
129
|
+
end
|
|
130
|
+
|
|
131
|
+
# What the inputs snapshot hashes: the cache key's files plus every file
|
|
132
|
+
# under spec/ and under the suite's own test directories, fixtures and
|
|
133
|
+
# config included, except the generated spec/examples.txt. A partial
|
|
134
|
+
# rebuild (#46) must see a change to anything a test can read, not only
|
|
135
|
+
# to what the key covers. Symlinks and other special entries are listed
|
|
136
|
+
# but never read (see CoverageInputsSnapshot.file_digest). The cache dir
|
|
137
|
+
# is left out so the pair never hashes itself.
|
|
138
|
+
def snapshot_files(files)
|
|
139
|
+
cache = File.join(File.expand_path(@config.path("cache_dir")), "")
|
|
140
|
+
dirs, outside = test_dirs
|
|
141
|
+
tree = dirs.flat_map do |dir|
|
|
142
|
+
Dir.glob("**/*", File::FNM_DOTMATCH, base: File.join(@root, dir)).map { |name| File.join(@root, dir, name) }
|
|
143
|
+
end
|
|
144
|
+
generated = File.join(@root, GENERATED_OUTPUT)
|
|
145
|
+
tree = tree.reject do |file|
|
|
146
|
+
file == generated || real_directory?(file) || File.expand_path(file).start_with?(cache)
|
|
147
|
+
end
|
|
148
|
+
marker = outside ? [OUTSIDE_TESTS] : []
|
|
149
|
+
[key_files(files), tree, marker].flatten.sort
|
|
150
|
+
end
|
|
151
|
+
|
|
152
|
+
# spec/ always, plus the first directory of every configured test file
|
|
153
|
+
# (a suite under test/ or elsewhere keeps its fixtures beside it), and
|
|
154
|
+
# whether a test file lies outside the project root. Both sides are
|
|
155
|
+
# compared as real paths, so a symlinked root still matches.
|
|
156
|
+
def test_dirs
|
|
157
|
+
prefix = "#{Selection::CoverageMap.canonical(@root)}/"
|
|
158
|
+
inside, outside = @integration.test_files.map { |file| Selection::CoverageMap.canonical(file) }
|
|
159
|
+
.partition { |file| file.start_with?(prefix) }
|
|
160
|
+
[["spec"] + inside.map { |file| file.delete_prefix(prefix).split("/").first }, !outside.empty?]
|
|
161
|
+
end
|
|
162
|
+
|
|
163
|
+
def real_directory?(file)
|
|
164
|
+
File.lstat(file).directory?
|
|
165
|
+
rescue SystemCallError
|
|
166
|
+
true
|
|
84
167
|
end
|
|
85
168
|
|
|
86
169
|
def lockfiles
|
|
@@ -99,6 +182,9 @@ module Saboteur
|
|
|
99
182
|
"Run through the saboteur executable and disable SimpleCov for the run."
|
|
100
183
|
end
|
|
101
184
|
|
|
185
|
+
parts = usage_checked { suite_parts }
|
|
186
|
+
return build_parts(files, parts) if parts.size > 1
|
|
187
|
+
|
|
102
188
|
scope = @config["tests"].empty? ? "the whole suite" : "the selected specs"
|
|
103
189
|
@diagnostics.inform("building coverage map (runs #{scope} once)")
|
|
104
190
|
# The suite child forked next inherits this heap; pack it first, to
|
|
@@ -113,13 +199,136 @@ module Saboteur
|
|
|
113
199
|
map_from(result, admitted)
|
|
114
200
|
end
|
|
115
201
|
|
|
116
|
-
|
|
202
|
+
# The spec files dealt into @parts disjoint parts; one part (the
|
|
203
|
+
# default, or a suite with fewer files than parts) keeps the single
|
|
204
|
+
# child build (#741).
|
|
205
|
+
def suite_parts
|
|
206
|
+
return [] unless @parts > 1
|
|
207
|
+
|
|
208
|
+
Engine::SuitePartition.parts(@integration.test_files, @parts)
|
|
209
|
+
end
|
|
210
|
+
|
|
211
|
+
# The same suite pass as build, run as several children at once, each
|
|
212
|
+
# over its own spec files. Fails loudly unless the parts hold exactly
|
|
213
|
+
# the suite's examples before the run and the combined map holds
|
|
214
|
+
# exactly its tests after it, and refuses a part that recorded no test.
|
|
215
|
+
def build_parts(files, parts)
|
|
216
|
+
total, counts = usage_checked { check_parts_cover_suite(parts) }
|
|
217
|
+
saved = saved_parts(files, parts.size, counts)
|
|
218
|
+
todo = saved.each_index.select { |index| saved[index].nil? }
|
|
219
|
+
@diagnostics.inform(parts_note(parts.size, todo.size))
|
|
220
|
+
built = todo.empty? ? [] : run_parts(files, parts, todo)
|
|
221
|
+
todo.zip(built) { |index, map| saved[index] = map }
|
|
222
|
+
map = combine_saved(saved, total, files, parts.size)
|
|
223
|
+
forget_parts(files, parts.size)
|
|
224
|
+
map
|
|
225
|
+
end
|
|
226
|
+
|
|
227
|
+
# Parts that cannot be combined are dropped, not kept: a bad saved
|
|
228
|
+
# part would otherwise fail every later run the same way.
|
|
229
|
+
def combine_saved(maps, total, files, count)
|
|
230
|
+
combine_parts(maps, total)
|
|
231
|
+
rescue ArgumentError => e
|
|
232
|
+
forget_parts(files, count)
|
|
233
|
+
raise Error, "the saved coverage parts do not combine (#{e.message}); they were removed, run again"
|
|
234
|
+
end
|
|
235
|
+
|
|
236
|
+
def parts_note(count, missing)
|
|
237
|
+
return "building coverage map (reusing all #{count} saved parts)" if missing.zero?
|
|
238
|
+
|
|
239
|
+
reused = count - missing
|
|
240
|
+
suffix = reused.zero? ? "" : ", reusing #{reused} saved"
|
|
241
|
+
"building coverage map (runs the suite once, in #{count} parallel parts#{suffix})"
|
|
242
|
+
end
|
|
243
|
+
|
|
244
|
+
# Runs only the parts in todo, as several children at once.
|
|
245
|
+
def run_parts(files, parts, todo)
|
|
246
|
+
Execution::ForkReadiness.prepare_heap
|
|
247
|
+
heartbeat = @diagnostics.progress_heartbeat("building coverage map")
|
|
248
|
+
admitted = build_timeout
|
|
249
|
+
results = @supervisor.run_many(todo.size, timeout: admitted, on_tick: heartbeat.method(:tick),
|
|
250
|
+
label: "coverage child died during the map build") do |writer, slot|
|
|
251
|
+
index = todo[slot]
|
|
252
|
+
own_test_env(index)
|
|
253
|
+
record(writer, files, only: parts[index], log: part_log_path(index))
|
|
254
|
+
end
|
|
255
|
+
part_maps(results, admitted, parts.size, files, todo)
|
|
256
|
+
end
|
|
257
|
+
|
|
258
|
+
# Every part that failed is reported, each with its own output, so one
|
|
259
|
+
# run shows all the failures instead of the first. A budget stop still
|
|
260
|
+
# raises at once: it says nothing about the suite. A part that did
|
|
261
|
+
# finish is saved either way, so the work is not lost (#427).
|
|
262
|
+
def part_maps(results, admitted, count, files, todo)
|
|
263
|
+
failures = []
|
|
264
|
+
budget_stop = nil
|
|
265
|
+
maps = results.each_with_index.map do |result, slot|
|
|
266
|
+
index = todo[slot]
|
|
267
|
+
map_from(result, admitted, log: part_log_path(index)).tap { |map| save_part(map, files, index, count) }
|
|
268
|
+
rescue BudgetPhaseTimeout => e
|
|
269
|
+
budget_stop = e
|
|
270
|
+
nil
|
|
271
|
+
rescue Error => e
|
|
272
|
+
failures << "part #{index + 1} of #{count}: #{e.message}"
|
|
273
|
+
nil
|
|
274
|
+
end
|
|
275
|
+
raise budget_stop if budget_stop
|
|
276
|
+
raise Error, failures.join("\n") unless failures.empty?
|
|
277
|
+
|
|
278
|
+
maps.each_with_index { |map, slot| part_map(map, todo[slot], count) }
|
|
279
|
+
end
|
|
280
|
+
|
|
281
|
+
# A part child gets its own TEST_ENV_NUMBER, as a WorkerPool worker
|
|
282
|
+
# does, so suites that key databases, ports or tmp dirs on it do not
|
|
283
|
+
# collide between parts (part 0 keeps the empty value).
|
|
284
|
+
def own_test_env(index)
|
|
285
|
+
ENV["TEST_ENV_NUMBER"] = index.zero? ? "" : (index + 1).to_s
|
|
286
|
+
end
|
|
287
|
+
|
|
288
|
+
# A bad file list is the caller's mistake: a usage error, not a crash.
|
|
289
|
+
def usage_checked
|
|
290
|
+
yield
|
|
291
|
+
rescue ArgumentError => e
|
|
292
|
+
raise ConfigError, e.message
|
|
293
|
+
end
|
|
294
|
+
|
|
295
|
+
def check_parts_cover_suite(parts)
|
|
296
|
+
total = @integration.example_count
|
|
297
|
+
counts = parts.map { |part| @integration.example_count(files: part) }
|
|
298
|
+
return [total, counts] if counts.sum == total
|
|
299
|
+
|
|
300
|
+
raise Error, "the coverage parts hold #{counts.sum} examples but the suite has #{total}; " \
|
|
301
|
+
"refusing to build a map from a partition that does not cover the suite"
|
|
302
|
+
end
|
|
303
|
+
|
|
304
|
+
def part_map(map, index, total)
|
|
305
|
+
return map unless map.tests.empty?
|
|
306
|
+
|
|
307
|
+
raise Error, "coverage part #{index + 1} of #{total} recorded no tests; refusing to build a partial map"
|
|
308
|
+
end
|
|
309
|
+
|
|
310
|
+
def combine_parts(maps, total)
|
|
311
|
+
map = Selection::CoverageMap.combine(maps)
|
|
312
|
+
return map if map.tests.size == total
|
|
313
|
+
|
|
314
|
+
raise Error, "the combined coverage map holds #{map.tests.size} tests but the suite has #{total}; " \
|
|
315
|
+
"refusing to use a partial map"
|
|
316
|
+
end
|
|
317
|
+
|
|
318
|
+
def part_log_path(index)
|
|
319
|
+
File.join(@config.path("cache_dir"), "coverage-build-#{index + 1}.log")
|
|
320
|
+
end
|
|
321
|
+
|
|
322
|
+
def map_from(result, admitted = @config["coverage_timeout"], log: log_path)
|
|
117
323
|
raise_budget_stop(admitted) if budget_stopped?(result, admitted)
|
|
118
324
|
raise Error, "coverage map build stopped: #{result.detail}" if result.stopped_by
|
|
119
|
-
|
|
325
|
+
if result.payload.empty?
|
|
326
|
+
raise Error,
|
|
327
|
+
"coverage map build failed: #{died(result.exit_status, log)}#{output_tail(log)}"
|
|
328
|
+
end
|
|
120
329
|
|
|
121
330
|
data = JSON.parse(result.payload)
|
|
122
|
-
raise Error, "coverage map build failed: #{data['error']}" if data["error"]
|
|
331
|
+
raise Error, "coverage map build failed: #{child_error(data['error'])}#{output_tail(log)}" if data["error"]
|
|
123
332
|
|
|
124
333
|
with_parent_load_lines(Selection::CoverageMap.from_json(result.payload))
|
|
125
334
|
end
|
|
@@ -157,30 +366,96 @@ module Saboteur
|
|
|
157
366
|
# these sources, so the map passes with a warning. Applies to a cached
|
|
158
367
|
# map too, so a map cached before this check cannot keep a run green,
|
|
159
368
|
# and the guard runs before saving, so an empty map is never cached.
|
|
160
|
-
def refuse_empty_map(map, files)
|
|
369
|
+
def refuse_empty_map(map, files, preloaded)
|
|
161
370
|
return map if files.empty? || !map.empty_for?(files)
|
|
162
371
|
|
|
372
|
+
# A file loaded before the executable's Coverage.start can never be
|
|
373
|
+
# instrumented, so emptiness there is a setup defect, not untested
|
|
374
|
+
# code: name it apart from the generic no-coverage refusal.
|
|
375
|
+
preloaded_empty = preloaded & files.map { |file| Selection::CoverageMap.canonical(file) }
|
|
376
|
+
if preloaded_empty.any?
|
|
377
|
+
message = preloaded_message(preloaded_empty)
|
|
378
|
+
raise Error, message if @config["tests"].empty?
|
|
379
|
+
|
|
380
|
+
@diagnostics.warn(message)
|
|
381
|
+
return map
|
|
382
|
+
end
|
|
163
383
|
if @config["tests"].empty?
|
|
164
384
|
raise Error, "Coverage recorded nothing in #{files.size} included files during a whole-suite " \
|
|
165
385
|
"build. No test in the suite may exercise these files, or the setup hid the app " \
|
|
166
386
|
"from Coverage#{bootsnap_note}. Run `saboteur doctor`."
|
|
167
387
|
end
|
|
168
|
-
@diagnostics.warn("
|
|
388
|
+
@diagnostics.warn("name the specs that exercise them in --test, or drop --test to run the whole suite\n " \
|
|
389
|
+
"Coverage recorded nothing in #{files.size} included files; " \
|
|
169
390
|
"the selected specs may not exercise them.")
|
|
170
391
|
map
|
|
171
392
|
end
|
|
172
393
|
|
|
394
|
+
def preloaded_message(files)
|
|
395
|
+
names = Text.name_list(files.map { |file| file.delete_prefix("#{@root}/") })
|
|
396
|
+
"#{names} loaded before saboteur started coverage (a require ran before the " \
|
|
397
|
+
"executable's Coverage.start - commonly the gemspec requiring the library while bundler/setup " \
|
|
398
|
+
"evaluates it, or RUBYOPT -r), so no test can ever cover #{files.size > 1 ? 'them' : 'it'}. " \
|
|
399
|
+
"Let the test suite load these " \
|
|
400
|
+
"files; a gemspec should require only its version file."
|
|
401
|
+
end
|
|
402
|
+
|
|
173
403
|
# Bootsnap is named only when it is loaded: blaming a library the app
|
|
174
404
|
# does not use sends the reader to the wrong place.
|
|
175
405
|
def bootsnap_note
|
|
176
406
|
" (Bootsnap's compile cache is the default Rails case)" if defined?(::Bootsnap)
|
|
177
407
|
end
|
|
178
408
|
|
|
179
|
-
def died(status)
|
|
409
|
+
def died(status, log)
|
|
180
410
|
how = if status&.signaled? then "was killed by signal #{Signal.signame(status.termsig)}"
|
|
181
411
|
else "exited with status #{status&.exitstatus}"
|
|
182
412
|
end
|
|
183
|
-
"the suite #{how} before reporting; its output is in #{
|
|
413
|
+
"the suite #{how} before reporting; its output is in #{log.delete_prefix("#{@root}/")}"
|
|
414
|
+
end
|
|
415
|
+
|
|
416
|
+
# The end of a failed child's log, where rspec prints its failures, so a
|
|
417
|
+
# CI run shows why without the cache directory. The suite's own output
|
|
418
|
+
# and nothing else. Every line is cleaned by plain_line and starts with
|
|
419
|
+
# "| ", so it is never the start of a workflow command.
|
|
420
|
+
def output_tail(log)
|
|
421
|
+
size = File.size(log)
|
|
422
|
+
cut = size > TAIL_BYTES
|
|
423
|
+
text = File.open(log) do |file|
|
|
424
|
+
file.seek(size - TAIL_BYTES) if cut
|
|
425
|
+
file.read
|
|
426
|
+
end
|
|
427
|
+
lines = Text.utf8(text).lines
|
|
428
|
+
lines.shift if cut
|
|
429
|
+
lines = lines.last(TAIL_LINES).map { |line| tail_line(line) }
|
|
430
|
+
return "" if lines.empty?
|
|
431
|
+
|
|
432
|
+
"\n the last #{lines.size} lines of #{log.delete_prefix("#{@root}/")}:\n#{lines.join("\n")}"
|
|
433
|
+
rescue SystemCallError
|
|
434
|
+
""
|
|
435
|
+
end
|
|
436
|
+
|
|
437
|
+
def tail_line(line)
|
|
438
|
+
"| #{plain_line(line)}"
|
|
439
|
+
end
|
|
440
|
+
|
|
441
|
+
# One line of a child's text, safe to print: colors stripped, control
|
|
442
|
+
# characters escaped, cut to a readable width, and the two things
|
|
443
|
+
# GitHub Actions reads as a workflow command broken where they stand: a
|
|
444
|
+
# "::" that starts the line after any leading whitespace (the runner
|
|
445
|
+
# trims all Unicode spaces first) and a "##[" anywhere in it. Other
|
|
446
|
+
# colons and brackets, as in "Saboteur::Error", are left alone.
|
|
447
|
+
def plain_line(line)
|
|
448
|
+
text = Text.display_safe(line.chomp.gsub(/\e\[[0-9;]*[A-Za-z]/, ""))
|
|
449
|
+
text.size > TAIL_LINE_WIDTH ? "#{text[0, TAIL_LINE_WIDTH]}..." : text
|
|
450
|
+
end
|
|
451
|
+
|
|
452
|
+
# The error a child wrote into its report. It is the child's text, which
|
|
453
|
+
# can carry anything the suite printed or raised, so it gets the log's
|
|
454
|
+
# treatment: the first line follows the message, every later line starts
|
|
455
|
+
# with "| ", and plain_line has cleaned them all.
|
|
456
|
+
def child_error(error)
|
|
457
|
+
first, *rest = Text.utf8(error.to_s).lines
|
|
458
|
+
[plain_line(first.to_s), *rest.first(TAIL_LINES).map { |line| tail_line(line) }].join("\n")
|
|
184
459
|
end
|
|
185
460
|
|
|
186
461
|
def log_path
|
|
@@ -189,9 +464,9 @@ module Saboteur
|
|
|
189
464
|
|
|
190
465
|
# Runs in the supervised child. The suite's output goes to a log, so a
|
|
191
466
|
# child that dies without reporting can still be diagnosed.
|
|
192
|
-
def record(writer, files)
|
|
193
|
-
FileUtils.mkdir_p(File.dirname(
|
|
194
|
-
$stdout.reopen(
|
|
467
|
+
def record(writer, files, only: nil, log: log_path, seed: @config["seed"])
|
|
468
|
+
FileUtils.mkdir_p(File.dirname(log))
|
|
469
|
+
$stdout.reopen(log, "w")
|
|
195
470
|
$stdout.sync = true
|
|
196
471
|
$stderr.reopen($stdout)
|
|
197
472
|
@hooks.run(:after_fork)
|
|
@@ -199,13 +474,13 @@ module Saboteur
|
|
|
199
474
|
# The suite's own files are recorded too: a spec can read a
|
|
200
475
|
# constant directly, and without its line hits no test maps to the
|
|
201
476
|
# read (#319).
|
|
202
|
-
recorder =
|
|
477
|
+
recorder = recorder_class.new(files + @integration.test_files)
|
|
203
478
|
recorder.start
|
|
204
|
-
Kernel.srand(
|
|
205
|
-
passed = @integration.run_all(recorder)
|
|
479
|
+
Kernel.srand(seed)
|
|
480
|
+
passed = only ? @integration.run_all(recorder, files: only, seed: seed) : @integration.run_all(recorder)
|
|
206
481
|
if passed == false
|
|
207
482
|
raise Error, "the unchanged suite failed while building coverage. Fix the suite, then run again " \
|
|
208
|
-
"with seed #{
|
|
483
|
+
"with seed #{seed}."
|
|
209
484
|
end
|
|
210
485
|
write_map(writer, recorder, Process.clock_gettime(Process::CLOCK_MONOTONIC) - started)
|
|
211
486
|
rescue Exception => e # rubocop:disable Lint/RescueException
|
|
@@ -73,8 +73,8 @@ module Saboteur
|
|
|
73
73
|
Omission = Data.define(:subject, :count, :limit)
|
|
74
74
|
DiscoveryFailure = Data.define(:file, :reason)
|
|
75
75
|
|
|
76
|
-
attr_reader :warnings, :invalid_mutations, :generation_events, :omissions, :discovery_failures,
|
|
77
|
-
:hazard_suppressions, :inline_suppressions
|
|
76
|
+
attr_reader :warnings, :live_announced, :invalid_mutations, :generation_events, :omissions, :discovery_failures,
|
|
77
|
+
:hazard_suppressions, :inline_suppressions, :verbose
|
|
78
78
|
|
|
79
79
|
def initialize(err, verbose: false)
|
|
80
80
|
@err = err
|
|
@@ -82,6 +82,7 @@ module Saboteur
|
|
|
82
82
|
@warnings = []
|
|
83
83
|
@invalid_mutations = 0
|
|
84
84
|
@generation_events = []
|
|
85
|
+
@live_announced = []
|
|
85
86
|
@omissions = []
|
|
86
87
|
@discovery_failures = []
|
|
87
88
|
@hazard_suppressions = 0
|
|
@@ -90,8 +91,19 @@ module Saboteur
|
|
|
90
91
|
@erb_template_note_emitted = false
|
|
91
92
|
end
|
|
92
93
|
|
|
94
|
+
# Every warning passes through here once: repository-controlled text
|
|
95
|
+
# (file names, patterns, thread names, error text) cannot start a CI
|
|
96
|
+
# workflow command on a line of its own, whichever site built it
|
|
97
|
+
# (#834). Already escaped text is unchanged.
|
|
93
98
|
def warn(message)
|
|
94
|
-
@warnings << message
|
|
99
|
+
@warnings << Text.display_safe_lines(message)
|
|
100
|
+
end
|
|
101
|
+
|
|
102
|
+
# Warnings built elsewhere (the ignore file's, the custom map's, a
|
|
103
|
+
# child's): each goes through #warn, so none skips the escaping.
|
|
104
|
+
def add_warnings(messages)
|
|
105
|
+
messages.each { |message| warn(message) }
|
|
106
|
+
nil
|
|
95
107
|
end
|
|
96
108
|
|
|
97
109
|
# ERB generator templates (Rails generators carry `.rb` files with ERB
|
|
@@ -148,7 +160,7 @@ module Saboteur
|
|
|
148
160
|
end
|
|
149
161
|
|
|
150
162
|
def inform(message)
|
|
151
|
-
@err.puts("saboteur: #{message}")
|
|
163
|
+
@err.puts("saboteur: #{Text.display_safe_lines(message)}")
|
|
152
164
|
end
|
|
153
165
|
|
|
154
166
|
# --verbose: one line per finished subject with its result marks, so a
|
|
@@ -162,10 +174,15 @@ module Saboteur
|
|
|
162
174
|
inform("#{subject.id} #{marks}")
|
|
163
175
|
end
|
|
164
176
|
|
|
165
|
-
# A message that is both a report warning and a stderr note.
|
|
177
|
+
# A message that is both a report warning and a stderr note. The
|
|
178
|
+
# terminal does not reprint it at the end of the run (#605), but it
|
|
179
|
+
# stays in report.warnings: JSON, HTML and agent consumers discard
|
|
180
|
+
# stderr and must still see it.
|
|
166
181
|
def announce(message)
|
|
167
|
-
|
|
168
|
-
|
|
182
|
+
safe = Text.display_safe_lines(message)
|
|
183
|
+
warn(safe)
|
|
184
|
+
inform(safe)
|
|
185
|
+
@live_announced << safe
|
|
169
186
|
end
|
|
170
187
|
|
|
171
188
|
# A heartbeat for a long phase: the supervisor's wait loop calls tick
|
|
@@ -192,11 +192,11 @@ module Saboteur
|
|
|
192
192
|
|
|
193
193
|
def rebuild(row, subject, list)
|
|
194
194
|
if row["error"]
|
|
195
|
-
@diagnostics.warn("#{subject.id}: #{row['error']}")
|
|
195
|
+
@diagnostics.warn("#{subject.id}: its mutants are reported as errors\n #{row['error']}")
|
|
196
196
|
return failed(row["error"], subject, list)
|
|
197
197
|
end
|
|
198
198
|
|
|
199
|
-
@diagnostics.
|
|
199
|
+
@diagnostics.add_warnings(row["warnings"])
|
|
200
200
|
decode(row, subject, list).each { |result| @cache&.record(result) unless result.reused? }
|
|
201
201
|
end
|
|
202
202
|
|
|
@@ -18,6 +18,7 @@ module Saboteur
|
|
|
18
18
|
def report_unmapped(guess)
|
|
19
19
|
@diagnostics.inform(guess.subjectless_message(since: config["since"])) if guess.subjectless.any?
|
|
20
20
|
inform_skipped_specs(guess)
|
|
21
|
+
@diagnostics.announce(guess.referenced_message) if guess.referenced.any?
|
|
21
22
|
stop_or_warn_unmapped(guess)
|
|
22
23
|
end
|
|
23
24
|
|
|
@@ -26,7 +27,9 @@ module Saboteur
|
|
|
26
27
|
# a gated run keep the hard stop.
|
|
27
28
|
def stop_or_warn_unmapped(guess)
|
|
28
29
|
return if guess.unmatched.empty?
|
|
29
|
-
|
|
30
|
+
if !config["since"] || guess.specs.empty? || gated?
|
|
31
|
+
raise ConfigError, guess.unmatched_message(verbose: @diagnostics.verbose)
|
|
32
|
+
end
|
|
30
33
|
|
|
31
34
|
guess.unmatched.each { |file| @diagnostics.announce(guess.unmatched_warning(file)) }
|
|
32
35
|
end
|
|
@@ -19,26 +19,90 @@ module Saboteur
|
|
|
19
19
|
return [{}, {}] unless Coverage.state == :suspended
|
|
20
20
|
|
|
21
21
|
wanted = files.to_set { |file| Selection::CoverageMap.canonical(file) }
|
|
22
|
+
calls = boot_calls(wanted)
|
|
22
23
|
CoverageReads.result(stop: false, clear: false)
|
|
23
24
|
.each_with_object([{}, {}]) do |(path, data), (lines_h, methods_h)|
|
|
24
25
|
file = Selection::CoverageMap.canonical(path)
|
|
25
26
|
next unless wanted.include?(file)
|
|
26
27
|
|
|
27
|
-
|
|
28
|
-
lines
|
|
28
|
+
called = calls.fetch(file, Set.new)
|
|
29
|
+
lines = ran_lines(file, data, called)
|
|
29
30
|
lines_h[file] = lines unless lines.empty?
|
|
30
|
-
defs =
|
|
31
|
-
methods_h[file] = defs
|
|
31
|
+
defs = called.map(&:first).uniq.sort
|
|
32
|
+
methods_h[file] = defs unless defs.empty?
|
|
32
33
|
end
|
|
33
34
|
end
|
|
34
35
|
|
|
36
|
+
def self.ran_lines(file, data, called)
|
|
37
|
+
lines = data[:lines].each_with_index.filter_map { |count, index| index + 1 if count&.positive? }
|
|
38
|
+
lines - uncalled_shared_lines(file, called)
|
|
39
|
+
end
|
|
40
|
+
|
|
41
|
+
# The def calls the executable's tracer saw while the app loaded,
|
|
42
|
+
# file => Set of [line, name]. Coverage's methods report is off (its
|
|
43
|
+
# per-read cost is what made the coverage map slow, #655).
|
|
44
|
+
def self.boot_calls(wanted)
|
|
45
|
+
tracer = MethodCalls.booted
|
|
46
|
+
return {} unless tracer
|
|
47
|
+
|
|
48
|
+
tracer.disable
|
|
49
|
+
tracer.drain.each_with_object({}) do |(path, hits), by_file|
|
|
50
|
+
file = Selection::CoverageMap.canonical(path)
|
|
51
|
+
(by_file[file] ||= Set.new).merge(hits) if wanted.include?(file)
|
|
52
|
+
end
|
|
53
|
+
end
|
|
54
|
+
|
|
55
|
+
# Include-listed files that are already loaded yet carry no coverage
|
|
56
|
+
# instrumentation: something required them before the executable's
|
|
57
|
+
# Coverage.start (a gemspec requiring the library while bundler/setup
|
|
58
|
+
# evaluates it, or RUBYOPT -r), so no test can ever be attributed to
|
|
59
|
+
# them. Read in the parent, where the snapshot still reflects the boot.
|
|
60
|
+
def self.preloaded_uninstrumented(files, loaded: $LOADED_FEATURES)
|
|
61
|
+
return [] unless Coverage.state == :suspended
|
|
62
|
+
|
|
63
|
+
instrumented = CoverageReads.result(stop: false, clear: false)
|
|
64
|
+
.each_key
|
|
65
|
+
.to_set { |path| Selection::CoverageMap.canonical(path) }
|
|
66
|
+
loaded_set = loaded.to_set { |path| Selection::CoverageMap.canonical(path) }
|
|
67
|
+
files.map { |file| Selection::CoverageMap.canonical(file) }
|
|
68
|
+
.uniq
|
|
69
|
+
.reject { |file| instrumented.include?(file) }
|
|
70
|
+
.select { |file| loaded_set.include?(file) }
|
|
71
|
+
end
|
|
72
|
+
|
|
35
73
|
# A one-line or endless definition executes its def line even when its
|
|
36
74
|
# body is never called. If any sibling on that physical line is uncalled,
|
|
37
75
|
# discard its ambiguous line evidence rather than crediting every body.
|
|
38
|
-
def self.uncalled_shared_lines(
|
|
39
|
-
|
|
40
|
-
|
|
41
|
-
|
|
76
|
+
def self.uncalled_shared_lines(file, called)
|
|
77
|
+
defs = one_line_defs(file)
|
|
78
|
+
defs.group_by(&:first).filter_map do |line, entries|
|
|
79
|
+
line if entries.any? { |entry| !called.include?(entry) }
|
|
80
|
+
end
|
|
81
|
+
end
|
|
82
|
+
|
|
83
|
+
DEFINERS = %i[define_method define_singleton_method].freeze
|
|
84
|
+
|
|
85
|
+
# [line, name] for every def, or define_method with a block, that starts
|
|
86
|
+
# and ends on one physical line. Coverage's methods report lists both.
|
|
87
|
+
def self.one_line_defs(file)
|
|
88
|
+
found = []
|
|
89
|
+
walk = lambda do |node|
|
|
90
|
+
name = definition_name(node)
|
|
91
|
+
found << [node.location.start_line, name] if name && node.location.start_line == node.location.end_line
|
|
92
|
+
node.compact_child_nodes.each(&walk)
|
|
93
|
+
end
|
|
94
|
+
walk.call(Prism.parse_file(file).value)
|
|
95
|
+
found
|
|
96
|
+
rescue SystemCallError
|
|
97
|
+
[]
|
|
98
|
+
end
|
|
99
|
+
|
|
100
|
+
def self.definition_name(node)
|
|
101
|
+
return node.name if node.is_a?(Prism::DefNode)
|
|
102
|
+
return unless node.is_a?(Prism::CallNode) && DEFINERS.include?(node.name) && node.block.is_a?(Prism::BlockNode)
|
|
103
|
+
|
|
104
|
+
first = Array(node.arguments&.arguments).first
|
|
105
|
+
first.unescaped.to_sym if first.is_a?(Prism::SymbolNode)
|
|
42
106
|
end
|
|
43
107
|
end
|
|
44
108
|
end
|