saboteur 0.3.0 → 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +280 -1
- data/README.md +635 -77
- data/exe/saboteur +6 -2
- data/lib/generators/saboteur/install/install_generator.rb +21 -0
- data/lib/saboteur/agent_skill.rb +46 -0
- data/lib/saboteur/baseline/acceptance.rb +218 -0
- data/lib/saboteur/baseline/command.rb +80 -0
- data/lib/saboteur/baseline/comparison.rb +105 -0
- data/lib/saboteur/baseline/completeness.rb +53 -0
- data/lib/saboteur/baseline/evidence.rb +114 -0
- data/lib/saboteur/baseline/identity.rb +78 -0
- data/lib/saboteur/baseline/manifest.rb +164 -0
- data/lib/saboteur/baseline/run_evidence.rb +117 -0
- data/lib/saboteur/baseline/workflow.rb +121 -0
- data/lib/saboteur/baseline_health.rb +1 -1
- data/lib/saboteur/cli/exit_status.rb +93 -0
- data/lib/saboteur/cli/run_reporting.rb +75 -0
- data/lib/saboteur/cli.rb +171 -63
- data/lib/saboteur/config.rb +101 -19
- data/lib/saboteur/config_file.rb +97 -0
- data/lib/saboteur/custom_map.rb +170 -0
- data/lib/saboteur/doctor.rb +57 -9
- data/lib/saboteur/engine/baseline_guard.rb +44 -5
- data/lib/saboteur/engine/boot.rb +9 -0
- data/lib/saboteur/engine/bytecode_equivalence.rb +95 -0
- data/lib/saboteur/engine/canary.rb +74 -0
- data/lib/saboteur/engine/candidate_selection.rb +18 -5
- data/lib/saboteur/engine/candidates.rb +59 -7
- data/lib/saboteur/engine/coverage_store.rb +53 -6
- data/lib/saboteur/engine/data_inputs.rb +43 -0
- data/lib/saboteur/engine/diagnostics.rb +176 -2
- data/lib/saboteur/engine/diff_scope.rb +18 -0
- data/lib/saboteur/engine/distribution.rb +54 -13
- data/lib/saboteur/engine/result_cache.rb +10 -5
- data/lib/saboteur/engine/run_fingerprint.rb +40 -10
- data/lib/saboteur/engine/sandbox_policy.rb +108 -0
- data/lib/saboteur/engine/scheduler.rb +99 -36
- data/lib/saboteur/engine/subject_selection.rb +24 -1
- data/lib/saboteur/engine/suite_files.rb +8 -1
- data/lib/saboteur/engine/test_guess.rb +52 -0
- data/lib/saboteur/engine/unchanged_syntax.rb +87 -0
- data/lib/saboteur/engine/verdict_check.rb +7 -6
- data/lib/saboteur/engine.rb +257 -22
- data/lib/saboteur/execution/protocol.rb +2 -1
- data/lib/saboteur/execution/reevaluator.rb +79 -14
- data/lib/saboteur/execution/runner.rb +120 -17
- data/lib/saboteur/execution/sandbox.rb +271 -0
- data/lib/saboteur/execution/stale_captures.rb +43 -0
- data/lib/saboteur/execution/supervisor.rb +113 -9
- data/lib/saboteur/execution/worker_pool.rb +20 -4
- data/lib/saboteur/execution/wrapper.rb +45 -0
- data/lib/saboteur/frameworks/rails.rb +329 -13
- data/lib/saboteur/gates.rb +83 -0
- data/lib/saboteur/help.rb +63 -23
- data/lib/saboteur/ignore_list.rb +76 -17
- data/lib/saboteur/inline_suppression.rb +65 -0
- data/lib/saboteur/install.rb +132 -0
- data/lib/saboteur/integration/base.rb +1 -0
- data/lib/saboteur/integration/focus_guard.rb +83 -0
- data/lib/saboteur/integration/rspec.rb +68 -4
- data/lib/saboteur/mutation_source/argument_removal.rb +86 -0
- data/lib/saboteur/mutation_source/builtin.rb +30 -5
- data/lib/saboteur/mutation_source/chain_removal.rb +80 -0
- data/lib/saboteur/mutation_source/condition_negation.rb +50 -0
- data/lib/saboteur/mutation_source/endpoint_replacement.rb +61 -0
- data/lib/saboteur/mutation_source/hazard_guard.rb +136 -0
- data/lib/saboteur/mutation_source/operator_assignment.rb +52 -0
- data/lib/saboteur/mutation_source/operator_registry.rb +57 -0
- data/lib/saboteur/mutation_source/pair_removal.rb +101 -0
- data/lib/saboteur/mutation_source/range_replacement.rb +43 -0
- data/lib/saboteur/preview.rb +106 -0
- data/lib/saboteur/replay.rb +1 -1
- data/lib/saboteur/reporter/agent.rb +87 -0
- data/lib/saboteur/reporter/agent_clipping.rb +29 -0
- data/lib/saboteur/reporter/agent_commands.rb +67 -0
- data/lib/saboteur/reporter/agent_incomplete.rb +60 -0
- data/lib/saboteur/reporter/agent_summary.rb +137 -0
- data/lib/saboteur/reporter/findings.rb +12 -19
- data/lib/saboteur/reporter/history_paths.rb +38 -0
- data/lib/saboteur/reporter/history_record.rb +132 -0
- data/lib/saboteur/reporter/html.rb +61 -2
- data/lib/saboteur/reporter/mutation_text.rb +36 -0
- data/lib/saboteur/reporter/next_steps.rb +30 -0
- data/lib/saboteur/reporter/palette.rb +37 -0
- data/lib/saboteur/reporter/score_history.rb +123 -0
- data/lib/saboteur/reporter/score_table.rb +57 -0
- data/lib/saboteur/reporter/stryker.rb +25 -5
- data/lib/saboteur/reporter/survivor_hints.rb +68 -0
- data/lib/saboteur/reporter/survivor_list.rb +126 -0
- data/lib/saboteur/reporter/survivor_ranking.rb +34 -0
- data/lib/saboteur/reporter/terminal.rb +157 -42
- data/lib/saboteur/reporter/test_matrix.rb +57 -0
- data/lib/saboteur/result.rb +86 -10
- data/lib/saboteur/score.rb +89 -0
- data/lib/saboteur/selection/coverage_map.rb +119 -13
- data/lib/saboteur/selection/git_diff.rb +28 -1
- data/lib/saboteur/selection/recorder.rb +55 -13
- data/lib/saboteur/subject.rb +40 -4
- data/lib/saboteur/subject_finder.rb +62 -12
- data/lib/saboteur/version.rb +1 -1
- data/lib/saboteur/working_tree.rb +56 -0
- data/lib/saboteur.rb +10 -0
- metadata +58 -2
|
@@ -3,6 +3,7 @@
|
|
|
3
3
|
|
|
4
4
|
require "digest"
|
|
5
5
|
require "json"
|
|
6
|
+
require_relative "data_inputs"
|
|
6
7
|
|
|
7
8
|
module Saboteur
|
|
8
9
|
class Engine
|
|
@@ -11,8 +12,8 @@ module Saboteur
|
|
|
11
12
|
class RunFingerprint
|
|
12
13
|
FORMAT_VERSION = 1
|
|
13
14
|
MANIFESTS = %w[Gemfile gems.rb Gemfile.lock gems.locked .rspec .rspec-local].freeze
|
|
14
|
-
LOCATION_SETTINGS = %w[cache_dir json_report html_report].freeze
|
|
15
|
-
ENTRY_KINDS = %w[ruby manifest boot hook source_plugin extra].freeze
|
|
15
|
+
LOCATION_SETTINGS = %w[cache_dir json_report html_report score_history].freeze
|
|
16
|
+
ENTRY_KINDS = %w[ruby manifest boot hook source_plugin extra data].freeze
|
|
16
17
|
EXCLUDED_DIRECTORIES = %w[.git vendor node_modules].freeze
|
|
17
18
|
Entry = Data.define(:path, :kind, :state, :sha256, :required)
|
|
18
19
|
Change = Data.define(:path, :reason)
|
|
@@ -178,11 +179,14 @@ module Saboteur
|
|
|
178
179
|
end
|
|
179
180
|
|
|
180
181
|
def output_overlap_issues
|
|
181
|
-
paths = required_files.map(&:first) + MANIFESTS.map { |name| File.join(@root, name) }
|
|
182
|
-
|
|
183
|
-
|
|
182
|
+
paths = required_files.map(&:first) + MANIFESTS.map { |name| File.join(@root, name) } +
|
|
183
|
+
DataInputs.files(@root, extra: @config["fingerprint_extra"])
|
|
184
|
+
# Compare physical boundaries without changing declared inventory identities.
|
|
185
|
+
paths = paths.map { |path| canonical_path(path) }.uniq
|
|
186
|
+
outputs = %w[json_report html_report score_history].filter_map { |key| @config.fetch(key) }
|
|
187
|
+
.map { |path| canonical_path(path) }
|
|
184
188
|
issues = outputs.filter_map do |path|
|
|
185
|
-
if path.end_with?(".rb") || paths.include?(path) || File.directory?(path)
|
|
189
|
+
if path.end_with?(".rb") || paths.include?(path) || File.directory?(path) || data_input?(path)
|
|
186
190
|
"output overlaps input: #{display_path(path)}"
|
|
187
191
|
end
|
|
188
192
|
end
|
|
@@ -192,24 +196,50 @@ module Saboteur
|
|
|
192
196
|
|
|
193
197
|
def cache_overlap_issues(paths)
|
|
194
198
|
cache = canonical_path(@config["cache_dir"])
|
|
195
|
-
protected = [@root] + %w[app lib spec].map { |name| File.join(@root, name) } + paths
|
|
196
|
-
|
|
199
|
+
protected = [@root] + %w[app lib spec test/fixtures].map { |name| File.join(@root, name) } + paths
|
|
200
|
+
inside = protected.any? do |path|
|
|
197
201
|
path == cache || path.start_with?("#{cache}/") || (path != @root && cache.start_with?("#{path}/"))
|
|
198
202
|
end
|
|
203
|
+
overlaps = inside || data_input?(cache)
|
|
199
204
|
overlaps ? ["cache directory overlaps inputs: #{display_path(cache)}"] : []
|
|
200
205
|
end
|
|
201
206
|
|
|
207
|
+
# Declared fixture, schema or config input, or a path inside a fixture tree.
|
|
208
|
+
def data_input?(path)
|
|
209
|
+
DataInputs.match?(@root, path, extra: @config["fingerprint_extra"])
|
|
210
|
+
end
|
|
211
|
+
|
|
202
212
|
def inventory(issues)
|
|
203
213
|
paths = project_files(issues).to_h { |path| [path, "ruby"] }
|
|
204
214
|
MANIFESTS.each { |name| paths[File.join(@root, name)] = "manifest" }
|
|
215
|
+
data_files(issues).each { |path| paths[path] ||= "data" }
|
|
205
216
|
required = merged_required_files
|
|
206
217
|
required.each { |path, (kind, _)| paths[path] = kind }
|
|
207
218
|
paths.map do |path, kind|
|
|
208
|
-
needed = required.key?(path) ? required[path].last : kind
|
|
219
|
+
needed = required.key?(path) ? required[path].last : !%w[manifest data].include?(kind)
|
|
209
220
|
read_entry(path, kind, needed, issues)
|
|
210
221
|
end
|
|
211
222
|
end
|
|
212
223
|
|
|
224
|
+
# Fixtures, schema dumps and database config. A pattern with no match is
|
|
225
|
+
# fine; a listed file that vanishes later shows up as "removed". Like Ruby
|
|
226
|
+
# files, data reached through a symlink (the file or a directory above it)
|
|
227
|
+
# is an undeclared boundary: its real path could be an output or lie outside
|
|
228
|
+
# the project, so the snapshot is incomplete instead of guessing.
|
|
229
|
+
def data_files(issues)
|
|
230
|
+
DataInputs.files(@root, extra: @config["fingerprint_extra"]).reject do |path|
|
|
231
|
+
next false unless symlinked?(path)
|
|
232
|
+
|
|
233
|
+
issues << "symlink input: #{display_path(path)}".freeze
|
|
234
|
+
true
|
|
235
|
+
end
|
|
236
|
+
end
|
|
237
|
+
|
|
238
|
+
def symlinked?(path)
|
|
239
|
+
current = @root
|
|
240
|
+
path.delete_prefix("#{@root}/").split("/").any? { |part| File.symlink?(current = File.join(current, part)) }
|
|
241
|
+
end
|
|
242
|
+
|
|
213
243
|
def merged_required_files
|
|
214
244
|
required_files.group_by(&:first).transform_values do |entries|
|
|
215
245
|
roles = entries.map { |_, kind, _| kind }
|
|
@@ -240,7 +270,7 @@ module Saboteur
|
|
|
240
270
|
|
|
241
271
|
def excluded?(path, name)
|
|
242
272
|
EXCLUDED_DIRECTORIES.include?(name) || path == canonical_path(@config["cache_dir"]) ||
|
|
243
|
-
%w[json_report html_report].filter_map { |key| @config.fetch(key) }.any? do |output|
|
|
273
|
+
%w[json_report html_report score_history].filter_map { |key| @config.fetch(key) }.any? do |output|
|
|
244
274
|
path == canonical_path(output)
|
|
245
275
|
end
|
|
246
276
|
end
|
|
@@ -0,0 +1,108 @@
|
|
|
1
|
+
# SPDX-License-Identifier: MIT
|
|
2
|
+
# frozen_string_literal: true
|
|
3
|
+
|
|
4
|
+
require "fileutils"
|
|
5
|
+
require "tmpdir"
|
|
6
|
+
|
|
7
|
+
module Saboteur
|
|
8
|
+
class Engine
|
|
9
|
+
# Decides what each mutant child's Landlock sandbox grants (#152), and
|
|
10
|
+
# notes once per run when the kernel cannot honor it.
|
|
11
|
+
class SandboxPolicy
|
|
12
|
+
def initialize(config:, root:, diagnostics:)
|
|
13
|
+
@config = config
|
|
14
|
+
@root = root
|
|
15
|
+
@diagnostics = diagnostics
|
|
16
|
+
end
|
|
17
|
+
|
|
18
|
+
# The third C1 layer (#152): each mutant child makes the project tree
|
|
19
|
+
# read-only except tmp/, the cache dir and the system temp dir. The
|
|
20
|
+
# claim needs Landlock ABI 3 (Linux 6.2+): earlier ABIs cannot restrict
|
|
21
|
+
# truncate(), so the run degrades honestly - children run with the
|
|
22
|
+
# guard and canary only, noted once - unless c.sandbox = "strict",
|
|
23
|
+
# which refuses instead.
|
|
24
|
+
def hook
|
|
25
|
+
return nil unless @config["sandbox"]
|
|
26
|
+
|
|
27
|
+
abi = sandbox_abi
|
|
28
|
+
return restrict_hook if abi >= 3
|
|
29
|
+
|
|
30
|
+
if @config["sandbox"] == "strict"
|
|
31
|
+
raise Error, "sandbox is strict but #{sandbox_gap(abi)}; refusing to run without the promised " \
|
|
32
|
+
"sandbox - set c.sandbox = true to run degraded, or false to opt out"
|
|
33
|
+
end
|
|
34
|
+
# Once, through the report's warnings only: warn + inform printed the
|
|
35
|
+
# same line twice (D's review, #178).
|
|
36
|
+
unless @sandbox_note_given
|
|
37
|
+
@sandbox_note_given = true
|
|
38
|
+
@diagnostics.warn("sandbox is on but #{sandbox_gap(abi)}; children run with the hazard guard and canary only")
|
|
39
|
+
end
|
|
40
|
+
nil
|
|
41
|
+
end
|
|
42
|
+
|
|
43
|
+
# ENOSYS/EOPNOTSUPP are a clean "no Landlock here"; any other probe
|
|
44
|
+
# error (EPERM under seccomp, ...) must not silently run unsandboxed.
|
|
45
|
+
def sandbox_abi
|
|
46
|
+
Execution::Sandbox.abi_version
|
|
47
|
+
rescue SystemCallError => e
|
|
48
|
+
raise Error, "sandbox is on but the Landlock probe failed (#{e.message}); refusing to run " \
|
|
49
|
+
"unsandboxed - set c.sandbox = false to proceed without it"
|
|
50
|
+
end
|
|
51
|
+
|
|
52
|
+
def sandbox_gap(abi)
|
|
53
|
+
return "this kernel lacks Landlock (needs Linux 6.2+)" if abi.zero?
|
|
54
|
+
|
|
55
|
+
"this kernel's Landlock ABI is #{abi} but ABI 3 (Linux 6.2+) is needed to restrict truncate()"
|
|
56
|
+
end
|
|
57
|
+
|
|
58
|
+
def restrict_hook
|
|
59
|
+
writable = sandbox_writable_dirs
|
|
60
|
+
->(keep_fds) { Execution::Sandbox.restrict!(writable: writable, keep_fds: keep_fds) }
|
|
61
|
+
end
|
|
62
|
+
|
|
63
|
+
# c.sandbox_writable adds the dirs a suite genuinely needs (db/, log/,
|
|
64
|
+
# ...): without a grant, writes there fail with EACCES in the child.
|
|
65
|
+
def sandbox_writable_dirs
|
|
66
|
+
user_dirs = @config["sandbox_writable"].map { |dir| File.expand_path(dir, @root) }
|
|
67
|
+
dirs = [File.join(@root, "tmp"), @config.path("cache_dir")] + user_dirs
|
|
68
|
+
dirs.each { |dir| FileUtils.mkdir_p(dir) }
|
|
69
|
+
real = (dirs.map { |dir| File.realpath(dir) } + [File.realpath(Dir.tmpdir)]).uniq
|
|
70
|
+
check_sandbox_write_set(real, user_dirs.map { |dir| File.realpath(dir) })
|
|
71
|
+
real
|
|
72
|
+
end
|
|
73
|
+
|
|
74
|
+
# The effective write set - every grant realpath'd, cache_dir, the
|
|
75
|
+
# project tmp and the system temp dir included - must not contain the
|
|
76
|
+
# project root: cache_dir = ".", a tmp symlink or a checkout inside the
|
|
77
|
+
# system temp dir would otherwise leave the whole tree writable without
|
|
78
|
+
# a word. c.sandbox = "strict" refuses: strict guards the effective
|
|
79
|
+
# policy, not only the kernel ABI. A user grant containing another
|
|
80
|
+
# grant only makes the narrower one redundant, so that stays a warning.
|
|
81
|
+
def check_sandbox_write_set(all_dirs, user_dirs)
|
|
82
|
+
holder = all_dirs.find { |dir| path_contains?(dir, @root) }
|
|
83
|
+
if holder
|
|
84
|
+
message = "sandbox grant #{holder} contains the project root; mutant children can write the whole tree"
|
|
85
|
+
raise Error, %(#{message} - refusing under c.sandbox = "strict") if @config["sandbox"] == "strict"
|
|
86
|
+
|
|
87
|
+
@diagnostics.warn(message)
|
|
88
|
+
end
|
|
89
|
+
user_dirs.each do |dir|
|
|
90
|
+
(all_dirs - [dir]).each do |other|
|
|
91
|
+
next unless path_contains?(dir, other)
|
|
92
|
+
|
|
93
|
+
@diagnostics.warn("sandbox grant #{dir} contains the granted dir #{other}; " \
|
|
94
|
+
"the narrower grant is redundant")
|
|
95
|
+
end
|
|
96
|
+
end
|
|
97
|
+
end
|
|
98
|
+
|
|
99
|
+
# Both sides are realpath'd, so a plain "#{dir}/" prefix test would miss
|
|
100
|
+
# dir == "/" (every path is under it; "//" prefixes nothing).
|
|
101
|
+
def path_contains?(dir, path)
|
|
102
|
+
return true if dir == "/"
|
|
103
|
+
|
|
104
|
+
path == dir || path.start_with?("#{dir}/")
|
|
105
|
+
end
|
|
106
|
+
end
|
|
107
|
+
end
|
|
108
|
+
end
|
|
@@ -7,16 +7,20 @@ module Saboteur
|
|
|
7
7
|
# unit of work parallel workers will share out.
|
|
8
8
|
#
|
|
9
9
|
# Order of business: skip what cannot be run (refinements, not loaded,
|
|
10
|
-
# shadowed, no covering test); coarse mutants; fine mutants
|
|
11
|
-
#
|
|
12
|
-
#
|
|
13
|
-
#
|
|
14
|
-
# tests' recorded durations.
|
|
10
|
+
# shadowed, no covering test); coarse mutants; all eligible fine mutants;
|
|
11
|
+
# then check every kill and timeout against the unchanged method
|
|
12
|
+
# (VerdictCheck). Each mutant runs only the tests that executed its edit,
|
|
13
|
+
# likeliest killers first, with a timeout from their recorded durations.
|
|
15
14
|
class Scheduler
|
|
15
|
+
WHOLE_FILE_HINT = "; add it to c.whole_file to re-evaluate the whole file"
|
|
16
|
+
|
|
16
17
|
# A subject to run, with what its mutants need. killers counts the
|
|
17
18
|
# tests that killed earlier mutants of this subject; slowdown is how
|
|
18
19
|
# much slower than recorded its tests ran in a child (see calibrate).
|
|
19
|
-
|
|
20
|
+
# wrapper names a known def-time wrapper (Execution::Wrapper), so the
|
|
21
|
+
# child re-evaluates it together with the def.
|
|
22
|
+
Plan = Struct.new(:subject, :tests, :killers, :slowdown, :wrapper,
|
|
23
|
+
keyword_init: true)
|
|
20
24
|
|
|
21
25
|
# Tests that killed more earlier mutants first; otherwise keep the order.
|
|
22
26
|
def self.prioritize(tests, killers)
|
|
@@ -35,20 +39,24 @@ module Saboteur
|
|
|
35
39
|
def call(subject, candidates)
|
|
36
40
|
return [] if candidates.empty?
|
|
37
41
|
|
|
38
|
-
reason = subject.skip_reason
|
|
42
|
+
reason, wrapper = subject.skip_reason ? [subject.skip_reason, nil] : load_problem(subject)
|
|
39
43
|
return skip_all(candidates, subject, reason) if reason
|
|
44
|
+
if wrapper == :sorbet && !subject.sig_source
|
|
45
|
+
return skip_all(candidates, subject,
|
|
46
|
+
"sorbet sig not found next to the def; cannot preserve runtime type checks")
|
|
47
|
+
end
|
|
40
48
|
|
|
41
49
|
tests = @map.tests_for(subject)
|
|
42
|
-
return
|
|
50
|
+
return unloadable(candidates, subject) if tests.empty?
|
|
43
51
|
|
|
44
|
-
plan = Plan.new(subject:, tests:, killers: Hash.new(0), slowdown: 1.0)
|
|
52
|
+
plan = Plan.new(subject:, tests:, killers: Hash.new(0), slowdown: 1.0, wrapper: wrapper)
|
|
45
53
|
reused = all_reused(plan, candidates)
|
|
46
54
|
return reused if reused
|
|
47
55
|
|
|
48
|
-
results = run_candidates(
|
|
56
|
+
results = run_candidates(candidates) do |mutation, text|
|
|
49
57
|
reuse(plan, mutation, text) || execute(plan, mutation, text)
|
|
50
58
|
end
|
|
51
|
-
checked(
|
|
59
|
+
checked(plan, results)
|
|
52
60
|
end
|
|
53
61
|
|
|
54
62
|
# The per-mutant timeout: timeout_factor times what the tests took
|
|
@@ -65,9 +73,11 @@ module Saboteur
|
|
|
65
73
|
|
|
66
74
|
# Verdicts that did not hold up are warned about once per subject;
|
|
67
75
|
# only checked verdicts are stored for --incremental.
|
|
68
|
-
def checked(
|
|
76
|
+
def checked(plan, results)
|
|
77
|
+
subject = plan.subject
|
|
69
78
|
run_unchanged = lambda do |tests, timeout|
|
|
70
|
-
@runner.run(subject: subject, text: subject.source, test_ids: tests, timeout: timeout
|
|
79
|
+
@runner.run(subject: subject, text: subject.source, test_ids: tests, timeout: timeout,
|
|
80
|
+
eval_prefix: eval_prefix(plan))
|
|
71
81
|
.tap { |outcome| note_leftovers(subject, "verdict check", outcome) }
|
|
72
82
|
end
|
|
73
83
|
check = VerdictCheck.new(run_unchanged: run_unchanged, baseline_timeout: @config["baseline_timeout"])
|
|
@@ -86,21 +96,13 @@ module Saboteur
|
|
|
86
96
|
return nil unless @cache
|
|
87
97
|
|
|
88
98
|
catch(:must_run) do
|
|
89
|
-
run_candidates(
|
|
99
|
+
run_candidates(candidates) { |mutation, text| reuse(plan, mutation, text) || throw(:must_run) }
|
|
90
100
|
end
|
|
91
101
|
end
|
|
92
102
|
|
|
93
|
-
def run_candidates(
|
|
103
|
+
def run_candidates(candidates, &run_one)
|
|
94
104
|
coarse, fine = candidates.partition { |mutation, _| mutation.coarse }
|
|
95
105
|
results = coarse.map { |mutation, text| run_one.call(mutation, text) }
|
|
96
|
-
# A surviving coarse mutant means nothing checks the body, so the fine
|
|
97
|
-
# mutants would only add noise. Not so when a rescue or else can
|
|
98
|
-
# return a value of its own: fine mutants that make the body raise
|
|
99
|
-
# can still be caught.
|
|
100
|
-
if results.any?(&:survived?) && !plan.subject.recovers?
|
|
101
|
-
return results + skip_all(fine, plan.subject, "coarse mutant survived", covered_by: plan.tests)
|
|
102
|
-
end
|
|
103
|
-
|
|
104
106
|
results + fine.map { |mutation, text| run_one.call(mutation, text) }
|
|
105
107
|
end
|
|
106
108
|
|
|
@@ -109,7 +111,7 @@ module Saboteur
|
|
|
109
111
|
return nil unless @cache
|
|
110
112
|
|
|
111
113
|
tests = tests_for(plan, mutation)
|
|
112
|
-
return
|
|
114
|
+
return uncoverable(plan, mutation, text) if tests.empty?
|
|
113
115
|
|
|
114
116
|
hit = @cache.lookup(plan.subject, mutation, tests)
|
|
115
117
|
return nil unless hit
|
|
@@ -121,14 +123,14 @@ module Saboteur
|
|
|
121
123
|
|
|
122
124
|
def execute(plan, mutation, text)
|
|
123
125
|
tests = tests_for(plan, mutation)
|
|
124
|
-
return
|
|
126
|
+
return uncoverable(plan, mutation, text) if tests.empty?
|
|
125
127
|
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
outcome = calibrate(plan, mutation, text, tests, timeout) if outcome.status == :timeout
|
|
128
|
+
outcome = run_mutant(plan, mutation, text, tests)
|
|
129
|
+
note_unreachable(plan, outcome) if outcome.status == :unreachable
|
|
129
130
|
result = MutantResult.new(subject: plan.subject, mutation: mutation, status: outcome.status,
|
|
130
131
|
mutated_source: text, reason: outcome.message, runtime: outcome.runtime,
|
|
131
|
-
killed_by: outcome.failed_ids, covered_by: tests, kill_cause: outcome.kill_cause
|
|
132
|
+
killed_by: outcome.failed_ids, covered_by: tests, kill_cause: outcome.kill_cause,
|
|
133
|
+
executed: outcome.executed_ids)
|
|
132
134
|
outcome.failed_ids.each { |id| plan.killers[id] += 1 }
|
|
133
135
|
note_leftovers(plan.subject, "#{mutation.operator} #{result.digest}", outcome)
|
|
134
136
|
@diagnostics.progress(result)
|
|
@@ -139,8 +141,12 @@ module Saboteur
|
|
|
139
141
|
# body, so it gets every test of the method), prior killers first, then
|
|
140
142
|
# fastest-first.
|
|
141
143
|
def tests_for(plan, mutation)
|
|
144
|
+
return self.class.prioritize(plan.tests, plan.killers) if mutation.coarse
|
|
145
|
+
|
|
142
146
|
line_numbers = mutation.edits.flat_map { |edit| plan.subject.coverage_lines_for(edit) }.uniq
|
|
143
|
-
tests = plan.tests & @map.
|
|
147
|
+
tests = plan.tests & @map.tests_for_edit(plan.subject, line_numbers)
|
|
148
|
+
branch_tests = @map.tests_for_branches(plan.subject, mutation.edits)
|
|
149
|
+
tests &= branch_tests if branch_tests
|
|
144
150
|
self.class.prioritize(tests, plan.killers)
|
|
145
151
|
end
|
|
146
152
|
|
|
@@ -148,6 +154,52 @@ module Saboteur
|
|
|
148
154
|
[@config["timeout_factor"] * runtime, @config["timeout_min"]].max
|
|
149
155
|
end
|
|
150
156
|
|
|
157
|
+
# What a subject no test ran gets. Code that only ran while the app
|
|
158
|
+
# loaded is not "no coverage" - it ran - but no forked test can re-run
|
|
159
|
+
# it, so it gets its own status, out of the score, with a hint.
|
|
160
|
+
def unloadable(candidates, subject, line_numbers = nil)
|
|
161
|
+
return skip_all(candidates, subject, "no coverage") unless
|
|
162
|
+
@map.ran_at_load?(subject.file, line_numbers || subject.coverage_lines)
|
|
163
|
+
|
|
164
|
+
candidates.map do |mutation, text|
|
|
165
|
+
MutantResult.new(subject: subject, mutation: mutation, status: :ran_at_load, mutated_source: text,
|
|
166
|
+
reason: Selection::CoverageMap::RAN_AT_LOAD)
|
|
167
|
+
end
|
|
168
|
+
end
|
|
169
|
+
|
|
170
|
+
# One mutation whose edited lines no test ran (see unloadable).
|
|
171
|
+
def uncoverable(plan, mutation, text)
|
|
172
|
+
line_numbers = mutation.edits.flat_map { |edit| plan.subject.coverage_lines_for(edit) }.uniq
|
|
173
|
+
result, = unloadable([[mutation, text]], plan.subject, line_numbers)
|
|
174
|
+
return result if result.status == :ran_at_load
|
|
175
|
+
return result unless @map.tests_for_branches(plan.subject, mutation.edits) == []
|
|
176
|
+
|
|
177
|
+
skipped, = skip_all([[mutation, text]], plan.subject, "no coverage (branch not taken)")
|
|
178
|
+
skipped
|
|
179
|
+
end
|
|
180
|
+
|
|
181
|
+
def run_mutant(plan, mutation, text, tests)
|
|
182
|
+
timeout = timeout_for(tests, plan.slowdown)
|
|
183
|
+
outcome = @runner.run(subject: plan.subject, text: text, test_ids: tests, timeout: timeout,
|
|
184
|
+
mutation: mutation, eval_prefix: eval_prefix(plan))
|
|
185
|
+
outcome.status == :timeout ? calibrate(plan, mutation, text, tests, timeout) : outcome
|
|
186
|
+
end
|
|
187
|
+
|
|
188
|
+
# Each mutant is checked separately: changing a body can change which
|
|
189
|
+
# definition a test reaches, so one unreachable verdict cannot skip later work.
|
|
190
|
+
def note_unreachable(plan, outcome)
|
|
191
|
+
@diagnostics.warn("#{plan.subject.id}: #{outcome.message}")
|
|
192
|
+
end
|
|
193
|
+
|
|
194
|
+
# The text a known def-time wrapper needs re-evaluated with the def.
|
|
195
|
+
# Sorbet's sig sits on the lines right before it; without it the def
|
|
196
|
+
# evaluates alone and runtime type checks silently drop for the child.
|
|
197
|
+
def eval_prefix(plan)
|
|
198
|
+
return nil unless plan.wrapper == :sorbet
|
|
199
|
+
|
|
200
|
+
plan.subject.sig_source
|
|
201
|
+
end
|
|
202
|
+
|
|
151
203
|
# Recorded durations come from one warm process; a fresh child can be
|
|
152
204
|
# slower (lazy loading, first queries). So a timeout is checked at
|
|
153
205
|
# once: the unchanged method runs the same tests with
|
|
@@ -156,7 +208,7 @@ module Saboteur
|
|
|
156
208
|
# the measured slowdown. A timeout that remains is the mutant's own.
|
|
157
209
|
def calibrate(plan, mutation, text, tests, timeout)
|
|
158
210
|
unchanged = @runner.run(subject: plan.subject, text: plan.subject.source, test_ids: tests,
|
|
159
|
-
timeout: @config["baseline_timeout"])
|
|
211
|
+
timeout: @config["baseline_timeout"], eval_prefix: eval_prefix(plan))
|
|
160
212
|
note_leftovers(plan.subject, "timeout check", unchanged)
|
|
161
213
|
return unchanged_failed(unchanged) unless unchanged.status == :survived
|
|
162
214
|
|
|
@@ -165,7 +217,8 @@ module Saboteur
|
|
|
165
217
|
longer = deadline(unchanged.runtime)
|
|
166
218
|
return Execution::Runner::Outcome.new(status: :timeout, runtime: timeout) if longer <= timeout
|
|
167
219
|
|
|
168
|
-
@runner.run(subject: plan.subject, text: text, test_ids: tests, timeout: longer, mutation: mutation
|
|
220
|
+
@runner.run(subject: plan.subject, text: text, test_ids: tests, timeout: longer, mutation: mutation,
|
|
221
|
+
eval_prefix: eval_prefix(plan))
|
|
169
222
|
end
|
|
170
223
|
|
|
171
224
|
def unchanged_failed(unchanged)
|
|
@@ -176,18 +229,28 @@ module Saboteur
|
|
|
176
229
|
end
|
|
177
230
|
|
|
178
231
|
# The method saboteur would redefine must be the one Ruby is running.
|
|
232
|
+
# Returns [skip reason, wrapper]: a known def-time wrapper (sorbet's
|
|
233
|
+
# sig is one, see Execution::Wrapper) is not a shadowing - the child
|
|
234
|
+
# re-evaluates it together with the def, so the check passes.
|
|
179
235
|
def load_problem(subject)
|
|
236
|
+
return [nil, nil] if @reevaluator.whole_file?(subject)
|
|
237
|
+
|
|
180
238
|
owner = @reevaluator.resolve_owner(subject)
|
|
181
239
|
target = subject.singleton ? owner.singleton_class : owner
|
|
182
240
|
method = own_method(target, subject.name)
|
|
183
|
-
return "method not loaded" unless method
|
|
241
|
+
return ["method not loaded", nil] unless method
|
|
184
242
|
|
|
185
243
|
file, line = method.source_location
|
|
186
|
-
|
|
244
|
+
if file && Selection::CoverageMap.canonical(file) == subject.file && line == subject.start_line
|
|
245
|
+
return [nil, nil]
|
|
246
|
+
end
|
|
247
|
+
|
|
248
|
+
wrapper = Execution::Wrapper.around(method, file)
|
|
249
|
+
return [nil, wrapper] if wrapper
|
|
187
250
|
|
|
188
|
-
"shadowed definition (runtime method at #{file}:#{line})"
|
|
251
|
+
["shadowed definition (runtime method at #{file}:#{line})#{WHOLE_FILE_HINT}", nil]
|
|
189
252
|
rescue Error => e
|
|
190
|
-
"owner not loaded: #{e.message}"
|
|
253
|
+
["owner not loaded: #{e.message}#{WHOLE_FILE_HINT}", nil]
|
|
191
254
|
end
|
|
192
255
|
|
|
193
256
|
def own_method(target, name)
|
|
@@ -33,7 +33,9 @@ module Saboteur
|
|
|
33
33
|
def find(files)
|
|
34
34
|
finder = SubjectFinder.new
|
|
35
35
|
files.flat_map do |file|
|
|
36
|
+
InlineSuppression.new(file, File.read(file))
|
|
36
37
|
parsed = finder.call(file)
|
|
38
|
+
@diagnostics.record_discovery_failure(file, reason: "invalid_file_syntax") unless parsed.errors.empty?
|
|
37
39
|
parsed.errors.first(1).each do |message|
|
|
38
40
|
@diagnostics.warn("#{file.delete_prefix("#{@root}/")} does not parse: #{message}")
|
|
39
41
|
end
|
|
@@ -54,9 +56,30 @@ module Saboteur
|
|
|
54
56
|
def by_change(subjects, ref)
|
|
55
57
|
@changed_lines = Selection::GitDiff.changed_lines(ref, root: @root)
|
|
56
58
|
.transform_keys { |file| Selection::CoverageMap.canonical(file) }
|
|
57
|
-
subjects.select do |subject|
|
|
59
|
+
changed = subjects.select do |subject|
|
|
58
60
|
@changed_lines[subject.file]&.any? { |line| subject.contains_line?(line) }
|
|
59
61
|
end
|
|
62
|
+
drop_unchanged_syntax(changed, ref)
|
|
63
|
+
end
|
|
64
|
+
|
|
65
|
+
# A method whose only changes are comments, blank lines or spacing has
|
|
66
|
+
# nothing new to test.
|
|
67
|
+
def drop_unchanged_syntax(subjects, ref)
|
|
68
|
+
return subjects if subjects.empty?
|
|
69
|
+
|
|
70
|
+
comparison = UnchangedSyntax.new(Selection::GitDiff.merge_base(ref, root: @root), root: @root)
|
|
71
|
+
same, kept = subjects.partition { |subject| comparison.unchanged?(subject) }
|
|
72
|
+
report_unchanged_syntax(same)
|
|
73
|
+
kept
|
|
74
|
+
end
|
|
75
|
+
|
|
76
|
+
def report_unchanged_syntax(subjects)
|
|
77
|
+
return if subjects.empty?
|
|
78
|
+
|
|
79
|
+
names = subjects.first(5).map(&:id).join(", ")
|
|
80
|
+
more = subjects.size > 5 ? ", ..." : ""
|
|
81
|
+
@diagnostics.inform("#{subjects.size} changed method(s) differ only in comments or formatting, " \
|
|
82
|
+
"so they are skipped: #{names}#{more}")
|
|
60
83
|
end
|
|
61
84
|
end
|
|
62
85
|
end
|
|
@@ -3,6 +3,7 @@
|
|
|
3
3
|
|
|
4
4
|
require "digest"
|
|
5
5
|
require "shellwords"
|
|
6
|
+
require_relative "data_inputs"
|
|
6
7
|
|
|
7
8
|
module Saboteur
|
|
8
9
|
class Engine
|
|
@@ -22,7 +23,7 @@ module Saboteur
|
|
|
22
23
|
end
|
|
23
24
|
|
|
24
25
|
def files
|
|
25
|
-
(shared_spec_files + boot_and_hook_files + option_files + required_by_options).uniq.sort
|
|
26
|
+
(shared_spec_files + boot_and_hook_files + option_files + required_by_options + data_files).uniq.sort
|
|
26
27
|
end
|
|
27
28
|
|
|
28
29
|
def digest
|
|
@@ -33,6 +34,12 @@ module Saboteur
|
|
|
33
34
|
|
|
34
35
|
private
|
|
35
36
|
|
|
37
|
+
# Fixtures, schema dumps and database config: not Ruby, but they change
|
|
38
|
+
# what the tests see, so a stale cache must not survive an edit to them.
|
|
39
|
+
def data_files
|
|
40
|
+
DataInputs.files(@root, extra: @config["fingerprint_extra"])
|
|
41
|
+
end
|
|
42
|
+
|
|
36
43
|
def shared_spec_files
|
|
37
44
|
Dir.glob("spec/**/*.rb", base: @root).reject { |file| file.end_with?("_spec.rb") }
|
|
38
45
|
.map { |file| File.join(@root, file) }
|
|
@@ -0,0 +1,52 @@
|
|
|
1
|
+
# SPDX-License-Identifier: MIT
|
|
2
|
+
# frozen_string_literal: true
|
|
3
|
+
|
|
4
|
+
module Saboteur
|
|
5
|
+
class Engine
|
|
6
|
+
# --guess-tests: the specs that sit next to the changed source files by
|
|
7
|
+
# naming convention (lib/a/b.rb -> spec/a/b_spec.rb). A guess, not a
|
|
8
|
+
# coverage fact: a spec that only reaches the file through another one
|
|
9
|
+
# is not found. The run scopes to the guesses only when every changed
|
|
10
|
+
# file has one.
|
|
11
|
+
class TestGuess
|
|
12
|
+
Result = Data.define(:specs, :unmatched, :changed)
|
|
13
|
+
|
|
14
|
+
SOURCE_ROOTS = %w[lib app].freeze
|
|
15
|
+
|
|
16
|
+
def initialize(config)
|
|
17
|
+
@root = File.realpath(config.root)
|
|
18
|
+
@config = config
|
|
19
|
+
end
|
|
20
|
+
|
|
21
|
+
# specs: existing spec files (relative paths); unmatched: changed
|
|
22
|
+
# source files with no guessed spec; changed: how many source files
|
|
23
|
+
# changed.
|
|
24
|
+
def call
|
|
25
|
+
changed = changed_sources
|
|
26
|
+
guesses = changed.to_h { |file| [file, guess_for(file)] }
|
|
27
|
+
Result.new(specs: guesses.values.compact.uniq.sort, unmatched: guesses.select { |_, spec| spec.nil? }.keys,
|
|
28
|
+
changed: changed.size)
|
|
29
|
+
end
|
|
30
|
+
|
|
31
|
+
private
|
|
32
|
+
|
|
33
|
+
# Changed files that saboteur would mutate, relative to the root.
|
|
34
|
+
def changed_sources
|
|
35
|
+
included = @config.included_files.to_set
|
|
36
|
+
changed = Selection::GitDiff.changed_lines(@config["since"], root: @root).keys
|
|
37
|
+
changed.map { |file| Selection::CoverageMap.canonical(file) }
|
|
38
|
+
.select { |file| included.include?(file) }
|
|
39
|
+
.map { |file| file.delete_prefix("#{@root}/") }.sort
|
|
40
|
+
end
|
|
41
|
+
|
|
42
|
+
def guess_for(file)
|
|
43
|
+
stem = file.delete_suffix(".rb")
|
|
44
|
+
rest = stem.split("/", 2)
|
|
45
|
+
candidates = ["spec/#{stem}_spec.rb"]
|
|
46
|
+
candidates.unshift("spec/#{rest.last}_spec.rb", "spec/lib/#{rest.last}_spec.rb") if
|
|
47
|
+
rest.size == 2 && SOURCE_ROOTS.include?(rest.first)
|
|
48
|
+
candidates.find { |path| File.file?(File.join(@root, path)) }
|
|
49
|
+
end
|
|
50
|
+
end
|
|
51
|
+
end
|
|
52
|
+
end
|
|
@@ -0,0 +1,87 @@
|
|
|
1
|
+
# SPDX-License-Identifier: MIT
|
|
2
|
+
# frozen_string_literal: true
|
|
3
|
+
|
|
4
|
+
require "prism"
|
|
5
|
+
require_relative "../inline_suppression"
|
|
6
|
+
|
|
7
|
+
module Saboteur
|
|
8
|
+
class Engine
|
|
9
|
+
# Tells whether a method still has, in the working tree, the syntax it had
|
|
10
|
+
# at the merge base of the --since ref. A change to a comment, a blank
|
|
11
|
+
# line or spacing leaves it the same, so no mutant of it asks a question
|
|
12
|
+
# the change raised. Comparison is of the Prism tree without locations
|
|
13
|
+
# and comments, plus the frozen_string_literal magic comment and the
|
|
14
|
+
# `sig` in front of the def, which both change what runs. `__LINE__` is
|
|
15
|
+
# the one node whose value is its location, so it keeps its line: a
|
|
16
|
+
# method that moved, or one with a line added above its `__LINE__`, is
|
|
17
|
+
# not the same.
|
|
18
|
+
#
|
|
19
|
+
# Anything that cannot be compared says the method changed: a file or
|
|
20
|
+
# method that is new at the base, a base that does not parse, a method
|
|
21
|
+
# name defined twice in a file.
|
|
22
|
+
class UnchangedSyntax
|
|
23
|
+
def initialize(base, root:)
|
|
24
|
+
@base = base
|
|
25
|
+
@root = root
|
|
26
|
+
@base_subjects = {}
|
|
27
|
+
@current_subjects = {}
|
|
28
|
+
end
|
|
29
|
+
|
|
30
|
+
def unchanged?(subject)
|
|
31
|
+
before = base_subject(subject)
|
|
32
|
+
return false unless before
|
|
33
|
+
|
|
34
|
+
return false unless suppression_signature(before) == suppression_signature(subject)
|
|
35
|
+
|
|
36
|
+
before.frozen_literals == subject.frozen_literals && before.sig_source == subject.sig_source &&
|
|
37
|
+
self.class.digest(before.node) == self.class.digest(subject.node)
|
|
38
|
+
end
|
|
39
|
+
|
|
40
|
+
# The node as plain data: type and fields, never locations (except
|
|
41
|
+
# the line a `__LINE__` stands for).
|
|
42
|
+
def self.digest(value)
|
|
43
|
+
case value
|
|
44
|
+
when Prism::SourceLineNode then [value.type, value.location.start_line]
|
|
45
|
+
when Prism::Node
|
|
46
|
+
fields = value.deconstruct_keys(nil).except(:node_id)
|
|
47
|
+
[value.type, fields.transform_values { |field| digest(field) }]
|
|
48
|
+
when Array then value.map { |item| digest(item) }
|
|
49
|
+
when Prism::Location then nil
|
|
50
|
+
else value
|
|
51
|
+
end
|
|
52
|
+
end
|
|
53
|
+
|
|
54
|
+
private
|
|
55
|
+
|
|
56
|
+
def suppression_signature(subject)
|
|
57
|
+
InlineSuppression.new(subject.file, subject.file_source).signature(subject)
|
|
58
|
+
end
|
|
59
|
+
|
|
60
|
+
def base_subject(subject)
|
|
61
|
+
return nil unless current_subjects(subject).key?(subject.id)
|
|
62
|
+
|
|
63
|
+
base_subjects(subject.file)[subject.id]
|
|
64
|
+
end
|
|
65
|
+
|
|
66
|
+
# Only the names defined once: with two defs of a name, which one a
|
|
67
|
+
# change belongs to is a guess.
|
|
68
|
+
def unique_by_id(source, file)
|
|
69
|
+
subjects = SubjectFinder.new.call(file, source: source).subjects
|
|
70
|
+
counts = subjects.map(&:id).tally
|
|
71
|
+
subjects.select { |subject| counts[subject.id] == 1 }.to_h { |subject| [subject.id, subject] }
|
|
72
|
+
end
|
|
73
|
+
|
|
74
|
+
def current_subjects(subject)
|
|
75
|
+
file = subject.file
|
|
76
|
+
@current_subjects.fetch(file) { @current_subjects[file] = unique_by_id(subject.file_source, file) }
|
|
77
|
+
end
|
|
78
|
+
|
|
79
|
+
def base_subjects(file)
|
|
80
|
+
@base_subjects.fetch(file) do
|
|
81
|
+
text = Selection::GitDiff.file_at(@base, file.delete_prefix("#{@root}/"), root: @root)
|
|
82
|
+
@base_subjects[file] = text ? unique_by_id(text, file) : {}
|
|
83
|
+
end
|
|
84
|
+
end
|
|
85
|
+
end
|
|
86
|
+
end
|
|
87
|
+
end
|