saboteur 0.3.0 → 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +280 -1
- data/README.md +635 -77
- data/exe/saboteur +6 -2
- data/lib/generators/saboteur/install/install_generator.rb +21 -0
- data/lib/saboteur/agent_skill.rb +46 -0
- data/lib/saboteur/baseline/acceptance.rb +218 -0
- data/lib/saboteur/baseline/command.rb +80 -0
- data/lib/saboteur/baseline/comparison.rb +105 -0
- data/lib/saboteur/baseline/completeness.rb +53 -0
- data/lib/saboteur/baseline/evidence.rb +114 -0
- data/lib/saboteur/baseline/identity.rb +78 -0
- data/lib/saboteur/baseline/manifest.rb +164 -0
- data/lib/saboteur/baseline/run_evidence.rb +117 -0
- data/lib/saboteur/baseline/workflow.rb +121 -0
- data/lib/saboteur/baseline_health.rb +1 -1
- data/lib/saboteur/cli/exit_status.rb +93 -0
- data/lib/saboteur/cli/run_reporting.rb +75 -0
- data/lib/saboteur/cli.rb +171 -63
- data/lib/saboteur/config.rb +101 -19
- data/lib/saboteur/config_file.rb +97 -0
- data/lib/saboteur/custom_map.rb +170 -0
- data/lib/saboteur/doctor.rb +57 -9
- data/lib/saboteur/engine/baseline_guard.rb +44 -5
- data/lib/saboteur/engine/boot.rb +9 -0
- data/lib/saboteur/engine/bytecode_equivalence.rb +95 -0
- data/lib/saboteur/engine/canary.rb +74 -0
- data/lib/saboteur/engine/candidate_selection.rb +18 -5
- data/lib/saboteur/engine/candidates.rb +59 -7
- data/lib/saboteur/engine/coverage_store.rb +53 -6
- data/lib/saboteur/engine/data_inputs.rb +43 -0
- data/lib/saboteur/engine/diagnostics.rb +176 -2
- data/lib/saboteur/engine/diff_scope.rb +18 -0
- data/lib/saboteur/engine/distribution.rb +54 -13
- data/lib/saboteur/engine/result_cache.rb +10 -5
- data/lib/saboteur/engine/run_fingerprint.rb +40 -10
- data/lib/saboteur/engine/sandbox_policy.rb +108 -0
- data/lib/saboteur/engine/scheduler.rb +99 -36
- data/lib/saboteur/engine/subject_selection.rb +24 -1
- data/lib/saboteur/engine/suite_files.rb +8 -1
- data/lib/saboteur/engine/test_guess.rb +52 -0
- data/lib/saboteur/engine/unchanged_syntax.rb +87 -0
- data/lib/saboteur/engine/verdict_check.rb +7 -6
- data/lib/saboteur/engine.rb +257 -22
- data/lib/saboteur/execution/protocol.rb +2 -1
- data/lib/saboteur/execution/reevaluator.rb +79 -14
- data/lib/saboteur/execution/runner.rb +120 -17
- data/lib/saboteur/execution/sandbox.rb +271 -0
- data/lib/saboteur/execution/stale_captures.rb +43 -0
- data/lib/saboteur/execution/supervisor.rb +113 -9
- data/lib/saboteur/execution/worker_pool.rb +20 -4
- data/lib/saboteur/execution/wrapper.rb +45 -0
- data/lib/saboteur/frameworks/rails.rb +329 -13
- data/lib/saboteur/gates.rb +83 -0
- data/lib/saboteur/help.rb +63 -23
- data/lib/saboteur/ignore_list.rb +76 -17
- data/lib/saboteur/inline_suppression.rb +65 -0
- data/lib/saboteur/install.rb +132 -0
- data/lib/saboteur/integration/base.rb +1 -0
- data/lib/saboteur/integration/focus_guard.rb +83 -0
- data/lib/saboteur/integration/rspec.rb +68 -4
- data/lib/saboteur/mutation_source/argument_removal.rb +86 -0
- data/lib/saboteur/mutation_source/builtin.rb +30 -5
- data/lib/saboteur/mutation_source/chain_removal.rb +80 -0
- data/lib/saboteur/mutation_source/condition_negation.rb +50 -0
- data/lib/saboteur/mutation_source/endpoint_replacement.rb +61 -0
- data/lib/saboteur/mutation_source/hazard_guard.rb +136 -0
- data/lib/saboteur/mutation_source/operator_assignment.rb +52 -0
- data/lib/saboteur/mutation_source/operator_registry.rb +57 -0
- data/lib/saboteur/mutation_source/pair_removal.rb +101 -0
- data/lib/saboteur/mutation_source/range_replacement.rb +43 -0
- data/lib/saboteur/preview.rb +106 -0
- data/lib/saboteur/replay.rb +1 -1
- data/lib/saboteur/reporter/agent.rb +87 -0
- data/lib/saboteur/reporter/agent_clipping.rb +29 -0
- data/lib/saboteur/reporter/agent_commands.rb +67 -0
- data/lib/saboteur/reporter/agent_incomplete.rb +60 -0
- data/lib/saboteur/reporter/agent_summary.rb +137 -0
- data/lib/saboteur/reporter/findings.rb +12 -19
- data/lib/saboteur/reporter/history_paths.rb +38 -0
- data/lib/saboteur/reporter/history_record.rb +132 -0
- data/lib/saboteur/reporter/html.rb +61 -2
- data/lib/saboteur/reporter/mutation_text.rb +36 -0
- data/lib/saboteur/reporter/next_steps.rb +30 -0
- data/lib/saboteur/reporter/palette.rb +37 -0
- data/lib/saboteur/reporter/score_history.rb +123 -0
- data/lib/saboteur/reporter/score_table.rb +57 -0
- data/lib/saboteur/reporter/stryker.rb +25 -5
- data/lib/saboteur/reporter/survivor_hints.rb +68 -0
- data/lib/saboteur/reporter/survivor_list.rb +126 -0
- data/lib/saboteur/reporter/survivor_ranking.rb +34 -0
- data/lib/saboteur/reporter/terminal.rb +157 -42
- data/lib/saboteur/reporter/test_matrix.rb +57 -0
- data/lib/saboteur/result.rb +86 -10
- data/lib/saboteur/score.rb +89 -0
- data/lib/saboteur/selection/coverage_map.rb +119 -13
- data/lib/saboteur/selection/git_diff.rb +28 -1
- data/lib/saboteur/selection/recorder.rb +55 -13
- data/lib/saboteur/subject.rb +40 -4
- data/lib/saboteur/subject_finder.rb +62 -12
- data/lib/saboteur/version.rb +1 -1
- data/lib/saboteur/working_tree.rb +56 -0
- data/lib/saboteur.rb +10 -0
- metadata +58 -2
|
@@ -16,21 +16,87 @@ module Saboteur
|
|
|
16
16
|
# to every test in the file of the next test that starts. Each test's
|
|
17
17
|
# duration is recorded so selected tests can run fastest first.
|
|
18
18
|
class CoverageMap
|
|
19
|
-
FORMAT =
|
|
19
|
+
FORMAT = 8
|
|
20
20
|
|
|
21
|
-
|
|
21
|
+
RAN_AT_LOAD = "ran at load: the code ran while the app loaded; a forked test cannot observe a change here"
|
|
22
|
+
|
|
23
|
+
attr_reader :lines, :tests, :build_seconds, :durations, :methods, :load_lines, :branches
|
|
22
24
|
|
|
23
25
|
# lines: { "/abs/file.rb" => { 12 => ["id1", "id2"] } }
|
|
26
|
+
# methods: { "/abs/file.rb" => { 12 => ["id1", "id2"] } } keyed by the
|
|
27
|
+
# def's first line; line coverage never counts an endless def's body,
|
|
28
|
+
# so methods carry those subjects (issue #107).
|
|
24
29
|
# durations: { "id1" => seconds }
|
|
25
|
-
def initialize(lines:, tests:, build_seconds: nil, durations: {})
|
|
30
|
+
def initialize(lines:, tests:, build_seconds: nil, durations: {}, methods: {}, load_lines: {}, branches: {})
|
|
26
31
|
@lines = lines
|
|
27
32
|
@tests = tests
|
|
28
33
|
@build_seconds = build_seconds
|
|
29
34
|
@durations = durations
|
|
35
|
+
@methods = methods
|
|
36
|
+
@load_lines = load_lines
|
|
37
|
+
@branches = branches
|
|
30
38
|
end
|
|
31
39
|
|
|
32
40
|
def tests_for(subject)
|
|
33
|
-
tests_for_lines(subject.file, subject.coverage_lines)
|
|
41
|
+
tests = tests_for_lines(subject.file, subject.coverage_lines)
|
|
42
|
+
return tests unless tests.empty?
|
|
43
|
+
|
|
44
|
+
by_method = methods[CoverageMap.canonical(subject.file)]
|
|
45
|
+
return [] unless by_method
|
|
46
|
+
|
|
47
|
+
span = subject.start_line..subject.end_line
|
|
48
|
+
ids = by_method.flat_map { |line, method_ids| span.cover?(line) ? method_ids : [] }.uniq
|
|
49
|
+
sort_test_ids(ids)
|
|
50
|
+
end
|
|
51
|
+
|
|
52
|
+
# Attach the parent's boot counters without losing the child's method coverage.
|
|
53
|
+
def with_load_lines(load_lines)
|
|
54
|
+
self.class.new(lines: lines, tests: tests, build_seconds: build_seconds, durations: durations,
|
|
55
|
+
methods: methods, load_lines: load_lines, branches: branches)
|
|
56
|
+
end
|
|
57
|
+
|
|
58
|
+
# A requested line ran at load but no test later hit it.
|
|
59
|
+
def ran_at_load?(file, line_numbers)
|
|
60
|
+
at_load = load_lines[CoverageMap.canonical(file)]
|
|
61
|
+
return false unless at_load
|
|
62
|
+
|
|
63
|
+
covered = lines[CoverageMap.canonical(file)] || {}
|
|
64
|
+
line_numbers.any? { |line| at_load.include?(line) && !covered.key?(line) }
|
|
65
|
+
end
|
|
66
|
+
|
|
67
|
+
# Tests for the lines of one edit. A branch no test executes keeps its
|
|
68
|
+
# "no coverage" skip, but a subject whose lines never record at all
|
|
69
|
+
# (an endless def) falls back to its method coverage (issue #107).
|
|
70
|
+
def tests_for_edit(subject, line_numbers)
|
|
71
|
+
tests = tests_for_lines(subject.file, line_numbers)
|
|
72
|
+
return tests unless tests.empty?
|
|
73
|
+
return [] unless tests_for_lines(subject.file, subject.coverage_lines).empty?
|
|
74
|
+
|
|
75
|
+
tests_for(subject)
|
|
76
|
+
end
|
|
77
|
+
|
|
78
|
+
# nil means Ruby supplied no arm evidence for this edit (including &&/||).
|
|
79
|
+
# An empty list means a known arm was never taken. Nested arms intersect.
|
|
80
|
+
def tests_for_branches(subject, edits)
|
|
81
|
+
arms = branches.fetch(CoverageMap.canonical(subject.file), {})
|
|
82
|
+
matching = edits.map do |edit|
|
|
83
|
+
first = position(subject.file_source, edit.start_byte)
|
|
84
|
+
last = position(subject.file_source, edit.end_byte)
|
|
85
|
+
selected = arms.select do |span, ids|
|
|
86
|
+
(span.first(2) <=> first) <= 0 && (last <=> span.last(2)) <= 0 &&
|
|
87
|
+
!ran_before_tests?(subject.file, span, ids)
|
|
88
|
+
end
|
|
89
|
+
selected.values.reduce { |ids, other| ids & other }
|
|
90
|
+
end
|
|
91
|
+
return nil if matching.include?(nil)
|
|
92
|
+
|
|
93
|
+
sort_test_ids(matching.flatten.uniq)
|
|
94
|
+
end
|
|
95
|
+
|
|
96
|
+
# True when neither boot nor tests recorded included code. Load-only
|
|
97
|
+
# evidence is valid, even though it supplies no judged mutation score.
|
|
98
|
+
def empty?
|
|
99
|
+
[lines, methods, load_lines].all? { |hits| hits.none? { |_file, values| values.any? } }
|
|
34
100
|
end
|
|
35
101
|
|
|
36
102
|
# Tests that executed any of the lines, fastest first (suite order breaks
|
|
@@ -46,19 +112,19 @@ module Saboteur
|
|
|
46
112
|
by_line = lines[CoverageMap.canonical(file)]
|
|
47
113
|
return [] unless by_line
|
|
48
114
|
|
|
49
|
-
|
|
50
|
-
order = test_order
|
|
51
|
-
ids.sort_by { |id| [durations.fetch(id, Float::INFINITY), order.fetch(id, Float::INFINITY)] }
|
|
115
|
+
sort_test_ids(line_numbers.flat_map { |line| by_line.fetch(line, []) }.uniq)
|
|
52
116
|
end
|
|
53
117
|
|
|
54
118
|
# Lines refer to tests by their index in tests, which keeps the cache
|
|
55
119
|
# small for suites with long test ids.
|
|
56
120
|
def to_json(*)
|
|
57
121
|
index = test_order
|
|
58
|
-
|
|
59
|
-
by_line.to_h { |line, ids| [line.to_s, ids.map { |id| index.fetch(id) }] }
|
|
122
|
+
encode = lambda do |hits|
|
|
123
|
+
hits.transform_values { |by_line| by_line.to_h { |line, ids| [line.to_s, ids.map { |id| index.fetch(id) }] } }
|
|
60
124
|
end
|
|
61
|
-
JSON.generate(format: FORMAT, tests: tests, build_seconds: build_seconds, lines:
|
|
125
|
+
JSON.generate(format: FORMAT, tests: tests, build_seconds: build_seconds, lines: encode.call(lines),
|
|
126
|
+
methods: encode.call(methods), load_lines: load_lines,
|
|
127
|
+
branches: encode_branches(index),
|
|
62
128
|
durations: tests.map { |id| durations[id]&.round(6) })
|
|
63
129
|
end
|
|
64
130
|
|
|
@@ -67,11 +133,22 @@ module Saboteur
|
|
|
67
133
|
raise Error, "coverage map format #{data['format']} is not #{FORMAT}" unless data["format"] == FORMAT
|
|
68
134
|
|
|
69
135
|
tests = data["tests"]
|
|
70
|
-
|
|
71
|
-
|
|
136
|
+
decode = lambda do |hits|
|
|
137
|
+
hits.transform_values do |by_line|
|
|
138
|
+
by_line.to_h { |line, indexes| [line.to_i, indexes.map { |i| tests.fetch(i) }] }
|
|
139
|
+
end
|
|
72
140
|
end
|
|
73
141
|
durations = tests.zip(data["durations"]).to_h.compact
|
|
74
|
-
|
|
142
|
+
load_lines = (data["load_lines"] || {}).transform_values { |list| list.map(&:to_i) }
|
|
143
|
+
new(lines: decode.call(data["lines"]), tests: tests, build_seconds: data["build_seconds"],
|
|
144
|
+
durations: durations, methods: decode.call(data["methods"] || {}), load_lines: load_lines,
|
|
145
|
+
branches: decode_branches(data.fetch("branches", {}), tests))
|
|
146
|
+
end
|
|
147
|
+
|
|
148
|
+
def self.decode_branches(branches, tests)
|
|
149
|
+
branches.transform_values do |arms|
|
|
150
|
+
arms.to_h { |span, ids| [span, ids.map { |id| tests.fetch(id) }] }
|
|
151
|
+
end
|
|
75
152
|
end
|
|
76
153
|
|
|
77
154
|
# Cache key over every input that can change which test runs which line.
|
|
@@ -105,6 +182,35 @@ module Saboteur
|
|
|
105
182
|
|
|
106
183
|
private
|
|
107
184
|
|
|
185
|
+
def encode_branches(index)
|
|
186
|
+
branches.transform_values do |arms|
|
|
187
|
+
arms.map { |span, ids| [span, ids.map { |id| index.fetch(id) }] }
|
|
188
|
+
end
|
|
189
|
+
end
|
|
190
|
+
|
|
191
|
+
# An arm whose body the app ran while loading (a def inside an if) has no
|
|
192
|
+
# per-test hits, yet tests ran the lines inside it. Only lines strictly
|
|
193
|
+
# between the arm's first and last line count: the boundary lines can
|
|
194
|
+
# also hold code outside the arm. An arm with no interior line, such as
|
|
195
|
+
# a one-line arm, cannot be told apart, so its empty list still means
|
|
196
|
+
# "not taken".
|
|
197
|
+
def ran_before_tests?(file, span, ids)
|
|
198
|
+
return false unless ids.empty?
|
|
199
|
+
|
|
200
|
+
covered = lines[CoverageMap.canonical(file)] || {}
|
|
201
|
+
((span[0] + 1)...span[2]).any? { |line| covered.key?(line) }
|
|
202
|
+
end
|
|
203
|
+
|
|
204
|
+
def position(source, offset)
|
|
205
|
+
prefix = source.byteslice(0, offset).b
|
|
206
|
+
[prefix.count("\n") + 1, offset - ((prefix.rindex("\n") || -1) + 1)]
|
|
207
|
+
end
|
|
208
|
+
|
|
209
|
+
def sort_test_ids(ids)
|
|
210
|
+
order = test_order
|
|
211
|
+
ids.sort_by { |id| [durations.fetch(id, Float::INFINITY), order.fetch(id, Float::INFINITY)] }
|
|
212
|
+
end
|
|
213
|
+
|
|
108
214
|
def test_order
|
|
109
215
|
@test_order ||= tests.each_with_index.to_h
|
|
110
216
|
end
|
|
@@ -13,8 +13,35 @@ module Saboteur
|
|
|
13
13
|
|
|
14
14
|
module_function
|
|
15
15
|
|
|
16
|
+
# Fails before any boot work when REF is not a commit here, as a usage
|
|
17
|
+
# error (exit 2): the name is wrong, or a shallow checkout lacks it.
|
|
18
|
+
def verify_ref!(ref, root:)
|
|
19
|
+
raise ConfigError, "--since needs a git ref, not an option: #{ref}" if ref.start_with?("-")
|
|
20
|
+
|
|
21
|
+
_, _, inside = Open3.capture3("git", "rev-parse", "--git-dir", chdir: root)
|
|
22
|
+
raise ConfigError, "--since needs a git repository" unless inside.success?
|
|
23
|
+
|
|
24
|
+
_, _, found = Open3.capture3("git", "rev-parse", "--verify", "--quiet", "#{ref}^{commit}", chdir: root)
|
|
25
|
+
return if found.success?
|
|
26
|
+
|
|
27
|
+
raise ConfigError, "--since #{ref}: not a commit in this repository " \
|
|
28
|
+
"(check the name; a shallow checkout may need a deeper fetch)"
|
|
29
|
+
end
|
|
30
|
+
|
|
31
|
+
def merge_base(ref, root:)
|
|
32
|
+
git(root, "merge-base", ref, "HEAD").strip
|
|
33
|
+
end
|
|
34
|
+
|
|
35
|
+
# The file's text at commit BASE, or nil when it did not exist there.
|
|
36
|
+
# path is relative to root.
|
|
37
|
+
def file_at(base, path, root:)
|
|
38
|
+
out, _, status = Open3.capture3("git", "show", "#{base}:./#{path}", chdir: root)
|
|
39
|
+
status.success? ? out.b : nil
|
|
40
|
+
end
|
|
41
|
+
|
|
16
42
|
def changed_lines(ref, root:)
|
|
17
|
-
|
|
43
|
+
verify_ref!(ref, root: root)
|
|
44
|
+
base = merge_base(ref, root: root)
|
|
18
45
|
top = git(root, "rev-parse", "--show-toplevel").strip
|
|
19
46
|
parse(git(root, "-c", "core.quotePath=false", "diff", "--unified=0", "--no-color", "--no-ext-diff", base,
|
|
20
47
|
"--"), top)
|
|
@@ -12,10 +12,14 @@ module Saboteur
|
|
|
12
12
|
def initialize(files)
|
|
13
13
|
@files = Set.new(files.map { |file| CoverageMap.canonical(file) })
|
|
14
14
|
@canonical = Hash.new { |hash, path| hash[path] = CoverageMap.canonical(path) }
|
|
15
|
-
@lines =
|
|
15
|
+
@lines = hit_table
|
|
16
|
+
@methods = hit_table
|
|
17
|
+
@branches = hit_table
|
|
18
|
+
@outside_branches = per_file { Set.new }
|
|
16
19
|
@tests = []
|
|
17
|
-
@tests_by_file =
|
|
18
|
-
@outside =
|
|
20
|
+
@tests_by_file = per_file { [] }
|
|
21
|
+
@outside = per_file { Set.new }
|
|
22
|
+
@outside_methods = per_file { Set.new }
|
|
19
23
|
@durations = {}
|
|
20
24
|
end
|
|
21
25
|
|
|
@@ -27,33 +31,64 @@ module Saboteur
|
|
|
27
31
|
def test_started(id, test_file)
|
|
28
32
|
@tests << id
|
|
29
33
|
@tests_by_file[test_file] << id
|
|
30
|
-
|
|
34
|
+
each_hit do |file, line, kind|
|
|
35
|
+
{ line: @outside, method: @outside_methods, branch: @outside_branches }.fetch(kind)[test_file] << [file, line]
|
|
36
|
+
end
|
|
31
37
|
@started_at = monotonic
|
|
32
38
|
end
|
|
33
39
|
|
|
34
40
|
def test_finished(id)
|
|
35
41
|
@durations[id] = monotonic - @started_at if @started_at
|
|
36
|
-
|
|
42
|
+
each_hit do |file, line, kind|
|
|
43
|
+
{ line: @lines, method: @methods, branch: @branches }.fetch(kind)[file][line] << id
|
|
44
|
+
end
|
|
37
45
|
end
|
|
38
46
|
|
|
39
47
|
def result(build_seconds: nil)
|
|
40
|
-
@outside
|
|
41
|
-
|
|
42
|
-
|
|
43
|
-
end
|
|
48
|
+
assign_outside(@outside, @lines)
|
|
49
|
+
assign_outside(@outside_methods, @methods)
|
|
50
|
+
assign_outside(@outside_branches, @branches)
|
|
44
51
|
lines = @lines.transform_values { |by_line| by_line.transform_values(&:uniq) }
|
|
45
|
-
|
|
52
|
+
methods = @methods.transform_values { |by_line| by_line.transform_values(&:uniq) }
|
|
53
|
+
CoverageMap.new(lines: lines, tests: @tests, build_seconds: build_seconds, durations: @durations,
|
|
54
|
+
methods: methods, branches: @branches.transform_values { |arms| arms.transform_values(&:uniq) })
|
|
46
55
|
end
|
|
47
56
|
|
|
48
57
|
private
|
|
49
58
|
|
|
59
|
+
def assign_outside(outside, table)
|
|
60
|
+
outside.each do |test_file, hits|
|
|
61
|
+
ids = @tests_by_file[test_file]
|
|
62
|
+
hits.each { |file, line| table[file][line] |= ids }
|
|
63
|
+
end
|
|
64
|
+
end
|
|
65
|
+
|
|
66
|
+
def record_branches(file, data)
|
|
67
|
+
data.fetch(:branches, {}).each do |(_kind, _id, *parent_span), arms|
|
|
68
|
+
arms.each do |(_arm_kind, _arm_id, *span), count|
|
|
69
|
+
next if span == parent_span
|
|
70
|
+
|
|
71
|
+
@branches[file][span] # Keep zero-hit arms as evidence, not as covered code.
|
|
72
|
+
yield span if count.positive?
|
|
73
|
+
end
|
|
74
|
+
end
|
|
75
|
+
end
|
|
76
|
+
|
|
77
|
+
def hit_table
|
|
78
|
+
Hash.new { |hash, file| hash[file] = Hash.new { |by_line, line| by_line[line] = [] } }
|
|
79
|
+
end
|
|
80
|
+
|
|
81
|
+
def per_file(&block)
|
|
82
|
+
Hash.new { |hash, file| hash[file] = block.call }
|
|
83
|
+
end
|
|
84
|
+
|
|
50
85
|
def monotonic
|
|
51
86
|
Process.clock_gettime(Process::CLOCK_MONOTONIC)
|
|
52
87
|
end
|
|
53
88
|
|
|
54
89
|
# Coverage is suspended while the delta is processed, so the recorder's
|
|
55
90
|
# own work (and anything it calls) is never attributed to a test.
|
|
56
|
-
def
|
|
91
|
+
def each_hit
|
|
57
92
|
snapshot = Coverage.result(stop: false, clear: true)
|
|
58
93
|
suspended = Coverage.running?
|
|
59
94
|
Coverage.suspend if suspended
|
|
@@ -61,8 +96,15 @@ module Saboteur
|
|
|
61
96
|
file = @canonical[path]
|
|
62
97
|
next unless @files.include?(file)
|
|
63
98
|
|
|
64
|
-
# Lines mode (set up by the executable); nil marks a line with no
|
|
65
|
-
|
|
99
|
+
# Lines mode (set up by the executable); nil marks a line with no
|
|
100
|
+
# code. Methods mode keys are [owner, name, start_line, ...]; an
|
|
101
|
+
# endless def's body never counts a line hit, but its calls count
|
|
102
|
+
# here (issue #107).
|
|
103
|
+
data[:lines].each_with_index { |count, index| yield file, index + 1, :line if count&.positive? }
|
|
104
|
+
record_branches(file, data) { |span| yield file, span, :branch }
|
|
105
|
+
data[:methods]&.each do |(_owner, _name, start_line, *), count|
|
|
106
|
+
yield file, start_line, :method if count.positive?
|
|
107
|
+
end
|
|
66
108
|
end
|
|
67
109
|
ensure
|
|
68
110
|
Coverage.resume if suspended
|
data/lib/saboteur/subject.rb
CHANGED
|
@@ -24,10 +24,11 @@ module Saboteur
|
|
|
24
24
|
# the latter, because the def text must then be evaluated on the singleton.
|
|
25
25
|
class Subject
|
|
26
26
|
attr_reader :name, :owner, :singleton, :sclass, :file, :start_byte, :end_byte,
|
|
27
|
-
:start_line, :end_line, :nesting, :node, :file_source, :frozen_literals
|
|
27
|
+
:start_line, :end_line, :nesting, :node, :file_source, :frozen_literals, :sig_span
|
|
28
28
|
attr_accessor :skip_reason
|
|
29
29
|
|
|
30
|
-
def initialize(name:, owner:, singleton:, sclass:, file:, node:, file_source:, nesting:, frozen_literals
|
|
30
|
+
def initialize(name:, owner:, singleton:, sclass:, file:, node:, file_source:, nesting:, frozen_literals:,
|
|
31
|
+
sig_span: nil)
|
|
31
32
|
@name = name
|
|
32
33
|
@owner = owner
|
|
33
34
|
@singleton = singleton
|
|
@@ -41,9 +42,20 @@ module Saboteur
|
|
|
41
42
|
@end_byte = node.location.end_offset
|
|
42
43
|
@start_line = node.location.start_line
|
|
43
44
|
@end_line = node.location.end_line
|
|
45
|
+
@sig_span = sig_span
|
|
44
46
|
@skip_reason = nil
|
|
45
47
|
end
|
|
46
48
|
|
|
49
|
+
# The `sig { ... }` call right before the def, when there is one. A
|
|
50
|
+
# sorbet-wrapped method re-evaluates sig and def together, so the
|
|
51
|
+
# runtime type checks stay on for the mutant (see Execution::Wrapper).
|
|
52
|
+
def sig_source
|
|
53
|
+
return nil unless @sig_span
|
|
54
|
+
|
|
55
|
+
first, last = @sig_span
|
|
56
|
+
file_source.byteslice(first, last - first).force_encoding(Encoding::UTF_8)
|
|
57
|
+
end
|
|
58
|
+
|
|
47
59
|
def id
|
|
48
60
|
"#{owner}#{singleton ? '.' : '#'}#{name}"
|
|
49
61
|
end
|
|
@@ -71,11 +83,14 @@ module Saboteur
|
|
|
71
83
|
|
|
72
84
|
# Lines whose execution means the edited code ran: from the first line of
|
|
73
85
|
# the innermost statement containing the edit to the edit's last line.
|
|
74
|
-
#
|
|
75
|
-
#
|
|
86
|
+
# Pattern headers do not always record line events. Their enclosing case
|
|
87
|
+
# and clause body supply execution evidence instead.
|
|
76
88
|
# An edit outside every statement (signature, around the def, whole
|
|
77
89
|
# body) maps to the whole method.
|
|
78
90
|
def coverage_lines_for(edit)
|
|
91
|
+
pattern_lines = pattern_coverage_lines(node.body, edit)
|
|
92
|
+
return pattern_lines if pattern_lines
|
|
93
|
+
|
|
79
94
|
statement = innermost_statement(node.body, edit.start_byte, edit.end_byte)
|
|
80
95
|
return coverage_lines unless statement
|
|
81
96
|
|
|
@@ -153,6 +168,27 @@ module Saboteur
|
|
|
153
168
|
found
|
|
154
169
|
end
|
|
155
170
|
|
|
171
|
+
def pattern_coverage_lines(child, edit, case_line = nil)
|
|
172
|
+
return nil unless child
|
|
173
|
+
|
|
174
|
+
case_line = child.location.start_line if child.is_a?(Prism::CaseMatchNode)
|
|
175
|
+
if child.is_a?(Prism::InNode) && contains_edit?(child.pattern, edit)
|
|
176
|
+
return [case_line, child.location.start_line, child.statements&.location&.start_line].compact.uniq
|
|
177
|
+
end
|
|
178
|
+
|
|
179
|
+
child.compact_child_nodes.each do |nested|
|
|
180
|
+
next unless contains_edit?(nested, edit)
|
|
181
|
+
|
|
182
|
+
lines = pattern_coverage_lines(nested, edit, case_line)
|
|
183
|
+
return lines if lines
|
|
184
|
+
end
|
|
185
|
+
nil
|
|
186
|
+
end
|
|
187
|
+
|
|
188
|
+
def contains_edit?(child, edit)
|
|
189
|
+
child.location.start_offset <= edit.start_byte && edit.end_byte <= child.location.end_offset
|
|
190
|
+
end
|
|
191
|
+
|
|
156
192
|
def validate_edit!(edit, previous_start)
|
|
157
193
|
unless File.expand_path(edit.file) == File.expand_path(file)
|
|
158
194
|
raise ArgumentError, "edit for #{edit.file} applied to subject in #{file}"
|
|
@@ -9,13 +9,21 @@ module Saboteur
|
|
|
9
9
|
#
|
|
10
10
|
# Included: instance methods and `def self.x` inside a class or module,
|
|
11
11
|
# defs inside `class << self`, and defs passed to a receiverless call such
|
|
12
|
-
# as `private def x`. Skipped: top-level defs, defs nested in defs,
|
|
13
|
-
# blocks (`Class.new do`, `included do`)
|
|
12
|
+
# as `private def x`. Skipped: top-level defs, defs nested in defs, and
|
|
13
|
+
# `def obj.x` for obj != self. Defs in blocks (`Class.new do`, `included do`)
|
|
14
|
+
# are found but marked skipped ("defined inside a block"), so the report
|
|
15
|
+
# names them instead of leaving them out. `X = Data.define`, `Struct.new`,
|
|
16
|
+
# `Class.new` or `Module.new` with a block names X as the owner; any other
|
|
17
|
+
# block, and a `A::X = ...` target, keeps the enclosing owner or the
|
|
18
|
+
# placeholder.
|
|
14
19
|
class SubjectFinder
|
|
15
20
|
Parsed = Data.define(:subjects, :errors)
|
|
21
|
+
ANONYMOUS_OWNER = "(anonymous block)"
|
|
22
|
+
CLASS_FACTORIES = { Data: :define, Struct: :new, Class: :new, Module: :new }.freeze
|
|
16
23
|
|
|
17
|
-
|
|
18
|
-
|
|
24
|
+
# source: the text to parse when it is not the file on disk (the file as
|
|
25
|
+
# it was at an earlier commit).
|
|
26
|
+
def call(file, source: File.binread(file))
|
|
19
27
|
result = Prism.parse(source, filepath: file)
|
|
20
28
|
return Parsed.new(subjects: [], errors: result.errors.map(&:message)) if result.failure?
|
|
21
29
|
|
|
@@ -34,23 +42,53 @@ module Saboteur
|
|
|
34
42
|
end
|
|
35
43
|
end
|
|
36
44
|
|
|
37
|
-
def walk(node, nesting, out, **ctx)
|
|
45
|
+
def walk(node, nesting, out, previous: nil, **ctx)
|
|
46
|
+
return if node.nil?
|
|
47
|
+
|
|
38
48
|
case node
|
|
39
|
-
when
|
|
40
|
-
|
|
49
|
+
when Prism::BlockNode, Prism::LambdaNode
|
|
50
|
+
walk(node.body, nesting, out, **ctx, in_block: true)
|
|
51
|
+
when Prism::ConstantWriteNode
|
|
52
|
+
walk_constant_assignment(node, nesting, out, **ctx)
|
|
41
53
|
when Prism::ModuleNode
|
|
42
54
|
walk(node.body, nesting + [Scope.new(kind: :module, path: node.constant_path.slice)], out, **ctx)
|
|
43
55
|
when Prism::ClassNode
|
|
44
56
|
walk(node.body, nesting + [Scope.new(kind: :class, path: node.constant_path.slice)], out, **ctx)
|
|
45
57
|
when Prism::SingletonClassNode
|
|
46
58
|
walk_singleton_class(node, nesting, out, **ctx)
|
|
59
|
+
when Prism::StatementsNode
|
|
60
|
+
previous_child = nil
|
|
61
|
+
node.body.each do |child|
|
|
62
|
+
walk(child, nesting, out, previous: previous_child, **ctx)
|
|
63
|
+
previous_child = child
|
|
64
|
+
end
|
|
47
65
|
when Prism::DefNode
|
|
48
|
-
add_subject(node, nesting, out, **ctx)
|
|
66
|
+
add_subject(node, nesting, out, previous: previous, **ctx)
|
|
49
67
|
else
|
|
50
68
|
node.compact_child_nodes.each { |child| walk(child, nesting, out, **ctx) }
|
|
51
69
|
end
|
|
52
70
|
end
|
|
53
71
|
|
|
72
|
+
# `Point = Data.define(:x) do ... end`: the defs belong to Point. Only the
|
|
73
|
+
# four core factories count, called on the bare constant; any other call
|
|
74
|
+
# that takes a block says nothing about the class it returns.
|
|
75
|
+
def walk_constant_assignment(node, nesting, out, **ctx)
|
|
76
|
+
call = node.value
|
|
77
|
+
return node.compact_child_nodes.each { |child| walk(child, nesting, out, **ctx) } unless class_factory?(call)
|
|
78
|
+
|
|
79
|
+
# The receiver and arguments stay in the enclosing scope; only the
|
|
80
|
+
# factory's own block belongs to the new class.
|
|
81
|
+
(call.compact_child_nodes - [call.block]).each { |child| walk(child, nesting, out, **ctx) }
|
|
82
|
+
walk(call.block, nesting + [Scope.new(kind: :class, path: node.name.to_s)], out, **ctx)
|
|
83
|
+
end
|
|
84
|
+
|
|
85
|
+
def class_factory?(call)
|
|
86
|
+
return false unless call.is_a?(Prism::CallNode) && call.block.is_a?(Prism::BlockNode)
|
|
87
|
+
return false unless call.receiver.is_a?(Prism::ConstantReadNode)
|
|
88
|
+
|
|
89
|
+
CLASS_FACTORIES[call.receiver.name] == call.name
|
|
90
|
+
end
|
|
91
|
+
|
|
54
92
|
def walk_singleton_class(node, nesting, out, **ctx)
|
|
55
93
|
return unless node.expression.is_a?(Prism::SelfNode)
|
|
56
94
|
return if nesting.empty? || nesting.last.sclass?
|
|
@@ -58,18 +96,30 @@ module Saboteur
|
|
|
58
96
|
walk(node.body, nesting + [Scope.new(kind: :sclass, path: "self")], out, **ctx)
|
|
59
97
|
end
|
|
60
98
|
|
|
61
|
-
def add_subject(node, nesting, out, file:, source:, frozen:)
|
|
99
|
+
def add_subject(node, nesting, out, file:, source:, frozen:, previous: nil, in_block: false)
|
|
62
100
|
in_sclass = nesting.last&.sclass? || false
|
|
63
101
|
# `def self.x` inside `class << self`, and `def obj.x`, are not subjects.
|
|
64
102
|
return unless node.receiver.nil? || (node.receiver.is_a?(Prism::SelfNode) && !in_sclass)
|
|
65
103
|
|
|
66
104
|
singleton = in_sclass || !node.receiver.nil?
|
|
67
105
|
owner = owner_path(nesting)
|
|
106
|
+
owner ||= ANONYMOUS_OWNER if in_block
|
|
68
107
|
return unless owner
|
|
69
108
|
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
|
|
109
|
+
subject = Subject.new(name: node.name, owner: owner, singleton: singleton, sclass: in_sclass,
|
|
110
|
+
file: file, node: node, file_source: source, nesting: nesting,
|
|
111
|
+
frozen_literals: frozen, sig_span: sig_span(previous))
|
|
112
|
+
subject.skip_reason = "defined inside a block" if in_block
|
|
113
|
+
out << subject
|
|
114
|
+
end
|
|
115
|
+
|
|
116
|
+
# The byte span of a `sig { ... }` or `T.sig { ... }` call on the lines
|
|
117
|
+
# right before the def, when there is one.
|
|
118
|
+
def sig_span(previous)
|
|
119
|
+
return nil unless previous.is_a?(Prism::CallNode) && previous.name == :sig && previous.block
|
|
120
|
+
return nil unless previous.receiver.nil? || previous.receiver.slice == "T"
|
|
121
|
+
|
|
122
|
+
[previous.location.start_offset, previous.location.end_offset]
|
|
73
123
|
end
|
|
74
124
|
|
|
75
125
|
# Lexical path of the enclosing class/module. `class ::Foo` restarts at the top.
|
data/lib/saboteur/version.rb
CHANGED
|
@@ -0,0 +1,56 @@
|
|
|
1
|
+
# SPDX-License-Identifier: MIT
|
|
2
|
+
# frozen_string_literal: true
|
|
3
|
+
|
|
4
|
+
require "open3"
|
|
5
|
+
|
|
6
|
+
module Saboteur
|
|
7
|
+
# The state of the working source a run looked at: the commit it was on and
|
|
8
|
+
# (the full SHA) and whether files differed from it, tracked changes and each
|
|
9
|
+
# untracked file alike.
|
|
10
|
+
# Saboteur's own output (cache and reports) is passed in as ignored paths, so a
|
|
11
|
+
# report from an earlier run does not make the tree look dirty. Without git, or
|
|
12
|
+
# outside a repository, the state is unknown.
|
|
13
|
+
class WorkingTree
|
|
14
|
+
State = Data.define(:head, :dirty, :changed_files) do
|
|
15
|
+
def label
|
|
16
|
+
return "unknown" if dirty.nil?
|
|
17
|
+
|
|
18
|
+
dirty ? "dirty" : "clean"
|
|
19
|
+
end
|
|
20
|
+
|
|
21
|
+
# Shown before a run: mutants run your code and can change the tree.
|
|
22
|
+
def warning
|
|
23
|
+
return unless dirty
|
|
24
|
+
|
|
25
|
+
"saboteur: warning: #{changed_files} uncommitted change(s) in the working tree. " \
|
|
26
|
+
"Mutants run your code and could alter or delete files; run on a clean, committed tree or a CI checkout."
|
|
27
|
+
end
|
|
28
|
+
end
|
|
29
|
+
|
|
30
|
+
UNKNOWN = State.new(head: nil, dirty: nil, changed_files: nil)
|
|
31
|
+
|
|
32
|
+
# ignored: paths relative to root that are not source (saboteur's cache and reports).
|
|
33
|
+
def initialize(root, ignored: [])
|
|
34
|
+
@root = root
|
|
35
|
+
@ignored = ignored
|
|
36
|
+
end
|
|
37
|
+
|
|
38
|
+
def state
|
|
39
|
+
head, head_status = Open3.capture2e("git", "rev-parse", "HEAD", chdir: @root)
|
|
40
|
+
changes, changes_status = Open3.capture2e(*status_command, chdir: @root)
|
|
41
|
+
return UNKNOWN unless head_status.success? && changes_status.success?
|
|
42
|
+
|
|
43
|
+
count = changes.lines.size
|
|
44
|
+
State.new(head: head.strip, dirty: count.positive?, changed_files: count)
|
|
45
|
+
rescue SystemCallError
|
|
46
|
+
UNKNOWN
|
|
47
|
+
end
|
|
48
|
+
|
|
49
|
+
private
|
|
50
|
+
|
|
51
|
+
def status_command
|
|
52
|
+
excluded = @ignored.map { |path| ":(exclude)#{path}" }
|
|
53
|
+
["git", "status", "--porcelain", "--untracked-files=all", "--", ".", *excluded]
|
|
54
|
+
end
|
|
55
|
+
end
|
|
56
|
+
end
|
data/lib/saboteur.rb
CHANGED
|
@@ -9,6 +9,9 @@ module Saboteur
|
|
|
9
9
|
# Invalid configuration or input files (exit code 2).
|
|
10
10
|
class ConfigError < Error; end
|
|
11
11
|
|
|
12
|
+
# Files vanished from the working tree while mutants ran (exit code 3).
|
|
13
|
+
class CanaryTripped < Error; end
|
|
14
|
+
|
|
12
15
|
class << self
|
|
13
16
|
# Set while hook files from config load, so they can call Saboteur.hook.
|
|
14
17
|
attr_accessor :current_hooks
|
|
@@ -36,18 +39,25 @@ require_relative "saboteur/execution/process_table"
|
|
|
36
39
|
require_relative "saboteur/execution/process_tree"
|
|
37
40
|
require_relative "saboteur/execution/supervisor"
|
|
38
41
|
require_relative "saboteur/execution/reevaluator"
|
|
42
|
+
require_relative "saboteur/execution/wrapper"
|
|
39
43
|
require_relative "saboteur/execution/runner"
|
|
40
44
|
require_relative "saboteur/execution/worker_pool"
|
|
41
45
|
require_relative "saboteur/execution/fork_readiness"
|
|
46
|
+
require_relative "saboteur/execution/sandbox"
|
|
42
47
|
require_relative "saboteur/selection/coverage_map"
|
|
43
48
|
require_relative "saboteur/selection/recorder"
|
|
44
49
|
require_relative "saboteur/selection/git_diff"
|
|
45
50
|
require_relative "saboteur/result"
|
|
51
|
+
require_relative "saboteur/custom_map"
|
|
52
|
+
require_relative "saboteur/gates"
|
|
46
53
|
require_relative "saboteur/config"
|
|
47
54
|
require_relative "saboteur/ignore_list"
|
|
48
55
|
require_relative "saboteur/frameworks/rails"
|
|
49
56
|
require_relative "saboteur/reporter/diff"
|
|
50
57
|
require_relative "saboteur/reporter/findings"
|
|
58
|
+
require_relative "saboteur/reporter/agent"
|
|
59
|
+
require_relative "saboteur/working_tree"
|
|
60
|
+
require_relative "saboteur/agent_skill"
|
|
51
61
|
require_relative "saboteur/reporter/terminal"
|
|
52
62
|
require_relative "saboteur/reporter/stryker"
|
|
53
63
|
require_relative "saboteur/reporter/html"
|