saboteur 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/CHANGELOG.md +47 -0
- data/LICENSE +21 -0
- data/README.md +145 -0
- data/exe/saboteur +12 -0
- data/lib/saboteur/cli.rb +112 -0
- data/lib/saboteur/config.rb +133 -0
- data/lib/saboteur/engine/boot.rb +83 -0
- data/lib/saboteur/engine/candidates.rb +63 -0
- data/lib/saboteur/engine/coverage_store.rb +99 -0
- data/lib/saboteur/engine/diagnostics.rb +37 -0
- data/lib/saboteur/engine/diff_scope.rb +32 -0
- data/lib/saboteur/engine/distribution.rb +74 -0
- data/lib/saboteur/engine/result_cache.rb +93 -0
- data/lib/saboteur/engine/scheduler.rb +215 -0
- data/lib/saboteur/engine/subject_selection.rb +63 -0
- data/lib/saboteur/engine/suite_files.rb +82 -0
- data/lib/saboteur/engine/verdict_check.rb +78 -0
- data/lib/saboteur/engine.rb +133 -0
- data/lib/saboteur/execution/process_table.rb +177 -0
- data/lib/saboteur/execution/process_tree.rb +99 -0
- data/lib/saboteur/execution/protocol.rb +41 -0
- data/lib/saboteur/execution/reevaluator.rb +144 -0
- data/lib/saboteur/execution/runner.rb +191 -0
- data/lib/saboteur/execution/supervisor.rb +337 -0
- data/lib/saboteur/execution/worker_pool.rb +249 -0
- data/lib/saboteur/frameworks/rails.rb +166 -0
- data/lib/saboteur/hooks.rb +51 -0
- data/lib/saboteur/ignore_list.rb +62 -0
- data/lib/saboteur/integration/base.rb +72 -0
- data/lib/saboteur/integration/rspec.rb +143 -0
- data/lib/saboteur/mutation_source/base.rb +26 -0
- data/lib/saboteur/mutation_source/builtin.rb +232 -0
- data/lib/saboteur/mutation_source/edit.rb +18 -0
- data/lib/saboteur/mutation_source/mutation.rb +28 -0
- data/lib/saboteur/reporter/diff.rb +97 -0
- data/lib/saboteur/reporter/stryker.rb +116 -0
- data/lib/saboteur/reporter/terminal.rb +136 -0
- data/lib/saboteur/result.rb +134 -0
- data/lib/saboteur/selection/coverage_map.rb +113 -0
- data/lib/saboteur/selection/git_diff.rb +64 -0
- data/lib/saboteur/selection/recorder.rb +72 -0
- data/lib/saboteur/subject.rb +166 -0
- data/lib/saboteur/subject_finder.rb +98 -0
- data/lib/saboteur/version.rb +6 -0
- data/lib/saboteur.rb +51 -0
- metadata +130 -0
|
@@ -0,0 +1,136 @@
|
|
|
1
|
+
# SPDX-License-Identifier: MIT
|
|
2
|
+
# frozen_string_literal: true
|
|
3
|
+
|
|
4
|
+
module Saboteur
|
|
5
|
+
module Reporter
|
|
6
|
+
# Plain-text summary: counts per status, score, timings, skip reasons,
|
|
7
|
+
# errors, then a unified diff for each survivor.
|
|
8
|
+
class Terminal
|
|
9
|
+
MAX_ERRORS = 10
|
|
10
|
+
|
|
11
|
+
def initialize(out)
|
|
12
|
+
@out = out
|
|
13
|
+
end
|
|
14
|
+
|
|
15
|
+
def call(report)
|
|
16
|
+
summary(report)
|
|
17
|
+
warnings(report)
|
|
18
|
+
errors(report)
|
|
19
|
+
survivors(report)
|
|
20
|
+
end
|
|
21
|
+
|
|
22
|
+
private
|
|
23
|
+
|
|
24
|
+
def summary(report)
|
|
25
|
+
seed = report.seed && ", seed #{report.seed} (repeat with --seed #{report.seed})"
|
|
26
|
+
@out.puts "saboteur #{VERSION}: #{report.subjects.size} subjects, #{report.results.size} mutants, " \
|
|
27
|
+
"wall #{format('%.2f', report.wall_time)} s#{seed}"
|
|
28
|
+
@out.puts status_line(report)
|
|
29
|
+
details(report)
|
|
30
|
+
end
|
|
31
|
+
|
|
32
|
+
def status_line(report)
|
|
33
|
+
c = report.counts
|
|
34
|
+
survived = "survived #{c[:survived]}"
|
|
35
|
+
ignored = report.survivors.size - report.unignored_survivors.size
|
|
36
|
+
survived += " (#{ignored} ignored)" if ignored.positive?
|
|
37
|
+
"killed #{c[:killed]} #{survived} timeout #{c[:timeout]} resource limit #{c[:resource_limit]} " \
|
|
38
|
+
"error #{c[:error]} skipped #{c[:skipped]}"
|
|
39
|
+
end
|
|
40
|
+
|
|
41
|
+
def details(report)
|
|
42
|
+
score = report.score
|
|
43
|
+
@out.puts "score #{score ? format('%.1f %%', score * 100) : 'n/a'} " \
|
|
44
|
+
"(killed+timeout)/(killed+survived+timeout); errors, resource limits and skips excluded"
|
|
45
|
+
mean = report.mean_runtime
|
|
46
|
+
@out.puts "mean time per executed mutant #{format('%.3f', mean)} s" if mean
|
|
47
|
+
@out.puts "invalid mutations dropped (did not parse): #{report.invalid_mutations}" if
|
|
48
|
+
report.invalid_mutations.positive?
|
|
49
|
+
report.skip_reasons.each { |reason, count| @out.puts "skipped: #{reason}: #{count}" }
|
|
50
|
+
reused = report.results.count(&:reused?)
|
|
51
|
+
@out.puts "reused #{reused} of #{report.results.size} verdicts from earlier runs (--incremental)" if
|
|
52
|
+
reused.positive?
|
|
53
|
+
kill_causes(report)
|
|
54
|
+
end
|
|
55
|
+
|
|
56
|
+
# How many kills came from a failed check and how many from a crash, and
|
|
57
|
+
# which operators mostly kill by crashing (their score says less).
|
|
58
|
+
def kill_causes(report)
|
|
59
|
+
killed = report.results.select { |result| result.status == :killed && result.kill_cause }
|
|
60
|
+
return if killed.empty?
|
|
61
|
+
|
|
62
|
+
crashes = killed.select(&:crash_kill?)
|
|
63
|
+
@out.puts "kills: #{killed.size - crashes.size} by a failed assertion, #{crashes.size} by a crash" \
|
|
64
|
+
"#{crash_classes(crashes)}"
|
|
65
|
+
@out.puts "crash kills by operator: #{crashes_by_operator(killed)}" if crashes.any?
|
|
66
|
+
end
|
|
67
|
+
|
|
68
|
+
def crash_classes(crashes)
|
|
69
|
+
return "" if crashes.empty?
|
|
70
|
+
|
|
71
|
+
counts = crashes.map(&:kill_cause).tally.sort_by { |_, count| -count }
|
|
72
|
+
" (#{counts.map { |name, count| "#{name} #{count}" }.join(', ')})"
|
|
73
|
+
end
|
|
74
|
+
|
|
75
|
+
def crashes_by_operator(killed)
|
|
76
|
+
parts = killed.group_by { |result| result.mutation.operator }.filter_map do |operator, results|
|
|
77
|
+
count = results.count(&:crash_kill?)
|
|
78
|
+
"#{operator} #{count} of #{results.size}" if count.positive?
|
|
79
|
+
end
|
|
80
|
+
parts.join(", ")
|
|
81
|
+
end
|
|
82
|
+
|
|
83
|
+
def warnings(report)
|
|
84
|
+
report.warnings.each { |warning| @out.puts "warning: #{warning}" }
|
|
85
|
+
end
|
|
86
|
+
|
|
87
|
+
def errors(report)
|
|
88
|
+
errored = report.results.select { |result| result.status == :error }
|
|
89
|
+
return if errored.empty?
|
|
90
|
+
|
|
91
|
+
@out.puts "\nErrors (not counted as killed):"
|
|
92
|
+
errored.first(MAX_ERRORS).each do |result|
|
|
93
|
+
@out.puts " #{result.subject.id} #{result.mutation.operator} #{result.digest}: " \
|
|
94
|
+
"#{result.reason.to_s.lines.first&.strip}"
|
|
95
|
+
end
|
|
96
|
+
@out.puts " ... #{errored.size - MAX_ERRORS} more" if errored.size > MAX_ERRORS
|
|
97
|
+
end
|
|
98
|
+
|
|
99
|
+
def survivors(report)
|
|
100
|
+
return if report.survivors.empty?
|
|
101
|
+
|
|
102
|
+
@out.puts "\nSurvivors:"
|
|
103
|
+
shown(report).each do |result, hidden|
|
|
104
|
+
location = "#{relative(result.subject.file, report.root)}:#{result.subject.start_line}"
|
|
105
|
+
@out.puts "\n#{result.subject.id} #{result.mutation.operator} #{result.digest} #{location}"
|
|
106
|
+
@out.puts "ignored: #{result.ignored}" if result.ignored
|
|
107
|
+
@out.puts result.diff(root: report.root)
|
|
108
|
+
@out.puts "#{hidden} more survivor#{'s' if hidden > 1} on this line in the JSON report" if hidden.positive?
|
|
109
|
+
end
|
|
110
|
+
end
|
|
111
|
+
|
|
112
|
+
# [[result, how many more on the same line]]. In review mode survivors
|
|
113
|
+
# on one line usually share a fate, so one is shown: live before
|
|
114
|
+
# ignored, then coarse before fine, then the earliest edit.
|
|
115
|
+
def shown(report)
|
|
116
|
+
return report.survivors.map { |result| [result, 0] } unless report.review?
|
|
117
|
+
|
|
118
|
+
lines = report.survivors.group_by { |result| [result.subject.file, edit_line(result)] }
|
|
119
|
+
lines.values.map do |same_line|
|
|
120
|
+
best = same_line.min_by do |result|
|
|
121
|
+
[result.ignored ? 1 : 0, result.mutation.coarse ? 0 : 1, result.mutation.edits.first.start_byte]
|
|
122
|
+
end
|
|
123
|
+
[best, same_line.size - 1]
|
|
124
|
+
end
|
|
125
|
+
end
|
|
126
|
+
|
|
127
|
+
def edit_line(result)
|
|
128
|
+
result.subject.line_of(result.mutation.edits.first.start_byte)
|
|
129
|
+
end
|
|
130
|
+
|
|
131
|
+
def relative(file, root)
|
|
132
|
+
file.delete_prefix("#{root}/")
|
|
133
|
+
end
|
|
134
|
+
end
|
|
135
|
+
end
|
|
136
|
+
end
|
|
@@ -0,0 +1,134 @@
|
|
|
1
|
+
# SPDX-License-Identifier: MIT
|
|
2
|
+
# frozen_string_literal: true
|
|
3
|
+
|
|
4
|
+
require_relative "reporter/diff"
|
|
5
|
+
|
|
6
|
+
module Saboteur
|
|
7
|
+
# resource_limit: the mutant's process tree hit the memory or process cap.
|
|
8
|
+
# Like error it is outside the score: a mutant that eats all memory was
|
|
9
|
+
# stopped by saboteur, not detected by a test.
|
|
10
|
+
STATUSES = %i[killed survived timeout resource_limit error skipped].freeze
|
|
11
|
+
|
|
12
|
+
# Outcome of one mutation. reason carries the skip reason, the error
|
|
13
|
+
# message or the timeout limit. ignored is the ignore-list reason, if any.
|
|
14
|
+
class MutantResult
|
|
15
|
+
attr_reader :subject, :mutation, :status, :reason, :runtime, :killed_by, :covered_by, :mutated_source,
|
|
16
|
+
:kill_cause
|
|
17
|
+
attr_accessor :ignored
|
|
18
|
+
|
|
19
|
+
def initialize(subject:, mutation:, status:, mutated_source:, reason: nil, runtime: nil, killed_by: [],
|
|
20
|
+
covered_by: [], kill_cause: nil, reused: false)
|
|
21
|
+
raise ArgumentError, "unknown status #{status.inspect}" unless STATUSES.include?(status)
|
|
22
|
+
|
|
23
|
+
@subject = subject
|
|
24
|
+
@mutation = mutation
|
|
25
|
+
@status = status
|
|
26
|
+
@reason = reason
|
|
27
|
+
@runtime = runtime
|
|
28
|
+
@killed_by = killed_by
|
|
29
|
+
@covered_by = covered_by
|
|
30
|
+
@mutated_source = mutated_source
|
|
31
|
+
@kill_cause = kill_cause
|
|
32
|
+
@reused = reused
|
|
33
|
+
@ignored = nil
|
|
34
|
+
end
|
|
35
|
+
|
|
36
|
+
# The same mutant, marked as an error: its verdict did not hold up.
|
|
37
|
+
def as_error(reason)
|
|
38
|
+
MutantResult.new(subject: subject, mutation: mutation, status: :error, mutated_source: mutated_source,
|
|
39
|
+
reason: reason, runtime: runtime, killed_by: killed_by, covered_by: covered_by)
|
|
40
|
+
end
|
|
41
|
+
|
|
42
|
+
def digest
|
|
43
|
+
@digest ||= mutation.digest(subject_start: subject.start_byte, original_source: subject.file_source)
|
|
44
|
+
end
|
|
45
|
+
|
|
46
|
+
def survived?
|
|
47
|
+
status == :survived
|
|
48
|
+
end
|
|
49
|
+
|
|
50
|
+
# Killed because a test's error, not its check, fired: NoMethodError and
|
|
51
|
+
# the like. Such kills say the code broke, not that a test verified it.
|
|
52
|
+
def crash_kill?
|
|
53
|
+
status == :killed && !kill_cause.nil? && kill_cause != "assertion"
|
|
54
|
+
end
|
|
55
|
+
|
|
56
|
+
# Taken from an earlier run (--incremental) instead of run again.
|
|
57
|
+
def reused?
|
|
58
|
+
@reused
|
|
59
|
+
end
|
|
60
|
+
|
|
61
|
+
def unignored_survivor?
|
|
62
|
+
survived? && ignored.nil?
|
|
63
|
+
end
|
|
64
|
+
|
|
65
|
+
def diff(root: nil)
|
|
66
|
+
path = root ? relative(subject.file, root) : subject.file
|
|
67
|
+
before, after = subject.surrounding_text
|
|
68
|
+
Reporter::Diff.unified(before + subject.source + after, before + mutated_source + after,
|
|
69
|
+
path: path, first_line: subject.start_line)
|
|
70
|
+
end
|
|
71
|
+
|
|
72
|
+
private
|
|
73
|
+
|
|
74
|
+
def relative(file, root)
|
|
75
|
+
file.start_with?("#{root}/") ? file.delete_prefix("#{root}/") : file
|
|
76
|
+
end
|
|
77
|
+
end
|
|
78
|
+
|
|
79
|
+
# Everything one run produced.
|
|
80
|
+
class Report
|
|
81
|
+
attr_reader :results, :subjects, :wall_time, :invalid_mutations, :warnings, :root, :seed
|
|
82
|
+
|
|
83
|
+
# REVIEW: diff mode; the terminal then shows one survivor per line.
|
|
84
|
+
# seed: the random seed every test run started from (nil when unknown).
|
|
85
|
+
def initialize(results:, subjects:, wall_time:, root:, invalid_mutations: 0, warnings: [], review: false,
|
|
86
|
+
seed: nil)
|
|
87
|
+
@results = results
|
|
88
|
+
@subjects = subjects
|
|
89
|
+
@wall_time = wall_time
|
|
90
|
+
@root = root
|
|
91
|
+
@invalid_mutations = invalid_mutations
|
|
92
|
+
@warnings = warnings
|
|
93
|
+
@review = review
|
|
94
|
+
@seed = seed
|
|
95
|
+
end
|
|
96
|
+
|
|
97
|
+
def review?
|
|
98
|
+
@review
|
|
99
|
+
end
|
|
100
|
+
|
|
101
|
+
def counts
|
|
102
|
+
STATUSES.to_h { |status| [status, results.count { |result| result.status == status }] }
|
|
103
|
+
end
|
|
104
|
+
|
|
105
|
+
# (killed + timeout) / (killed + survived + timeout); nil with nothing tested.
|
|
106
|
+
def score
|
|
107
|
+
c = counts
|
|
108
|
+
detected = c[:killed] + c[:timeout]
|
|
109
|
+
tested = detected + c[:survived]
|
|
110
|
+
tested.zero? ? nil : detected.fdiv(tested)
|
|
111
|
+
end
|
|
112
|
+
|
|
113
|
+
def survivors
|
|
114
|
+
results.select(&:survived?)
|
|
115
|
+
end
|
|
116
|
+
|
|
117
|
+
def unignored_survivors
|
|
118
|
+
results.select(&:unignored_survivor?)
|
|
119
|
+
end
|
|
120
|
+
|
|
121
|
+
def skip_reasons
|
|
122
|
+
results.select { |result| result.status == :skipped }.group_by(&:reason).transform_values(&:size)
|
|
123
|
+
end
|
|
124
|
+
|
|
125
|
+
def executed
|
|
126
|
+
results.reject { |result| result.status == :skipped }
|
|
127
|
+
end
|
|
128
|
+
|
|
129
|
+
def mean_runtime
|
|
130
|
+
timed = executed.reject(&:reused?).filter_map(&:runtime)
|
|
131
|
+
timed.empty? ? nil : timed.sum / timed.size
|
|
132
|
+
end
|
|
133
|
+
end
|
|
134
|
+
end
|
|
@@ -0,0 +1,113 @@
|
|
|
1
|
+
# SPDX-License-Identifier: MIT
|
|
2
|
+
# frozen_string_literal: true
|
|
3
|
+
|
|
4
|
+
require "coverage"
|
|
5
|
+
require "digest"
|
|
6
|
+
require "fileutils"
|
|
7
|
+
require "json"
|
|
8
|
+
|
|
9
|
+
module Saboteur
|
|
10
|
+
module Selection
|
|
11
|
+
# file:line -> ids of the tests that executed that line.
|
|
12
|
+
#
|
|
13
|
+
# Built once per cache key in a forked child that runs the whole suite and
|
|
14
|
+
# takes a per-test delta with Coverage.result(stop: false, clear: true).
|
|
15
|
+
# Lines executed outside any test (before(:all), before(:suite)) are mapped
|
|
16
|
+
# to every test in the file of the next test that starts. Each test's
|
|
17
|
+
# duration is recorded so selected tests can run fastest first.
|
|
18
|
+
class CoverageMap
|
|
19
|
+
FORMAT = 3
|
|
20
|
+
|
|
21
|
+
attr_reader :lines, :tests, :build_seconds, :durations
|
|
22
|
+
|
|
23
|
+
# lines: { "/abs/file.rb" => { 12 => ["id1", "id2"] } }
|
|
24
|
+
# durations: { "id1" => seconds }
|
|
25
|
+
def initialize(lines:, tests:, build_seconds: nil, durations: {})
|
|
26
|
+
@lines = lines
|
|
27
|
+
@tests = tests
|
|
28
|
+
@build_seconds = build_seconds
|
|
29
|
+
@durations = durations
|
|
30
|
+
end
|
|
31
|
+
|
|
32
|
+
def tests_for(subject)
|
|
33
|
+
tests_for_lines(subject.file, subject.coverage_lines)
|
|
34
|
+
end
|
|
35
|
+
|
|
36
|
+
# Tests that executed any of the lines, fastest first (suite order breaks
|
|
37
|
+
# ties), no duplicates. With fail-fast, a mutant is killed by its
|
|
38
|
+
# cheapest killing test.
|
|
39
|
+
# The time the tests took while the map was built, or nil when one of
|
|
40
|
+
# them has no recorded duration.
|
|
41
|
+
def recorded_runtime(test_ids)
|
|
42
|
+
test_ids.sum { |id| durations.fetch(id) { return nil } }
|
|
43
|
+
end
|
|
44
|
+
|
|
45
|
+
def tests_for_lines(file, line_numbers)
|
|
46
|
+
by_line = lines[CoverageMap.canonical(file)]
|
|
47
|
+
return [] unless by_line
|
|
48
|
+
|
|
49
|
+
ids = line_numbers.flat_map { |line| by_line.fetch(line, []) }.uniq
|
|
50
|
+
order = test_order
|
|
51
|
+
ids.sort_by { |id| [durations.fetch(id, Float::INFINITY), order.fetch(id, Float::INFINITY)] }
|
|
52
|
+
end
|
|
53
|
+
|
|
54
|
+
# Lines refer to tests by their index in tests, which keeps the cache
|
|
55
|
+
# small for suites with long test ids.
|
|
56
|
+
def to_json(*)
|
|
57
|
+
index = test_order
|
|
58
|
+
encoded = lines.transform_values do |by_line|
|
|
59
|
+
by_line.to_h { |line, ids| [line.to_s, ids.map { |id| index.fetch(id) }] }
|
|
60
|
+
end
|
|
61
|
+
JSON.generate(format: FORMAT, tests: tests, build_seconds: build_seconds, lines: encoded,
|
|
62
|
+
durations: tests.map { |id| durations[id]&.round(6) })
|
|
63
|
+
end
|
|
64
|
+
|
|
65
|
+
def self.from_json(json)
|
|
66
|
+
data = JSON.parse(json)
|
|
67
|
+
raise Error, "coverage map format #{data['format']} is not #{FORMAT}" unless data["format"] == FORMAT
|
|
68
|
+
|
|
69
|
+
tests = data["tests"]
|
|
70
|
+
lines = data["lines"].transform_values do |by_line|
|
|
71
|
+
by_line.to_h { |line, indexes| [line.to_i, indexes.map { |i| tests.fetch(i) }] }
|
|
72
|
+
end
|
|
73
|
+
durations = tests.zip(data["durations"]).to_h.compact
|
|
74
|
+
new(lines: lines, tests: tests, build_seconds: data["build_seconds"], durations: durations)
|
|
75
|
+
end
|
|
76
|
+
|
|
77
|
+
# Cache key over every input that can change which test runs which line.
|
|
78
|
+
def self.cache_key(files:, extra:)
|
|
79
|
+
digest = Digest::SHA256.new
|
|
80
|
+
digest << VERSION << "\0" << extra.to_s << "\0"
|
|
81
|
+
files.sort.each do |file|
|
|
82
|
+
digest << file << "\0" << (File.file?(file) ? Digest::SHA256.file(file).hexdigest : "missing") << "\0"
|
|
83
|
+
end
|
|
84
|
+
digest.hexdigest
|
|
85
|
+
end
|
|
86
|
+
|
|
87
|
+
def self.canonical(path)
|
|
88
|
+
File.realpath(path)
|
|
89
|
+
rescue SystemCallError
|
|
90
|
+
File.expand_path(path)
|
|
91
|
+
end
|
|
92
|
+
|
|
93
|
+
def self.load(path)
|
|
94
|
+
from_json(File.read(path)) if File.file?(path)
|
|
95
|
+
rescue JSON::ParserError, Error
|
|
96
|
+
nil
|
|
97
|
+
end
|
|
98
|
+
|
|
99
|
+
def save(path)
|
|
100
|
+
FileUtils.mkdir_p(File.dirname(path))
|
|
101
|
+
tmp = "#{path}.#{Process.pid}.tmp"
|
|
102
|
+
File.write(tmp, to_json)
|
|
103
|
+
File.rename(tmp, path)
|
|
104
|
+
end
|
|
105
|
+
|
|
106
|
+
private
|
|
107
|
+
|
|
108
|
+
def test_order
|
|
109
|
+
@test_order ||= tests.each_with_index.to_h
|
|
110
|
+
end
|
|
111
|
+
end
|
|
112
|
+
end
|
|
113
|
+
end
|
|
@@ -0,0 +1,64 @@
|
|
|
1
|
+
# SPDX-License-Identifier: MIT
|
|
2
|
+
# frozen_string_literal: true
|
|
3
|
+
|
|
4
|
+
require "open3"
|
|
5
|
+
|
|
6
|
+
module Saboteur
|
|
7
|
+
module Selection
|
|
8
|
+
# Lines changed between the merge base of REF and HEAD, plus uncommitted
|
|
9
|
+
# changes, as { "/abs/path.rb" => Set[line numbers in the current file] }.
|
|
10
|
+
module GitDiff
|
|
11
|
+
NO_FILE = "/dev/null" # rubocop:disable Style/FileNull -- literal in git diff output
|
|
12
|
+
HUNK = /\A@@ -\d+(?:,\d+)? \+(\d+)(?:,(\d+))? @@/
|
|
13
|
+
|
|
14
|
+
module_function
|
|
15
|
+
|
|
16
|
+
def changed_lines(ref, root:)
|
|
17
|
+
base = git(root, "merge-base", ref, "HEAD").strip
|
|
18
|
+
top = git(root, "rev-parse", "--show-toplevel").strip
|
|
19
|
+
parse(git(root, "-c", "core.quotePath=false", "diff", "--unified=0", "--no-color", "--no-ext-diff", base,
|
|
20
|
+
"--"), top)
|
|
21
|
+
end
|
|
22
|
+
|
|
23
|
+
def parse(diff, top)
|
|
24
|
+
changes = Hash.new { |hash, file| hash[file] = Set.new }
|
|
25
|
+
file = nil
|
|
26
|
+
diff.each_line do |line|
|
|
27
|
+
if line.start_with?("+++ ")
|
|
28
|
+
path = unquote(line.chomp.delete_prefix("+++ ").delete_suffix("\t"))
|
|
29
|
+
file = path == NO_FILE ? nil : File.join(top, path.delete_prefix("b/"))
|
|
30
|
+
elsif file && (match = HUNK.match(line))
|
|
31
|
+
start = match[1].to_i
|
|
32
|
+
count = (match[2] || "1").to_i
|
|
33
|
+
# A pure deletion touches the lines on either side of it.
|
|
34
|
+
lines = count.zero? ? [start, start + 1] : (start...(start + count)).to_a
|
|
35
|
+
changes[file].merge(lines)
|
|
36
|
+
end
|
|
37
|
+
end
|
|
38
|
+
changes.default_proc = nil
|
|
39
|
+
changes
|
|
40
|
+
end
|
|
41
|
+
|
|
42
|
+
ESCAPES = { "n" => "\n", "t" => "\t", '"' => '"', "\\" => "\\", "a" => "\a", "b" => "\b",
|
|
43
|
+
"f" => "\f", "r" => "\r", "v" => "\v" }.freeze
|
|
44
|
+
|
|
45
|
+
# git C-quotes a path with special bytes: "b/h\303\251llo.rb". It also
|
|
46
|
+
# ends a path that holds a space with a tab, which the caller strips.
|
|
47
|
+
def unquote(path)
|
|
48
|
+
return path unless path.start_with?('"') && path.end_with?('"')
|
|
49
|
+
|
|
50
|
+
path[1...-1].gsub(/\\([0-7]{3}|.)/) do
|
|
51
|
+
code = Regexp.last_match(1)
|
|
52
|
+
code.length == 3 ? code.to_i(8).chr : ESCAPES.fetch(code, code)
|
|
53
|
+
end.b.force_encoding(Encoding::UTF_8)
|
|
54
|
+
end
|
|
55
|
+
|
|
56
|
+
def git(root, *args)
|
|
57
|
+
out, err, status = Open3.capture3("git", *args, chdir: root)
|
|
58
|
+
raise Error, "git #{args.first} failed: #{err.strip}" unless status.success?
|
|
59
|
+
|
|
60
|
+
out
|
|
61
|
+
end
|
|
62
|
+
end
|
|
63
|
+
end
|
|
64
|
+
end
|
|
@@ -0,0 +1,72 @@
|
|
|
1
|
+
# SPDX-License-Identifier: MIT
|
|
2
|
+
# frozen_string_literal: true
|
|
3
|
+
|
|
4
|
+
require "coverage"
|
|
5
|
+
require_relative "coverage_map"
|
|
6
|
+
|
|
7
|
+
module Saboteur
|
|
8
|
+
module Selection
|
|
9
|
+
# Collects per-test line deltas while the suite runs. The integration
|
|
10
|
+
# calls test_started / test_finished around every test.
|
|
11
|
+
class Recorder
|
|
12
|
+
def initialize(files)
|
|
13
|
+
@files = Set.new(files.map { |file| CoverageMap.canonical(file) })
|
|
14
|
+
@canonical = Hash.new { |hash, path| hash[path] = CoverageMap.canonical(path) }
|
|
15
|
+
@lines = Hash.new { |hash, file| hash[file] = Hash.new { |by_line, line| by_line[line] = [] } }
|
|
16
|
+
@tests = []
|
|
17
|
+
@tests_by_file = Hash.new { |hash, file| hash[file] = [] }
|
|
18
|
+
@outside = Hash.new { |hash, file| hash[file] = Set.new }
|
|
19
|
+
@durations = {}
|
|
20
|
+
end
|
|
21
|
+
|
|
22
|
+
def start
|
|
23
|
+
Coverage.resume if Coverage.state == :suspended
|
|
24
|
+
Coverage.result(stop: false, clear: true)
|
|
25
|
+
end
|
|
26
|
+
|
|
27
|
+
def test_started(id, test_file)
|
|
28
|
+
@tests << id
|
|
29
|
+
@tests_by_file[test_file] << id
|
|
30
|
+
each_hit_line { |file, line| @outside[test_file] << [file, line] }
|
|
31
|
+
@started_at = monotonic
|
|
32
|
+
end
|
|
33
|
+
|
|
34
|
+
def test_finished(id)
|
|
35
|
+
@durations[id] = monotonic - @started_at if @started_at
|
|
36
|
+
each_hit_line { |file, line| @lines[file][line] << id }
|
|
37
|
+
end
|
|
38
|
+
|
|
39
|
+
def result(build_seconds: nil)
|
|
40
|
+
@outside.each do |test_file, hits|
|
|
41
|
+
ids = @tests_by_file[test_file]
|
|
42
|
+
hits.each { |file, line| @lines[file][line] |= ids }
|
|
43
|
+
end
|
|
44
|
+
lines = @lines.transform_values { |by_line| by_line.transform_values(&:uniq) }
|
|
45
|
+
CoverageMap.new(lines: lines, tests: @tests, build_seconds: build_seconds, durations: @durations)
|
|
46
|
+
end
|
|
47
|
+
|
|
48
|
+
private
|
|
49
|
+
|
|
50
|
+
def monotonic
|
|
51
|
+
Process.clock_gettime(Process::CLOCK_MONOTONIC)
|
|
52
|
+
end
|
|
53
|
+
|
|
54
|
+
# Coverage is suspended while the delta is processed, so the recorder's
|
|
55
|
+
# own work (and anything it calls) is never attributed to a test.
|
|
56
|
+
def each_hit_line
|
|
57
|
+
snapshot = Coverage.result(stop: false, clear: true)
|
|
58
|
+
suspended = Coverage.running?
|
|
59
|
+
Coverage.suspend if suspended
|
|
60
|
+
snapshot.each do |path, data|
|
|
61
|
+
file = @canonical[path]
|
|
62
|
+
next unless @files.include?(file)
|
|
63
|
+
|
|
64
|
+
# Lines mode (set up by the executable); nil marks a line with no code.
|
|
65
|
+
data[:lines].each_with_index { |count, index| yield file, index + 1 if count&.positive? }
|
|
66
|
+
end
|
|
67
|
+
ensure
|
|
68
|
+
Coverage.resume if suspended
|
|
69
|
+
end
|
|
70
|
+
end
|
|
71
|
+
end
|
|
72
|
+
end
|
|
@@ -0,0 +1,166 @@
|
|
|
1
|
+
# SPDX-License-Identifier: MIT
|
|
2
|
+
# frozen_string_literal: true
|
|
3
|
+
|
|
4
|
+
module Saboteur
|
|
5
|
+
# One lexical scope enclosing a def: `module X`, `class X` or `class << self`.
|
|
6
|
+
Scope = Data.define(:kind, :path) do
|
|
7
|
+
def header
|
|
8
|
+
case kind
|
|
9
|
+
when :module then "module #{path}"
|
|
10
|
+
when :class then "class #{path}"
|
|
11
|
+
when :sclass then "class << self"
|
|
12
|
+
end
|
|
13
|
+
end
|
|
14
|
+
|
|
15
|
+
def sclass?
|
|
16
|
+
kind == :sclass
|
|
17
|
+
end
|
|
18
|
+
end
|
|
19
|
+
|
|
20
|
+
# A method defined with `def`: the unit saboteur mutates.
|
|
21
|
+
#
|
|
22
|
+
# Byte offsets index the file as read from disk. singleton is true for
|
|
23
|
+
# `def self.x` and for defs inside `class << self`; sclass is true only for
|
|
24
|
+
# the latter, because the def text must then be evaluated on the singleton.
|
|
25
|
+
class Subject
|
|
26
|
+
attr_reader :name, :owner, :singleton, :sclass, :file, :start_byte, :end_byte,
|
|
27
|
+
:start_line, :end_line, :nesting, :node, :file_source, :frozen_literals
|
|
28
|
+
attr_accessor :skip_reason
|
|
29
|
+
|
|
30
|
+
def initialize(name:, owner:, singleton:, sclass:, file:, node:, file_source:, nesting:, frozen_literals:)
|
|
31
|
+
@name = name
|
|
32
|
+
@owner = owner
|
|
33
|
+
@singleton = singleton
|
|
34
|
+
@sclass = sclass
|
|
35
|
+
@file = file
|
|
36
|
+
@node = node
|
|
37
|
+
@file_source = file_source
|
|
38
|
+
@nesting = nesting
|
|
39
|
+
@frozen_literals = frozen_literals
|
|
40
|
+
@start_byte = node.location.start_offset
|
|
41
|
+
@end_byte = node.location.end_offset
|
|
42
|
+
@start_line = node.location.start_line
|
|
43
|
+
@end_line = node.location.end_line
|
|
44
|
+
@skip_reason = nil
|
|
45
|
+
end
|
|
46
|
+
|
|
47
|
+
def id
|
|
48
|
+
"#{owner}#{singleton ? '.' : '#'}#{name}"
|
|
49
|
+
end
|
|
50
|
+
|
|
51
|
+
def source
|
|
52
|
+
@source ||= file_source.byteslice(start_byte, end_byte - start_byte).force_encoding(Encoding::UTF_8)
|
|
53
|
+
end
|
|
54
|
+
|
|
55
|
+
# Text on the def's first line before it, and on its last line after it,
|
|
56
|
+
# so diffs show whole lines with their indentation.
|
|
57
|
+
def surrounding_text
|
|
58
|
+
line_start = (file_source.byteslice(0, start_byte).rindex("\n") || -1) + 1
|
|
59
|
+
line_end = file_source.index("\n", end_byte) || file_source.bytesize
|
|
60
|
+
[file_source.byteslice(line_start, start_byte - line_start),
|
|
61
|
+
file_source.byteslice(end_byte, line_end - end_byte)].map { |text| text.force_encoding(Encoding::UTF_8) }
|
|
62
|
+
end
|
|
63
|
+
|
|
64
|
+
# Lines whose execution means the method body ran. For a multi-line def
|
|
65
|
+
# the def line itself runs at load time, so it is left out.
|
|
66
|
+
def coverage_lines
|
|
67
|
+
return [start_line] if start_line == end_line
|
|
68
|
+
|
|
69
|
+
((start_line + 1)..end_line).to_a
|
|
70
|
+
end
|
|
71
|
+
|
|
72
|
+
# Lines whose execution means the edited code ran: from the first line of
|
|
73
|
+
# the innermost statement containing the edit to the edit's last line.
|
|
74
|
+
# A statement's first line always records a coverage event when it runs,
|
|
75
|
+
# so a test that never ran the edited code cannot have hit these lines.
|
|
76
|
+
# An edit outside every statement (signature, around the def, whole
|
|
77
|
+
# body) maps to the whole method.
|
|
78
|
+
def coverage_lines_for(edit)
|
|
79
|
+
statement = innermost_statement(node.body, edit.start_byte, edit.end_byte)
|
|
80
|
+
return coverage_lines unless statement
|
|
81
|
+
|
|
82
|
+
first = statement.location.start_line
|
|
83
|
+
last = line_of([edit.end_byte - 1, edit.start_byte].max)
|
|
84
|
+
(first..[last, first].max).to_a
|
|
85
|
+
end
|
|
86
|
+
|
|
87
|
+
# True when anything in the body can turn an error into a value: a
|
|
88
|
+
# rescue or else clause, a modifier rescue, a nested begin/rescue.
|
|
89
|
+
def recovers?
|
|
90
|
+
rescuing?(node.body)
|
|
91
|
+
end
|
|
92
|
+
|
|
93
|
+
# The 1-based line of the file that holds this byte offset.
|
|
94
|
+
def line_of(byte_offset)
|
|
95
|
+
file_source.byteslice(0, byte_offset).count("\n") + 1
|
|
96
|
+
end
|
|
97
|
+
|
|
98
|
+
def contains_line?(line)
|
|
99
|
+
line.between?(start_line, end_line)
|
|
100
|
+
end
|
|
101
|
+
|
|
102
|
+
# Applies the mutation's edits to this method's text and returns the
|
|
103
|
+
# mutated method text. Raises ArgumentError for edits outside the method
|
|
104
|
+
# or overlapping each other.
|
|
105
|
+
def apply(edits)
|
|
106
|
+
text = file_source.byteslice(start_byte, end_byte - start_byte).b
|
|
107
|
+
previous_start = nil
|
|
108
|
+
edits.sort_by(&:start_byte).reverse_each do |edit|
|
|
109
|
+
validate_edit!(edit, previous_start)
|
|
110
|
+
previous_start = edit.start_byte
|
|
111
|
+
relative = edit.start_byte - start_byte
|
|
112
|
+
text = text.byteslice(0, relative) + edit.replacement.b + text.byteslice((edit.end_byte - start_byte)..)
|
|
113
|
+
end
|
|
114
|
+
text.force_encoding(Encoding::UTF_8)
|
|
115
|
+
end
|
|
116
|
+
|
|
117
|
+
def to_s
|
|
118
|
+
id
|
|
119
|
+
end
|
|
120
|
+
|
|
121
|
+
private
|
|
122
|
+
|
|
123
|
+
def rescuing?(child)
|
|
124
|
+
case child
|
|
125
|
+
when nil, Prism::DefNode, Prism::ClassNode, Prism::ModuleNode then false
|
|
126
|
+
when Prism::RescueModifierNode then true
|
|
127
|
+
when Prism::BeginNode then !(child.rescue_clause.nil? && child.else_clause.nil?) || descend?(child)
|
|
128
|
+
else descend?(child)
|
|
129
|
+
end
|
|
130
|
+
end
|
|
131
|
+
|
|
132
|
+
def descend?(child)
|
|
133
|
+
child.compact_child_nodes.any? { |grandchild| rescuing?(grandchild) }
|
|
134
|
+
end
|
|
135
|
+
|
|
136
|
+
def innermost_statement(node, start_byte, end_byte, found = nil)
|
|
137
|
+
return found if node.nil?
|
|
138
|
+
|
|
139
|
+
if node.is_a?(Prism::StatementsNode)
|
|
140
|
+
statement = node.body.find do |child|
|
|
141
|
+
child.location.start_offset <= start_byte && end_byte <= child.location.end_offset
|
|
142
|
+
end
|
|
143
|
+
return found unless statement
|
|
144
|
+
|
|
145
|
+
found = statement
|
|
146
|
+
node = statement
|
|
147
|
+
end
|
|
148
|
+
node.compact_child_nodes.each do |child|
|
|
149
|
+
next unless child.location.start_offset <= start_byte && end_byte <= child.location.end_offset
|
|
150
|
+
|
|
151
|
+
return innermost_statement(child, start_byte, end_byte, found)
|
|
152
|
+
end
|
|
153
|
+
found
|
|
154
|
+
end
|
|
155
|
+
|
|
156
|
+
def validate_edit!(edit, previous_start)
|
|
157
|
+
unless File.expand_path(edit.file) == File.expand_path(file)
|
|
158
|
+
raise ArgumentError, "edit for #{edit.file} applied to subject in #{file}"
|
|
159
|
+
end
|
|
160
|
+
unless edit.start_byte >= start_byte && edit.end_byte <= end_byte
|
|
161
|
+
raise ArgumentError, "edit #{edit.start_byte}...#{edit.end_byte} is outside #{id} (#{start_byte}...#{end_byte})"
|
|
162
|
+
end
|
|
163
|
+
raise ArgumentError, "overlapping edits in mutation of #{id}" if previous_start && edit.end_byte > previous_start
|
|
164
|
+
end
|
|
165
|
+
end
|
|
166
|
+
end
|