saboteur 0.6.0 → 0.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +1024 -0
- data/README.md +199 -952
- data/exe/saboteur +34 -7
- data/lib/saboteur/baseline/workflow.rb +2 -1
- data/lib/saboteur/baseline_health.rb +40 -6
- data/lib/saboteur/cli/exit_status.rb +80 -4
- data/lib/saboteur/cli/help_options.rb +17 -0
- data/lib/saboteur/cli/history_command.rb +135 -0
- data/lib/saboteur/cli/presets.rb +40 -0
- data/lib/saboteur/cli/run_budget_options.rb +37 -0
- data/lib/saboteur/cli/run_reporting.rb +74 -5
- data/lib/saboteur/cli/selection_options.rb +31 -0
- data/lib/saboteur/cli/shard_options.rb +17 -0
- data/lib/saboteur/cli/source_set.rb +53 -0
- data/lib/saboteur/cli/survivors_command.rb +157 -0
- data/lib/saboteur/cli.rb +136 -98
- data/lib/saboteur/config.rb +163 -15
- data/lib/saboteur/config_file.rb +8 -0
- data/lib/saboteur/constant_reads.rb +182 -0
- data/lib/saboteur/constant_subject.rb +70 -0
- data/lib/saboteur/constant_subject_finder.rb +281 -0
- data/lib/saboteur/coverage_reads.rb +29 -0
- data/lib/saboteur/crash_report.rb +154 -0
- data/lib/saboteur/custom_map.rb +3 -1
- data/lib/saboteur/dirty_tree_warning.rb +128 -0
- data/lib/saboteur/doctor.rb +49 -1
- data/lib/saboteur/engine/baseline_guard.rb +136 -14
- data/lib/saboteur/engine/boot.rb +16 -1
- data/lib/saboteur/engine/canary.rb +41 -10
- data/lib/saboteur/engine/candidate_selection.rb +28 -1
- data/lib/saboteur/engine/candidates.rb +43 -3
- data/lib/saboteur/engine/coverage_guard.rb +29 -0
- data/lib/saboteur/engine/coverage_store.rb +79 -15
- data/lib/saboteur/engine/defined_order_prefix.rb +43 -0
- data/lib/saboteur/engine/diagnostics.rb +41 -2
- data/lib/saboteur/engine/diff_scope.rb +24 -5
- data/lib/saboteur/engine/distribution.rb +134 -8
- data/lib/saboteur/engine/early_stop_recording.rb +47 -0
- data/lib/saboteur/engine/gated_run.rb +35 -0
- data/lib/saboteur/engine/load_coverage.rb +45 -0
- data/lib/saboteur/engine/out_of_scope_guard.rb +21 -0
- data/lib/saboteur/engine/pipelines.rb +107 -0
- data/lib/saboteur/engine/result_cache.rb +67 -9
- data/lib/saboteur/engine/run_budget.rb +90 -0
- data/lib/saboteur/engine/run_fingerprint.rb +4 -0
- data/lib/saboteur/engine/sandbox_policy.rb +56 -11
- data/lib/saboteur/engine/scheduler.rb +222 -20
- data/lib/saboteur/engine/selection_announcement.rb +54 -0
- data/lib/saboteur/engine/shard.rb +69 -0
- data/lib/saboteur/engine/subject_selection.rb +373 -19
- data/lib/saboteur/engine/suite_files.rb +3 -1
- data/lib/saboteur/engine/test_guess.rb +438 -25
- data/lib/saboteur/engine/thread_warnings.rb +53 -0
- data/lib/saboteur/engine/unchanged_syntax.rb +17 -2
- data/lib/saboteur/engine/unscoped_suite_warning.rb +41 -0
- data/lib/saboteur/engine/verdict_check.rb +54 -18
- data/lib/saboteur/engine.rb +202 -158
- data/lib/saboteur/execution/external_runner.rb +471 -0
- data/lib/saboteur/execution/marker_sweep.rb +79 -0
- data/lib/saboteur/execution/peak_memory.rb +28 -0
- data/lib/saboteur/execution/process_table.rb +12 -0
- data/lib/saboteur/execution/process_tree.rb +65 -5
- data/lib/saboteur/execution/protocol.rb +2 -1
- data/lib/saboteur/execution/reevaluator.rb +68 -12
- data/lib/saboteur/execution/runner.rb +225 -24
- data/lib/saboteur/execution/sandbox.rb +21 -3
- data/lib/saboteur/execution/seatbelt.rb +226 -0
- data/lib/saboteur/execution/supervisor.rb +45 -13
- data/lib/saboteur/execution/worker_pool.rb +110 -8
- data/lib/saboteur/frameworks/sqlite_memory_database.rb +49 -0
- data/lib/saboteur/gates.rb +43 -2
- data/lib/saboteur/help.rb +108 -13
- data/lib/saboteur/ignore_list.rb +40 -7
- data/lib/saboteur/install.rb +4 -3
- data/lib/saboteur/integration/base.rb +7 -2
- data/lib/saboteur/integration/rspec.rb +175 -9
- data/lib/saboteur/mutation_source/argument_removal.rb +1 -1
- data/lib/saboteur/mutation_source/builtin.rb +44 -14
- data/lib/saboteur/mutation_source/chain_removal.rb +1 -1
- data/lib/saboteur/mutation_source/condition_negation.rb +1 -1
- data/lib/saboteur/mutation_source/constant_literals.rb +66 -0
- data/lib/saboteur/mutation_source/element_removal.rb +73 -0
- data/lib/saboteur/mutation_source/endpoint_replacement.rb +1 -1
- data/lib/saboteur/mutation_source/entry_edits.rb +44 -0
- data/lib/saboteur/mutation_source/filter_replacement.rb +1 -1
- data/lib/saboteur/mutation_source/hazard_guard.rb +129 -3
- data/lib/saboteur/mutation_source/interpolation_removal.rb +41 -0
- data/lib/saboteur/mutation_source/literal_replacement.rb +1 -1
- data/lib/saboteur/mutation_source/method_swap.rb +193 -0
- data/lib/saboteur/mutation_source/mutation.rb +5 -3
- data/lib/saboteur/mutation_source/operand_replacement.rb +54 -0
- data/lib/saboteur/mutation_source/operator_assignment.rb +1 -1
- data/lib/saboteur/mutation_source/operator_examples.rb +6 -1
- data/lib/saboteur/mutation_source/operator_registry.rb +7 -2
- data/lib/saboteur/mutation_source/operator_visit.rb +39 -0
- data/lib/saboteur/mutation_source/pair_removal.rb +13 -31
- data/lib/saboteur/mutation_source/range_replacement.rb +1 -1
- data/lib/saboteur/mutation_source/regex_replacement.rb +93 -0
- data/lib/saboteur/preview.rb +122 -13
- data/lib/saboteur/refinement_activations.rb +60 -0
- data/lib/saboteur/reporter/agent.rb +9 -1
- data/lib/saboteur/reporter/agent_commands.rb +47 -15
- data/lib/saboteur/reporter/agent_incomplete.rb +38 -2
- data/lib/saboteur/reporter/agent_summary.rb +4 -3
- data/lib/saboteur/reporter/findings.rb +5 -2
- data/lib/saboteur/reporter/history_policy.rb +11 -0
- data/lib/saboteur/reporter/history_reader.rb +47 -0
- data/lib/saboteur/reporter/html.rb +26 -4
- data/lib/saboteur/reporter/mutation_text.rb +6 -0
- data/lib/saboteur/reporter/next_steps.rb +14 -4
- data/lib/saboteur/reporter/score_history.rb +5 -1
- data/lib/saboteur/reporter/stryker.rb +40 -7
- data/lib/saboteur/reporter/survivor_classification.rb +101 -0
- data/lib/saboteur/reporter/survivor_hints.rb +3 -0
- data/lib/saboteur/reporter/survivor_list.rb +15 -9
- data/lib/saboteur/reporter/terminal.rb +74 -17
- data/lib/saboteur/result.rb +38 -7
- data/lib/saboteur/selection/coverage_map.rb +107 -11
- data/lib/saboteur/selection/dependency_shield.rb +80 -0
- data/lib/saboteur/selection/external_test_map.rb +27 -0
- data/lib/saboteur/selection/git_diff.rb +105 -13
- data/lib/saboteur/selection/layout_only_lines.rb +80 -0
- data/lib/saboteur/selection/recorder.rb +22 -4
- data/lib/saboteur/selection/source_scope.rb +22 -0
- data/lib/saboteur/selection/untracked_changes.rb +126 -0
- data/lib/saboteur/subject.rb +53 -7
- data/lib/saboteur/subject_finder.rb +173 -29
- data/lib/saboteur/text.rb +42 -0
- data/lib/saboteur/version.rb +1 -1
- data/lib/saboteur/working_tree.rb +38 -15
- data/lib/saboteur.rb +29 -1
- metadata +50 -2
|
@@ -2,6 +2,7 @@
|
|
|
2
2
|
# frozen_string_literal: true
|
|
3
3
|
|
|
4
4
|
require_relative "process_table"
|
|
5
|
+
require_relative "marker_sweep"
|
|
5
6
|
|
|
6
7
|
module Saboteur
|
|
7
8
|
module Execution
|
|
@@ -12,14 +13,19 @@ module Saboteur
|
|
|
12
13
|
# setsid (a nested saboteur run does exactly that) and leave the group.
|
|
13
14
|
# Parent links survive that, so the tree is rebuilt from the process table
|
|
14
15
|
# on every poll, and every pid ever seen in it is remembered, so a
|
|
15
|
-
# descendant whose parent exits (re-parented to init) is still known
|
|
16
|
+
# descendant whose parent exits (re-parented to init) is still known -
|
|
17
|
+
# but only if some poll SAW the parent link while the intermediate lived.
|
|
18
|
+
# A descendant that leaves the group and loses that parent between two
|
|
19
|
+
# polls is never seen (#365); the marker sweep in kill_all is the net
|
|
20
|
+
# for exactly that case.
|
|
16
21
|
#
|
|
17
22
|
# Identity is pid plus start time, so a recycled pid is never signalled.
|
|
18
23
|
class ProcessTree
|
|
19
24
|
attr_reader :root
|
|
20
25
|
|
|
21
|
-
def initialize(root)
|
|
26
|
+
def initialize(root, marker: nil)
|
|
22
27
|
@root = root
|
|
28
|
+
@marker = marker
|
|
23
29
|
@known = {} # pid => start time
|
|
24
30
|
end
|
|
25
31
|
|
|
@@ -34,6 +40,27 @@ module Saboteur
|
|
|
34
40
|
end
|
|
35
41
|
end
|
|
36
42
|
|
|
43
|
+
# kill_all minus an excluded root, with the pre-kill member
|
|
44
|
+
# descriptions: [killed pids, notes]. The notes are taken first -
|
|
45
|
+
# after the kill the command lines are gone (#486).
|
|
46
|
+
def kill_all_except(root)
|
|
47
|
+
notes = describe_except(root)
|
|
48
|
+
[kill_all - [root], notes]
|
|
49
|
+
end
|
|
50
|
+
|
|
51
|
+
# One "pid (command)" note per live member other than an excluded
|
|
52
|
+
# root, for the leftover-children refusal (#486). Best effort: a
|
|
53
|
+
# process that vanished mid-look lists its pid alone. Call before the
|
|
54
|
+
# kill - after it, the command lines are gone.
|
|
55
|
+
def describe_except(root, table = ProcessTable.snapshot)
|
|
56
|
+
live_members(table).filter_map do |entry|
|
|
57
|
+
next if entry.pid == root
|
|
58
|
+
|
|
59
|
+
command = ProcessTable.command(entry.pid)
|
|
60
|
+
command ? "#{entry.pid} (#{command})" : entry.pid.to_s
|
|
61
|
+
end
|
|
62
|
+
end
|
|
63
|
+
|
|
37
64
|
# Count and resident memory of the live members. rss_baseline_kb is the
|
|
38
65
|
# supervisor's own RSS: forked children share its pages, so only growth
|
|
39
66
|
# beyond it is counted per process.
|
|
@@ -44,15 +71,20 @@ module Saboteur
|
|
|
44
71
|
end
|
|
45
72
|
|
|
46
73
|
# Stops every member (a stopped process cannot fork), rescans until no new
|
|
47
|
-
# member appears, then kills them all.
|
|
74
|
+
# member appears, then kills them all. Each round also sweeps for
|
|
75
|
+
# never-seen escapees carrying the run's marker (#365). Returns the
|
|
76
|
+
# pids it killed.
|
|
48
77
|
def kill_all(rounds: 20)
|
|
49
78
|
killed = []
|
|
50
79
|
rounds.times do
|
|
51
80
|
live = stop_until_stable
|
|
52
|
-
|
|
81
|
+
escaped = marked_escapees
|
|
82
|
+
break if live.empty? && escaped.empty?
|
|
53
83
|
|
|
54
84
|
live.each { |entry| signal("KILL", entry.pid) }
|
|
55
|
-
|
|
85
|
+
signalled = escaped.select { |entry| escapee_alive?(entry) }
|
|
86
|
+
signalled.each { |entry| signal("KILL", entry.pid) }
|
|
87
|
+
killed.concat(live.map(&:pid), signalled.map(&:pid))
|
|
56
88
|
sleep 0.01
|
|
57
89
|
end
|
|
58
90
|
killed.uniq
|
|
@@ -60,6 +92,34 @@ module Saboteur
|
|
|
60
92
|
|
|
61
93
|
private
|
|
62
94
|
|
|
95
|
+
# Live, non-zombie pids carrying the run's marker that the tree never
|
|
96
|
+
# learned: the setsid-and-orphan escapees (#365). Members match the
|
|
97
|
+
# marker too, but the same round kills them anyway.
|
|
98
|
+
def marked_escapees
|
|
99
|
+
return [] unless @marker && MarkerSweep.available?
|
|
100
|
+
|
|
101
|
+
table = ProcessTable.snapshot
|
|
102
|
+
root_started = @known[root]
|
|
103
|
+
MarkerSweep.marked_pids(@marker, Supervisor::TREE_ENV).filter_map do |pid|
|
|
104
|
+
entry = table[pid]
|
|
105
|
+
next unless entry && !entry.zombie && !@known.key?(pid)
|
|
106
|
+
|
|
107
|
+
# A process started before the run's root carries the marker only
|
|
108
|
+
# through an exec chain the run did not spawn: never ours (#389).
|
|
109
|
+
next if root_started && entry.started < root_started
|
|
110
|
+
|
|
111
|
+
entry
|
|
112
|
+
end
|
|
113
|
+
end
|
|
114
|
+
|
|
115
|
+
# A pid is not an identity: the escapee can exit between the sweep's
|
|
116
|
+
# snapshot and the KILL, and the kernel can hand the pid to someone
|
|
117
|
+
# else. Signal only the process the sweep actually saw (#389).
|
|
118
|
+
def escapee_alive?(entry)
|
|
119
|
+
fresh = ProcessTable.snapshot[entry.pid]
|
|
120
|
+
fresh && !fresh.zombie && fresh.started == entry.started
|
|
121
|
+
end
|
|
122
|
+
|
|
63
123
|
def grow(table)
|
|
64
124
|
@known[root] = table[root].started if @known.empty? && table[root]
|
|
65
125
|
loop do
|
|
@@ -2,6 +2,7 @@
|
|
|
2
2
|
# frozen_string_literal: true
|
|
3
3
|
|
|
4
4
|
require "json"
|
|
5
|
+
require_relative "../text"
|
|
5
6
|
|
|
6
7
|
module Saboteur
|
|
7
8
|
module Execution
|
|
@@ -31,7 +32,7 @@ module Saboteur
|
|
|
31
32
|
# Events in order; lines that are not this protocol's are ignored.
|
|
32
33
|
def parse(payload)
|
|
33
34
|
payload.each_line.filter_map do |line|
|
|
34
|
-
message = JSON.parse(line)
|
|
35
|
+
message = JSON.parse(Text.utf8_strict(line))
|
|
35
36
|
message if message.is_a?(Hash) && message["v"] == VERSION
|
|
36
37
|
rescue JSON::ParserError
|
|
37
38
|
nil
|
|
@@ -54,9 +54,54 @@ module Saboteur
|
|
|
54
54
|
nil
|
|
55
55
|
end
|
|
56
56
|
|
|
57
|
+
# The method object the subject's def currently resolves to,
|
|
58
|
+
# normalized through a known def-time wrapper. The runner compares
|
|
59
|
+
# what the mutant's evaluation installed with what is in place after
|
|
60
|
+
# the tests: a suite that re-loads the file mid-run (a before(:each)
|
|
61
|
+
# `load "lib/..."`, rolify-style) quietly replaces the mutant, and a
|
|
62
|
+
# "survivor" then only proves the tests never ran it. nil when this
|
|
63
|
+
# cannot be told (a missing or non-Ruby method): the caller keeps the
|
|
64
|
+
# verdict.
|
|
65
|
+
def installed_definition(subject)
|
|
66
|
+
owner = resolve_owner(subject)
|
|
67
|
+
target = subject.singleton ? owner.singleton_class : owner
|
|
68
|
+
method = target.instance_method(subject.name)
|
|
69
|
+
Wrapper.original_def(method) || method
|
|
70
|
+
rescue StandardError
|
|
71
|
+
nil
|
|
72
|
+
end
|
|
73
|
+
|
|
74
|
+
# The definition now in place for the subject versus the one the
|
|
75
|
+
# mutant's evaluation installed: a "file:line" site when the tests
|
|
76
|
+
# provably replaced it - a before(:each) re-loading the subject's
|
|
77
|
+
# own file (rolify-style `load "lib/..."`, #597) puts the original
|
|
78
|
+
# back before any test calls it, so a "survivor" proves nothing.
|
|
79
|
+
# Only a positively identified overwrite counts: the replacement
|
|
80
|
+
# comes from the subject's own file, and no prepend or
|
|
81
|
+
# allow_any_instance_of-style wrapper whose super chain still
|
|
82
|
+
# reaches the mutant (those tests did run it). Anything
|
|
83
|
+
# undecidable returns nil and the verdict stands.
|
|
84
|
+
def overwritten_site(subject, installed)
|
|
85
|
+
now = installed_definition(subject)
|
|
86
|
+
return nil if now.nil? || now == installed
|
|
87
|
+
|
|
88
|
+
method = now
|
|
89
|
+
method = method.super_method until method.nil? || method == installed
|
|
90
|
+
return nil if method # the mutant is still reachable through super
|
|
91
|
+
|
|
92
|
+
file, line = now.source_location
|
|
93
|
+
return nil unless file && File.expand_path(file) == File.expand_path(subject.file)
|
|
94
|
+
|
|
95
|
+
"#{file}:#{line}"
|
|
96
|
+
rescue StandardError
|
|
97
|
+
nil
|
|
98
|
+
end
|
|
99
|
+
|
|
57
100
|
# True when the subject's id matches a whole_file entry (exact or a
|
|
58
101
|
# File.fnmatch glob).
|
|
59
102
|
def whole_file?(subject)
|
|
103
|
+
return true if subject.whole_file_only?
|
|
104
|
+
|
|
60
105
|
@whole_file.any? { |pattern| File.fnmatch?(pattern, subject.id) }
|
|
61
106
|
end
|
|
62
107
|
|
|
@@ -87,12 +132,15 @@ module Saboteur
|
|
|
87
132
|
|
|
88
133
|
# Resolves the subject's owner the way Ruby would resolve the nesting in
|
|
89
134
|
# the file. const_get also triggers autoload (Zeitwerk) before any reopening.
|
|
135
|
+
# A `class << self` scope resolves the scopes after it (a factory
|
|
136
|
+
# constant assigned there binds on the singleton class, #352); when the
|
|
137
|
+
# subject's own scope is the singleton class, the owner is still the
|
|
138
|
+
# attached class.
|
|
90
139
|
def resolve_owner(subject)
|
|
91
|
-
scopes = [Object]
|
|
92
|
-
|
|
93
|
-
scopes << resolve_path(scope.path, scopes)
|
|
140
|
+
scopes = subject.nesting.each_with_object([Object]) do |scope, resolved|
|
|
141
|
+
resolved << (scope.sclass? ? resolved.last.singleton_class : resolve_path(scope.path, resolved))
|
|
94
142
|
end
|
|
95
|
-
owner = scopes.last
|
|
143
|
+
owner = subject.sclass ? scopes[-2] : scopes.last
|
|
96
144
|
raise Error, "#{subject.owner} is not a class or module" unless owner.is_a?(Module)
|
|
97
145
|
|
|
98
146
|
owner
|
|
@@ -139,28 +187,36 @@ module Saboteur
|
|
|
139
187
|
|
|
140
188
|
# The file as it is on disk, with the method's bytes replaced by text,
|
|
141
189
|
# evaluated at top level under the file's own name and first line.
|
|
142
|
-
# Redefinition warnings are expected and silenced.
|
|
143
190
|
def eval_whole_file(subject, text)
|
|
144
191
|
source = subject.file_source.dup.force_encoding(Encoding::UTF_8)
|
|
145
192
|
spliced = source.byteslice(0, subject.start_byte) + text + source.byteslice(subject.end_byte..)
|
|
146
|
-
|
|
147
|
-
$VERBOSE = nil
|
|
148
|
-
TOPLEVEL_BINDING.eval(spliced, subject.file, 1)
|
|
149
|
-
ensure
|
|
150
|
-
$VERBOSE = verbose
|
|
193
|
+
silencing_warnings { TOPLEVEL_BINDING.eval(spliced, subject.file, 1) }
|
|
151
194
|
end
|
|
152
195
|
|
|
153
196
|
def eval_in_owner(owner, subject, text, extra_lines)
|
|
154
197
|
code, line = with_magic_comment(subject, text, extra_lines)
|
|
155
198
|
eval_target = subject.sclass ? owner.singleton_class : owner
|
|
156
|
-
eval_target.class_eval(code, subject.file, line)
|
|
199
|
+
silencing_warnings { eval_target.class_eval(code, subject.file, line) }
|
|
157
200
|
end
|
|
158
201
|
|
|
159
202
|
def eval_nested(subject, text, extra_lines)
|
|
160
203
|
headers = subject.nesting.map(&:header)
|
|
161
204
|
wrapped = "#{headers.join('; ')}; #{text}; #{(['end'] * headers.size).join('; ')}"
|
|
162
205
|
code, line = with_magic_comment(subject, wrapped, extra_lines)
|
|
163
|
-
TOPLEVEL_BINDING.eval(code, subject.file, line)
|
|
206
|
+
silencing_warnings { TOPLEVEL_BINDING.eval(code, subject.file, line) }
|
|
207
|
+
end
|
|
208
|
+
|
|
209
|
+
# Re-evaluation redefines the method by design, and a "method
|
|
210
|
+
# redefined" warning is only ever emitted under $VERBOSE. Suites that
|
|
211
|
+
# raise on warnings (--warnings plus a raising Warning.warn or
|
|
212
|
+
# Warning.process handler, dry-rb/hanami-style) must not see it:
|
|
213
|
+
# before the guard every mutant errored as RuntimeError (#603).
|
|
214
|
+
def silencing_warnings
|
|
215
|
+
verbose = $VERBOSE
|
|
216
|
+
$VERBOSE = nil
|
|
217
|
+
yield
|
|
218
|
+
ensure
|
|
219
|
+
$VERBOSE = verbose
|
|
164
220
|
end
|
|
165
221
|
|
|
166
222
|
# The first line of code sits extra_lines above the def, so the def
|
|
@@ -3,6 +3,7 @@
|
|
|
3
3
|
|
|
4
4
|
require "json"
|
|
5
5
|
require_relative "stale_captures"
|
|
6
|
+
require_relative "../crash_report"
|
|
6
7
|
|
|
7
8
|
module Saboteur
|
|
8
9
|
module Execution
|
|
@@ -39,17 +40,42 @@ module Saboteur
|
|
|
39
40
|
end
|
|
40
41
|
end
|
|
41
42
|
|
|
43
|
+
# The child log (or its whole directory) vanished mid-job (#234).
|
|
44
|
+
# Carries the digest of the mutant whose run just ended when the job
|
|
45
|
+
# had one, so a parallel parent can prefer this report over a
|
|
46
|
+
# bystander's bare "unknown mutant" report (#285).
|
|
47
|
+
class LogVanished < Error
|
|
48
|
+
attr_reader :culprit_digest
|
|
49
|
+
|
|
50
|
+
def initialize(message, culprit_digest: nil)
|
|
51
|
+
@culprit_digest = culprit_digest
|
|
52
|
+
super(message)
|
|
53
|
+
end
|
|
54
|
+
end
|
|
55
|
+
|
|
42
56
|
LOG_TAIL_BYTES = 2000
|
|
57
|
+
CRASH_LOG_BYTES = 2_000
|
|
58
|
+
CRASH_LOG_LINES = 8
|
|
59
|
+
# File mtimes follow the filesystem's clock, which can trail the
|
|
60
|
+
# process clock; a log written during this run can stamp a little
|
|
61
|
+
# before its start, so the freshness check allows that skew.
|
|
62
|
+
CRASH_LOG_CLOCK_SLACK = 2
|
|
63
|
+
# Freshness matching can attach another run's log (clock slack, two
|
|
64
|
+
# nested runs in one child), so the label must not claim exact
|
|
65
|
+
# attribution (#296).
|
|
66
|
+
CRASH_LOG_LABEL = "recent saboteur crash log (freshness-matched, may be another run's):"
|
|
43
67
|
|
|
44
68
|
# log_dir: where children write their output; a child's log is kept
|
|
45
|
-
# only when it ends in an error.
|
|
69
|
+
# only when it ends in an error. crash_log: the crash log of a saboteur
|
|
70
|
+
# run nested inside a test; by default it sits next to the logs dir,
|
|
71
|
+
# where the engine's cache dir keeps it.
|
|
46
72
|
# seed: every child seeds Ruby's random numbers with it before its
|
|
47
73
|
# tests, so a test that uses rand sees the same values in every run.
|
|
48
74
|
# sandbox: when given, a callable run first in every child (Landlock
|
|
49
75
|
# makes the project tree read-only there, #152). It raises rather than
|
|
50
76
|
# run half-restricted, which the child reports as an error.
|
|
51
77
|
def initialize(integration:, reevaluator:, log_dir:, hooks: Hooks.new, supervisor: Supervisor.new, seed: 1,
|
|
52
|
-
sandbox: nil)
|
|
78
|
+
sandbox: nil, canary: nil, crash_log: nil)
|
|
53
79
|
@integration = integration
|
|
54
80
|
@reevaluator = reevaluator
|
|
55
81
|
@hooks = hooks
|
|
@@ -57,6 +83,8 @@ module Saboteur
|
|
|
57
83
|
@supervisor = supervisor
|
|
58
84
|
@seed = seed
|
|
59
85
|
@sandbox = sandbox
|
|
86
|
+
@canary = canary
|
|
87
|
+
@crash_log = crash_log || File.join(File.dirname(log_dir), CrashReport::LOG_NAME)
|
|
60
88
|
end
|
|
61
89
|
|
|
62
90
|
# text: method source to evaluate in the child (the unchanged source for
|
|
@@ -69,23 +97,33 @@ module Saboteur
|
|
|
69
97
|
@sequence = @sequence.to_i + 1
|
|
70
98
|
job = Job.new(subject:, text:, test_ids:, timeout:, mutation:, eval_prefix:,
|
|
71
99
|
log_path: File.join(@log_dir, "child-#{Process.pid}-#{@sequence}.log"))
|
|
72
|
-
|
|
100
|
+
log_file = open_log(job)
|
|
101
|
+
result = @supervisor.run(timeout: timeout) { |writer| child(writer, job, log_file) }
|
|
73
102
|
outcome = outcome_for(result, job)
|
|
74
|
-
|
|
103
|
+
check_tree_still_there(job)
|
|
104
|
+
check_log_still_there(job)
|
|
105
|
+
File.delete(job.log_path) if outcome.status != :error
|
|
75
106
|
outcome
|
|
107
|
+
ensure
|
|
108
|
+
log_file&.close
|
|
76
109
|
end
|
|
77
110
|
|
|
78
111
|
private
|
|
79
112
|
|
|
80
113
|
# --- child ----------------------------------------------------------
|
|
81
114
|
|
|
82
|
-
def child(writer, job)
|
|
115
|
+
def child(writer, job, log_file)
|
|
83
116
|
# Redirect before restricting: the sweep closes inherited stdio
|
|
84
117
|
# redirected to a regular file outside the grants (#178), and a
|
|
85
118
|
# reopened-afterwards $stdout would start from a closed stream.
|
|
86
|
-
# The
|
|
87
|
-
|
|
88
|
-
|
|
119
|
+
# The parent opened the log before the fork, so the inherited
|
|
120
|
+
# descriptor survives on the keep list - the cache dir itself
|
|
121
|
+
# stays read-only inside the sandbox (#234). stdio was just
|
|
122
|
+
# reopened onto that log, so 1 and 2 point at a regular file
|
|
123
|
+
# outside the grants and the sweep would close them; keep them
|
|
124
|
+
# explicitly - keeping only the log's own fd is not enough (#283).
|
|
125
|
+
redirect_output(log_file)
|
|
126
|
+
@sandbox&.call([writer.fileno, log_file.fileno, $stdout.fileno, $stderr.fileno])
|
|
89
127
|
Process.setproctitle(title(job))
|
|
90
128
|
@hooks.run(:after_fork)
|
|
91
129
|
# Before the evaluation: a whole-file subject runs the file's own
|
|
@@ -106,6 +144,7 @@ module Saboteur
|
|
|
106
144
|
@reevaluator.call(job.subject, job.text, prefix: job.eval_prefix)
|
|
107
145
|
replaced = @reevaluator.replaced_definition(job.subject)
|
|
108
146
|
Protocol.emit(writer, "replaced", site: replaced) if replaced
|
|
147
|
+
@installed_definition = job.mutation ? @reevaluator.installed_definition(job.subject) : nil
|
|
109
148
|
end
|
|
110
149
|
|
|
111
150
|
# Capture detection is best-effort: a subject shape it cannot scan
|
|
@@ -118,18 +157,103 @@ module Saboteur
|
|
|
118
157
|
|
|
119
158
|
def emit_result(writer, job, has_captures)
|
|
120
159
|
method = replacement_method(job) if has_captures && job.mutation
|
|
121
|
-
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
end
|
|
160
|
+
flags = { replacement_called: method ? false : nil, mutant_called: false }
|
|
161
|
+
target = method || (job.mutation ? @installed_definition : nil)
|
|
162
|
+
trace = call_probe(target, flags)
|
|
163
|
+
started_at = Time.now
|
|
164
|
+
suite = proc { @hooks.around(job.mutation) { run_tests(job.test_ids) } }
|
|
165
|
+
result = trace ? trace.enable(target: target, &suite) : suite.call
|
|
128
166
|
if result.nil?
|
|
129
167
|
Protocol.emit(writer, "error", message: "around_mutant hook did not run the tests")
|
|
130
168
|
else
|
|
131
|
-
|
|
169
|
+
called = flags[:replacement_called]
|
|
170
|
+
Protocol.emit(writer, "result", **with_crash_detail(result, started_at), replacement_called: called)
|
|
171
|
+
# A reload after the mutant already ran (an after-hook restore)
|
|
172
|
+
# is no reason to doubt the verdict; only a never-invoked mutant
|
|
173
|
+
# can have been hidden by the overwrite.
|
|
174
|
+
emit_overwritten(writer, job, flags[:mutant_called])
|
|
175
|
+
end
|
|
176
|
+
end
|
|
177
|
+
|
|
178
|
+
# A reload after the mutant already ran (an after-hook restore) is
|
|
179
|
+
# no reason to doubt the verdict; only a never-invoked mutant can
|
|
180
|
+
# have been hidden by the overwrite.
|
|
181
|
+
def emit_overwritten(writer, job, mutant_called)
|
|
182
|
+
overwritten = job.mutation && @installed_definition &&
|
|
183
|
+
@reevaluator.overwritten_site(job.subject, @installed_definition)
|
|
184
|
+
Protocol.emit(writer, "overwritten", site: overwritten) if overwritten && !mutant_called
|
|
185
|
+
end
|
|
186
|
+
|
|
187
|
+
# One call probe on the installed definition feeds both verdicts:
|
|
188
|
+
# replacement_called for stale captures, and overwrite detection for
|
|
189
|
+
# a mutant the suite replaced before any test ran it (#597).
|
|
190
|
+
def call_probe(target, flags)
|
|
191
|
+
target && TracePoint.new(:call) do
|
|
192
|
+
flags[:mutant_called] = true
|
|
193
|
+
flags[:replacement_called] = true unless flags[:replacement_called].nil?
|
|
194
|
+
end
|
|
195
|
+
end
|
|
196
|
+
|
|
197
|
+
# A test can run saboteur itself (the dogfood's cli_spec): the nested
|
|
198
|
+
# CLI reports its own crashes as exit 3 and logs them to
|
|
199
|
+
# internal-error.log. A log written during this run explains the
|
|
200
|
+
# failure better than the bare expectation diff, so its first lines
|
|
201
|
+
# join each failure detail; an older log is another run's (#296).
|
|
202
|
+
def with_crash_detail(result, started_at)
|
|
203
|
+
excerpt = crash_log_excerpt(started_at)
|
|
204
|
+
details = result[:failure_details]
|
|
205
|
+
return result unless excerpt && details && !details.empty?
|
|
206
|
+
|
|
207
|
+
result.merge(failure_details: details.transform_values { |d| [d, CRASH_LOG_LABEL, excerpt].join("\n") })
|
|
208
|
+
end
|
|
209
|
+
|
|
210
|
+
# The nested CLI writes its log to the cache dir, or to the pid-keyed
|
|
211
|
+
# temp fallback when the sandbox holds the cache dir read-only (#296).
|
|
212
|
+
def crash_log_excerpt(started_at)
|
|
213
|
+
excerpt = tmp_log_candidates.filter_map { |path| fresh_log_excerpt(path, started_at) }
|
|
214
|
+
.max_by { |mtime, _text| mtime }
|
|
215
|
+
excerpt&.last
|
|
216
|
+
rescue StandardError
|
|
217
|
+
nil
|
|
218
|
+
end
|
|
219
|
+
|
|
220
|
+
# The log paths are predictable and live in shared dirs, so read the
|
|
221
|
+
# log the way CrashReport writes it: through a no-follow, non-blocking
|
|
222
|
+
# descriptor,
|
|
223
|
+
# and validate on that same fd - a regular file, owned by this uid,
|
|
224
|
+
# with no group/other permission bits, fresh by the fd's own mtime.
|
|
225
|
+
# One open, no check-then-reopen race; a symlink, a foreign, wide or
|
|
226
|
+
# non-regular file, or any IO failure is no excerpt, never an
|
|
227
|
+
# exception (#296).
|
|
228
|
+
def fresh_log_excerpt(path, started_at)
|
|
229
|
+
# No safe open, no excerpt: without O_NOFOLLOW a symlink redirects
|
|
230
|
+
# the read, without O_NONBLOCK a no-writer FIFO blocks it (#296).
|
|
231
|
+
return unless CrashReport.safe_log_io?
|
|
232
|
+
|
|
233
|
+
flags = File::RDONLY | CrashReport.nofollow_flag | CrashReport.nonblock_flag
|
|
234
|
+
File.open(path, flags) do |file|
|
|
235
|
+
stat = file.stat
|
|
236
|
+
next unless stat.file? && stat.uid == Process.uid && stat.mode.nobits?(0o077)
|
|
237
|
+
next unless stat.mtime >= started_at - CRASH_LOG_CLOCK_SLACK
|
|
238
|
+
|
|
239
|
+
text = file.read(CRASH_LOG_BYTES)
|
|
240
|
+
# Scrub before the excerpt travels as JSON: the bounded read can
|
|
241
|
+
# end mid-character and the log can hold locale-invalid bytes, and
|
|
242
|
+
# a JSON::GeneratorError here would turn a kill into a child error
|
|
243
|
+
# (#296).
|
|
244
|
+
[stat.mtime, text ? CrashReport.scrub(text).lines.first(CRASH_LOG_LINES).join : nil]
|
|
132
245
|
end
|
|
246
|
+
rescue SystemCallError, IOError
|
|
247
|
+
nil
|
|
248
|
+
end
|
|
249
|
+
|
|
250
|
+
# The excerpt is diagnostic enrichment: when CrashReport itself is the
|
|
251
|
+
# mutated file, tmp_log_path can raise or return garbage - keep the
|
|
252
|
+
# primary candidate and never let the lookup change a verdict (#296).
|
|
253
|
+
def tmp_log_candidates
|
|
254
|
+
[@crash_log, CrashReport.tmp_log_path].compact.uniq.grep(String)
|
|
255
|
+
rescue StandardError
|
|
256
|
+
[@crash_log].compact
|
|
133
257
|
end
|
|
134
258
|
|
|
135
259
|
# Targeted tracing distinguishes the new Ruby body from an old Method
|
|
@@ -151,19 +275,69 @@ module Saboteur
|
|
|
151
275
|
def run_tests(test_ids)
|
|
152
276
|
result = @integration.run(test_ids)
|
|
153
277
|
{ passed: result.passed, failed: result.failed_ids, executed: result.executed_ids, runtime: result.runtime,
|
|
154
|
-
failure_classes: result.failure_classes, failure_messages: result.failure_messages
|
|
278
|
+
failure_classes: result.failure_classes, failure_messages: result.failure_messages,
|
|
279
|
+
failure_details: result.failure_details }
|
|
155
280
|
rescue SystemExit => e
|
|
156
281
|
# A test called `exit`. The tests noticed something; count it as a failure.
|
|
157
282
|
{ passed: false, failed: [], executed: [], runtime: 0.0, exited: e.status }
|
|
158
283
|
end
|
|
159
284
|
|
|
160
|
-
def redirect_output(
|
|
161
|
-
$stdout.reopen(
|
|
285
|
+
def redirect_output(log_file)
|
|
286
|
+
$stdout.reopen(log_file)
|
|
162
287
|
$stderr.reopen($stdout)
|
|
163
288
|
$stdout.sync = true
|
|
164
289
|
$stderr.sync = true
|
|
165
290
|
end
|
|
166
291
|
|
|
292
|
+
# Names the just-run mutant when there is one (a verdict-check rerun
|
|
293
|
+
# has none): the most likely culprit, worth naming for the report.
|
|
294
|
+
def culprit(job)
|
|
295
|
+
culprit_digest(job) || "an unknown mutant"
|
|
296
|
+
end
|
|
297
|
+
|
|
298
|
+
# The 16-hex digest of the just-run mutant, or nil when the job ran
|
|
299
|
+
# none (a neutral or verdict-check rerun): a parallel parent prefers
|
|
300
|
+
# the vanished-log report that carries one (#285).
|
|
301
|
+
def culprit_digest(job)
|
|
302
|
+
return unless job.mutation && job.subject.respond_to?(:start_byte)
|
|
303
|
+
|
|
304
|
+
job.mutation.digest(subject_start: job.subject.start_byte,
|
|
305
|
+
original_source: job.subject.file_source)
|
|
306
|
+
end
|
|
307
|
+
|
|
308
|
+
# Opened in the parent, not the sandboxed child: the cache dir is
|
|
309
|
+
# not writable inside the sandbox (#234), and a log directory that
|
|
310
|
+
# is already gone is a tool problem to report, not a mutant result.
|
|
311
|
+
def open_log(job)
|
|
312
|
+
File.open(job.log_path, "w")
|
|
313
|
+
rescue SystemCallError => e
|
|
314
|
+
raise Error, "cannot open the child log #{job.log_path} (#{e.class}: #{e.message})"
|
|
315
|
+
end
|
|
316
|
+
|
|
317
|
+
# A mutant's tests can delete saboteur's own files when the sandbox
|
|
318
|
+
# is off or degraded (#234): a vanished log makes the verdict
|
|
319
|
+
# worthless, so fail as a tool error instead of trusting the run.
|
|
320
|
+
# The mutant that just ran is the likely - not proven - cause: a
|
|
321
|
+
# leftover process from an earlier mutant could have done it.
|
|
322
|
+
# The per-mutant probe (#235): names the mutant whose run just wiped
|
|
323
|
+
# tracked files, instead of blaming the subject's last mutant when the
|
|
324
|
+
# loss surfaces at subject end. Runs before the log check, so the
|
|
325
|
+
# hazard report still wins when a mutant took both (#283).
|
|
326
|
+
def check_tree_still_there(job)
|
|
327
|
+
# culprit_digest, not the display fallback: the digest doubles as
|
|
328
|
+
# trip metadata, and "an unknown mutant" must not rank as one (#285).
|
|
329
|
+
@canary&.probe!(subject: job.subject, digest: culprit_digest(job))
|
|
330
|
+
end
|
|
331
|
+
|
|
332
|
+
def check_log_still_there(job)
|
|
333
|
+
return if File.exist?(job.log_path)
|
|
334
|
+
|
|
335
|
+
cause = "a mutant's tests may have hit saboteur's files"
|
|
336
|
+
raise LogVanished.new("the child log #{job.log_path} vanished mid-run - the mutant that just ran, " \
|
|
337
|
+
"#{culprit(job)}, is the likely cause: #{cause}",
|
|
338
|
+
culprit_digest: culprit_digest(job))
|
|
339
|
+
end
|
|
340
|
+
|
|
167
341
|
# --- parent ---------------------------------------------------------
|
|
168
342
|
|
|
169
343
|
def outcome_for(result, job)
|
|
@@ -197,7 +371,7 @@ module Saboteur
|
|
|
197
371
|
end
|
|
198
372
|
end
|
|
199
373
|
|
|
200
|
-
def classify_result(result, runtime, test_ids)
|
|
374
|
+
def classify_result(result, runtime, test_ids, verbose_failure: false)
|
|
201
375
|
failed = Array(result["failed"])
|
|
202
376
|
executed = Array(result["executed"])
|
|
203
377
|
return Outcome.new(status: :killed, runtime: runtime, message: "test called exit") if result.key?("exited")
|
|
@@ -206,9 +380,7 @@ module Saboteur
|
|
|
206
380
|
unexpected = failed - test_ids
|
|
207
381
|
return mismatch(runtime, "failed tests not requested: #{unexpected.first(3).join(', ')}") if unexpected.any?
|
|
208
382
|
|
|
209
|
-
return
|
|
210
|
-
message: result.fetch("failure_messages", {})[failed.first],
|
|
211
|
-
kill_cause: kill_cause(result.fetch("failure_classes", {})[failed.first]))
|
|
383
|
+
return killed_outcome(result, runtime, failed, executed, verbose_failure)
|
|
212
384
|
end
|
|
213
385
|
unless result["passed"]
|
|
214
386
|
return Outcome.new(status: :killed, runtime: runtime, executed_ids: executed,
|
|
@@ -222,14 +394,30 @@ module Saboteur
|
|
|
222
394
|
Outcome.new(status: :survived, runtime: runtime, executed_ids: executed)
|
|
223
395
|
end
|
|
224
396
|
|
|
397
|
+
def killed_outcome(result, runtime, failed, executed, verbose_failure)
|
|
398
|
+
Outcome.new(status: :killed, runtime: runtime, failed_ids: failed, executed_ids: executed,
|
|
399
|
+
message: failure_message(result, failed.first, verbose_failure),
|
|
400
|
+
kill_cause: kill_cause(result.fetch("failure_classes", {})[failed.first]))
|
|
401
|
+
end
|
|
402
|
+
|
|
403
|
+
# A neutral check (the verdict re-run, no mutation) needs the full
|
|
404
|
+
# failure to explain why the killer fails unchanged (#296).
|
|
405
|
+
def failure_message(result, test_id, verbose_failure)
|
|
406
|
+
message = result.fetch("failure_messages", {})[test_id]
|
|
407
|
+
verbose_failure ? result.fetch("failure_details", {})[test_id] || message : message
|
|
408
|
+
end
|
|
409
|
+
|
|
225
410
|
def classify_events(result, events, runtime, job)
|
|
226
|
-
outcome = classify_result(result, runtime, job.test_ids)
|
|
411
|
+
outcome = classify_result(result, runtime, job.test_ids, verbose_failure: job.mutation.nil?)
|
|
227
412
|
replaced = events.find { |event| event["event"] == "replaced" }
|
|
228
413
|
return replaced_by_later_definition(outcome, replaced) if never_ran?(outcome, replaced, job)
|
|
229
414
|
|
|
230
415
|
captures = events.find { |event| event["event"] == "captures" }
|
|
231
416
|
return unreachable(outcome, captures) if unobservable?(outcome, captures, job, result["replacement_called"])
|
|
232
417
|
|
|
418
|
+
overwritten = events.find { |event| event["event"] == "overwritten" }
|
|
419
|
+
return overwritten_by_later_load(outcome, overwritten) if replaced_while_running?(outcome, overwritten, job)
|
|
420
|
+
|
|
233
421
|
outcome
|
|
234
422
|
end
|
|
235
423
|
|
|
@@ -252,6 +440,19 @@ module Saboteur
|
|
|
252
440
|
captures && job.mutation && outcome.status == :survived && replacement_called == false
|
|
253
441
|
end
|
|
254
442
|
|
|
443
|
+
# A surviving mutant the tests themselves overwrote: a before(:each)
|
|
444
|
+
# re-loading the file replaced it before any test ran (rolify's suite
|
|
445
|
+
# shape, #597), so the verdict says nothing about the tests.
|
|
446
|
+
def replaced_while_running?(outcome, overwritten, job)
|
|
447
|
+
overwritten && job.mutation && outcome.status == :survived
|
|
448
|
+
end
|
|
449
|
+
|
|
450
|
+
def overwritten_by_later_load(outcome, overwritten)
|
|
451
|
+
Outcome.new(status: :unreachable, runtime: outcome.runtime,
|
|
452
|
+
message: "the tests (or their hooks) replaced the definition while they ran " \
|
|
453
|
+
"(#{overwritten['site']}), so this mutant never ran")
|
|
454
|
+
end
|
|
455
|
+
|
|
255
456
|
# A survivor that no test could have observed: a Method object
|
|
256
457
|
# captured at load still calls the original. Not a verdict on the
|
|
257
458
|
# tests, so it is its own status, out of the score.
|
|
@@ -57,8 +57,15 @@ module Saboteur
|
|
|
57
57
|
return @abi_version if defined?(@abi_version)
|
|
58
58
|
return @abi_version = 0 unless linux?
|
|
59
59
|
|
|
60
|
-
|
|
61
|
-
errno
|
|
60
|
+
# Build the errno reader BEFORE the probe: its lazy dlopen/dlsym run
|
|
61
|
+
# libc calls that can set errno, so building it between the probe and
|
|
62
|
+
# the read can replace the real answer with a stale 0.
|
|
63
|
+
LibC.errno_location_fn
|
|
64
|
+
version, errno = probe_once
|
|
65
|
+
# A failed syscall always sets errno, so (failure, errno 0) is never
|
|
66
|
+
# the kernel's real answer - the read was clobbered between the call
|
|
67
|
+
# and here. Probe once more, now with every Fiddle function warm.
|
|
68
|
+
version, errno = probe_once if !version.positive? && errno.zero?
|
|
62
69
|
@abi_version = probe_abi(version, errno)
|
|
63
70
|
rescue Fiddle::DLError
|
|
64
71
|
# No usable libc (an unusual musl/ldso layout): the sandbox cannot
|
|
@@ -70,6 +77,12 @@ module Saboteur
|
|
|
70
77
|
# no Landlock to give: run unsandboxed, noted once. Anything else
|
|
71
78
|
# (EPERM under a seccomp policy, ...) means the kernel is there but
|
|
72
79
|
# refused - fail closed rather than run without the promised sandbox.
|
|
80
|
+
# One create-ruleset version probe with its errno read paired to it.
|
|
81
|
+
def probe_once
|
|
82
|
+
version = LibC.syscall_fn.call(SYS_CREATE_RULESET, 0, 0, CREATE_RULESET_VERSION, 0)
|
|
83
|
+
[version, last_errno]
|
|
84
|
+
end
|
|
85
|
+
|
|
73
86
|
def probe_abi(version, errno)
|
|
74
87
|
return version if version.positive?
|
|
75
88
|
return 0 if UNSUPPORTED_ERRNOS.include?(errno)
|
|
@@ -160,7 +173,12 @@ module Saboteur
|
|
|
160
173
|
return unless stat.file?
|
|
161
174
|
|
|
162
175
|
granted = target.delete_suffix(" (deleted)")
|
|
163
|
-
|
|
176
|
+
# readlink targets arrive tagged with default_external, so under
|
|
177
|
+
# LANG=C a non-ASCII target compared against a UTF-8 grant pattern
|
|
178
|
+
# raises Encoding::CompatibilityError before any test runs (CI #272).
|
|
179
|
+
# Compare bytes.
|
|
180
|
+
target_bytes = granted.b
|
|
181
|
+
return if writable.any? { |dir| target_bytes == dir.b || target_bytes.start_with?(dir.b + "/") } # rubocop:disable Style/StringConcatenation -- interpolation with a BINARY part raises on high bytes
|
|
164
182
|
|
|
165
183
|
flags = LibC.fcntl_fn.call(descriptor, F_GETFL, 0)
|
|
166
184
|
return close_fd(descriptor) if flags.negative?
|