saboteur 0.6.0 → 0.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +367 -0
- data/README.md +166 -40
- data/exe/saboteur +34 -7
- data/lib/saboteur/baseline/workflow.rb +2 -1
- data/lib/saboteur/baseline_health.rb +40 -6
- data/lib/saboteur/cli/exit_status.rb +49 -4
- data/lib/saboteur/cli/help_options.rb +16 -0
- data/lib/saboteur/cli/presets.rb +36 -0
- data/lib/saboteur/cli/run_budget_options.rb +37 -0
- data/lib/saboteur/cli/run_reporting.rb +74 -5
- data/lib/saboteur/cli.rb +67 -46
- data/lib/saboteur/config.rb +66 -11
- data/lib/saboteur/crash_report.rb +154 -0
- data/lib/saboteur/custom_map.rb +3 -1
- data/lib/saboteur/doctor.rb +40 -1
- data/lib/saboteur/engine/baseline_guard.rb +79 -5
- data/lib/saboteur/engine/boot.rb +12 -1
- data/lib/saboteur/engine/canary.rb +41 -10
- data/lib/saboteur/engine/candidate_selection.rb +9 -1
- data/lib/saboteur/engine/coverage_guard.rb +29 -0
- data/lib/saboteur/engine/coverage_store.rb +48 -6
- data/lib/saboteur/engine/diagnostics.rb +23 -2
- data/lib/saboteur/engine/diff_scope.rb +24 -5
- data/lib/saboteur/engine/distribution.rb +124 -8
- data/lib/saboteur/engine/early_stop_recording.rb +46 -0
- data/lib/saboteur/engine/result_cache.rb +2 -1
- data/lib/saboteur/engine/run_budget.rb +63 -0
- data/lib/saboteur/engine/run_fingerprint.rb +4 -0
- data/lib/saboteur/engine/sandbox_policy.rb +18 -11
- data/lib/saboteur/engine/scheduler.rb +114 -18
- data/lib/saboteur/engine/subject_selection.rb +84 -5
- data/lib/saboteur/engine/suite_files.rb +3 -1
- data/lib/saboteur/engine/test_guess.rb +145 -22
- data/lib/saboteur/engine/thread_warnings.rb +53 -0
- data/lib/saboteur/engine/unscoped_suite_warning.rb +41 -0
- data/lib/saboteur/engine/verdict_check.rb +54 -18
- data/lib/saboteur/engine.rb +166 -95
- data/lib/saboteur/execution/protocol.rb +2 -1
- data/lib/saboteur/execution/runner.rb +179 -17
- data/lib/saboteur/execution/sandbox.rb +6 -1
- data/lib/saboteur/execution/supervisor.rb +18 -3
- data/lib/saboteur/execution/worker_pool.rb +87 -7
- data/lib/saboteur/help.rb +55 -11
- data/lib/saboteur/ignore_list.rb +11 -1
- data/lib/saboteur/install.rb +4 -3
- data/lib/saboteur/integration/base.rb +7 -2
- data/lib/saboteur/integration/rspec.rb +40 -4
- data/lib/saboteur/mutation_source/builtin.rb +20 -11
- data/lib/saboteur/mutation_source/hazard_guard.rb +21 -3
- data/lib/saboteur/mutation_source/interpolation_removal.rb +41 -0
- data/lib/saboteur/mutation_source/method_swap.rb +45 -0
- data/lib/saboteur/mutation_source/mutation.rb +5 -3
- data/lib/saboteur/mutation_source/operand_replacement.rb +54 -0
- data/lib/saboteur/mutation_source/operator_examples.rb +4 -1
- data/lib/saboteur/mutation_source/operator_registry.rb +4 -1
- data/lib/saboteur/mutation_source/operator_visit.rb +39 -0
- data/lib/saboteur/preview.rb +2 -1
- data/lib/saboteur/reporter/agent_commands.rb +38 -13
- data/lib/saboteur/reporter/agent_incomplete.rb +27 -2
- data/lib/saboteur/reporter/next_steps.rb +14 -4
- data/lib/saboteur/reporter/score_history.rb +5 -1
- data/lib/saboteur/reporter/stryker.rb +13 -4
- data/lib/saboteur/reporter/survivor_hints.rb +3 -0
- data/lib/saboteur/reporter/terminal.rb +34 -10
- data/lib/saboteur/result.rb +22 -7
- data/lib/saboteur/selection/coverage_map.rb +2 -1
- data/lib/saboteur/selection/git_diff.rb +15 -5
- data/lib/saboteur/selection/untracked_changes.rb +126 -0
- data/lib/saboteur/subject.rb +11 -0
- data/lib/saboteur/text.rb +42 -0
- data/lib/saboteur/version.rb +1 -1
- data/lib/saboteur.rb +16 -1
- metadata +17 -2
|
@@ -0,0 +1,154 @@
|
|
|
1
|
+
# SPDX-License-Identifier: MIT
|
|
2
|
+
# frozen_string_literal: true
|
|
3
|
+
|
|
4
|
+
require "fileutils"
|
|
5
|
+
require "json"
|
|
6
|
+
require "tmpdir"
|
|
7
|
+
require "pathname"
|
|
8
|
+
|
|
9
|
+
module Saboteur
|
|
10
|
+
# Last-resort crash reporting, shared by the CLI's StandardError rescue
|
|
11
|
+
# and the exe-level wrapper: one bounded line on stderr, the full
|
|
12
|
+
# backtrace in a log file the line names, the backtrace on stderr with
|
|
13
|
+
# --verbose, and the same failure as agent JSON when an agent listens.
|
|
14
|
+
# Always returns EXIT_TOOL.
|
|
15
|
+
module CrashReport
|
|
16
|
+
ISSUES_URL = "https://github.com/vamsipavanmahesh/saboteur/issues"
|
|
17
|
+
|
|
18
|
+
# Same value as CLI::EXIT_TOOL, kept local so the reporter still works
|
|
19
|
+
# when the CLI itself never finished loading.
|
|
20
|
+
EXIT_TOOL = 3
|
|
21
|
+
|
|
22
|
+
# The crash log's file name under the run's cache dir; the runner reads
|
|
23
|
+
# it back to explain a nested run's crash in the failure detail (#296).
|
|
24
|
+
LOG_NAME = "internal-error.log"
|
|
25
|
+
|
|
26
|
+
module_function
|
|
27
|
+
|
|
28
|
+
def call(error, verbose:, agent:, err:, out:, log_path: nil)
|
|
29
|
+
summary = "internal error (#{error.class}: #{one_line(error.message)})"
|
|
30
|
+
line = "saboteur: #{summary} - please report it: #{ISSUES_URL}"
|
|
31
|
+
written = write_log(log_path, error)
|
|
32
|
+
line = "#{line}; full backtrace: #{display_path(written)}" if written
|
|
33
|
+
err.puts(line)
|
|
34
|
+
err.puts(full_text(error)) if verbose
|
|
35
|
+
out.print(agent_json(summary)) if agent
|
|
36
|
+
EXIT_TOOL
|
|
37
|
+
end
|
|
38
|
+
|
|
39
|
+
# A crash message can span lines or carry bytes that are invalid in
|
|
40
|
+
# UTF-8 (a LANG=C failure, say); the one-line diagnostic must survive
|
|
41
|
+
# both, and stay one bounded line.
|
|
42
|
+
def one_line(message)
|
|
43
|
+
text = scrub(message).gsub(/\s+/, " ").strip
|
|
44
|
+
text.length > 160 ? "#{text[0...160]}..." : text
|
|
45
|
+
end
|
|
46
|
+
|
|
47
|
+
# Class, full message and backtrace, scrubbed so locale-invalid bytes
|
|
48
|
+
# cannot corrupt the log or the terminal.
|
|
49
|
+
def full_text(error)
|
|
50
|
+
(["#{error.class}: #{scrub(error.message)}"] + Array(error.backtrace).map { |frame| scrub(frame) }).join("\n")
|
|
51
|
+
end
|
|
52
|
+
|
|
53
|
+
# Writes the log next to the other run state; returns the written path,
|
|
54
|
+
# or nil when the log cannot be written - the crash itself must still
|
|
55
|
+
# be reported, so an unwritable directory is not a second failure.
|
|
56
|
+
# Reporter::Agent.failure's payload, reproduced minimally for crashes
|
|
57
|
+
# before the CLI's reporter files load: same schema and pretty rendering;
|
|
58
|
+
# the summary is already normalized, so no message cap is needed here.
|
|
59
|
+
def agent_json(summary)
|
|
60
|
+
payload =
|
|
61
|
+
if defined?(Reporter::Agent)
|
|
62
|
+
Reporter::Agent.failure(summary, EXIT_TOOL)
|
|
63
|
+
else
|
|
64
|
+
{ version: 1, status: "error", verdict: "tool_failure", exitCode: EXIT_TOOL,
|
|
65
|
+
message: summary, next: { help: "saboteur help run" } }
|
|
66
|
+
end
|
|
67
|
+
"#{JSON.pretty_generate(payload)}\n"
|
|
68
|
+
end
|
|
69
|
+
|
|
70
|
+
def write_log(log_path, error)
|
|
71
|
+
return nil unless log_path
|
|
72
|
+
|
|
73
|
+
# The primary path is user-configured but still predictable: without
|
|
74
|
+
# O_NOFOLLOW a planted symlink would redirect the truncate/chmod, and
|
|
75
|
+
# without O_NONBLOCK a planted FIFO would block the open. Where the
|
|
76
|
+
# platform lacks either, only the EXCL-created temp fallback is safe
|
|
77
|
+
# (#296).
|
|
78
|
+
if safe_log_io?
|
|
79
|
+
begin
|
|
80
|
+
FileUtils.mkdir_p(File.dirname(log_path))
|
|
81
|
+
# Private like the temp fallback: the runner only reads logs with
|
|
82
|
+
# no group/other bits, from the same no-follow fd (#296).
|
|
83
|
+
flags = File::CREAT | File::WRONLY | File::TRUNC | nofollow_flag | nonblock_flag
|
|
84
|
+
File.open(log_path, flags, 0o600) do |file|
|
|
85
|
+
file.chmod(0o600)
|
|
86
|
+
file.write("#{full_text(error)}\n")
|
|
87
|
+
end
|
|
88
|
+
return log_path
|
|
89
|
+
rescue SystemCallError, IOError
|
|
90
|
+
# Fall through to the temp fallback.
|
|
91
|
+
end
|
|
92
|
+
end
|
|
93
|
+
write_tmp_log(error)
|
|
94
|
+
end
|
|
95
|
+
|
|
96
|
+
# The fallback is computed lazily, only once the primary write failed:
|
|
97
|
+
# an eager tmp_log_path call could itself raise (Dir.tmpdir finds no
|
|
98
|
+
# usable dir) and mask the crash it was meant to log (#296).
|
|
99
|
+
#
|
|
100
|
+
# The fallback path is predictable and lives in a shared directory, so
|
|
101
|
+
# it is created privately: CREAT|EXCL|WRONLY|NOFOLLOW never follows or
|
|
102
|
+
# overwrites a pre-existing file or symlink, and the mode pins the log
|
|
103
|
+
# to 0600. A taken or hostile path means no log - the crash report
|
|
104
|
+
# itself must still go out, so every failure here degrades to nil.
|
|
105
|
+
def write_tmp_log(error)
|
|
106
|
+
File.open(tmp_log_path, File::CREAT | File::EXCL | File::WRONLY | nofollow_flag, 0o600) do |file|
|
|
107
|
+
file.write("#{full_text(error)}\n")
|
|
108
|
+
end
|
|
109
|
+
tmp_log_path
|
|
110
|
+
rescue SystemCallError, IOError, ArgumentError
|
|
111
|
+
nil
|
|
112
|
+
end
|
|
113
|
+
|
|
114
|
+
# O_NOFOLLOW where the platform has it (every CI target does).
|
|
115
|
+
def nofollow_flag
|
|
116
|
+
File.const_defined?(:NOFOLLOW) ? File::NOFOLLOW : 0
|
|
117
|
+
end
|
|
118
|
+
|
|
119
|
+
# O_NONBLOCK where the platform has it (every CI target does).
|
|
120
|
+
def nonblock_flag
|
|
121
|
+
File.const_defined?(:NONBLOCK) ? File::NONBLOCK : 0
|
|
122
|
+
end
|
|
123
|
+
|
|
124
|
+
# The private crash-log write/read needs both flags: without O_NOFOLLOW
|
|
125
|
+
# a planted symlink redirects the open, without O_NONBLOCK a planted
|
|
126
|
+
# FIFO blocks it. Where either is missing, the log is omitted rather
|
|
127
|
+
# than opened unsafely (#296).
|
|
128
|
+
def safe_log_io?
|
|
129
|
+
File.const_defined?(:NOFOLLOW) && File.const_defined?(:NONBLOCK)
|
|
130
|
+
end
|
|
131
|
+
|
|
132
|
+
# Where the log lands when a configured cache dir is not writable - a
|
|
133
|
+
# sandboxed child holds the project read-only, but the system temp dir is
|
|
134
|
+
# granted (#296). Only a configured log path falls back: cache_dir = nil
|
|
135
|
+
# still means no log at all. Keyed by pid so parallel children never
|
|
136
|
+
# share a file; the runner reads the same path back in the same process.
|
|
137
|
+
def tmp_log_path(pid: Process.pid)
|
|
138
|
+
File.join(Dir.tmpdir, "saboteur-#{pid}-#{LOG_NAME}")
|
|
139
|
+
end
|
|
140
|
+
|
|
141
|
+
# The message names the log; relative to the working directory when it
|
|
142
|
+
# sits inside it, so the line stays short.
|
|
143
|
+
def display_path(path)
|
|
144
|
+
relative = Pathname(path).relative_path_from(Pathname(Dir.pwd)).to_s
|
|
145
|
+
relative.start_with?("..") ? path : relative
|
|
146
|
+
rescue ArgumentError
|
|
147
|
+
path
|
|
148
|
+
end
|
|
149
|
+
|
|
150
|
+
def scrub(string)
|
|
151
|
+
string.to_s.dup.force_encoding(Encoding::UTF_8).scrub("?")
|
|
152
|
+
end
|
|
153
|
+
end
|
|
154
|
+
end
|
data/lib/saboteur/custom_map.rb
CHANGED
|
@@ -1,6 +1,8 @@
|
|
|
1
1
|
# SPDX-License-Identifier: MIT
|
|
2
2
|
# frozen_string_literal: true
|
|
3
3
|
|
|
4
|
+
require_relative "text"
|
|
5
|
+
|
|
4
6
|
module Saboteur
|
|
5
7
|
# Explicit source-file -> spec-file mapping from config/saboteur_custom_map.rb:
|
|
6
8
|
#
|
|
@@ -109,7 +111,7 @@ module Saboteur
|
|
|
109
111
|
def duplicate_keys(path)
|
|
110
112
|
seen = Hash.new(0)
|
|
111
113
|
File.readlines(path).each do |line|
|
|
112
|
-
seen[::Regexp.last_match(1)] += 1 if line =~ /^\s*"(.+?)"\s*=>/
|
|
114
|
+
seen[::Regexp.last_match(1)] += 1 if Text.utf8(line) =~ /^\s*"(.+?)"\s*=>/
|
|
113
115
|
end
|
|
114
116
|
seen.select { |_, count| count > 1 }.keys
|
|
115
117
|
end
|
data/lib/saboteur/doctor.rb
CHANGED
|
@@ -3,6 +3,8 @@
|
|
|
3
3
|
|
|
4
4
|
require "coverage"
|
|
5
5
|
require_relative "baseline_health"
|
|
6
|
+
require_relative "engine/sandbox_policy"
|
|
7
|
+
require_relative "execution/sandbox"
|
|
6
8
|
|
|
7
9
|
module Saboteur
|
|
8
10
|
# Reports the current environment, never mutations. The two unchanged suite
|
|
@@ -27,7 +29,8 @@ module Saboteur
|
|
|
27
29
|
@out.puts "Rails: #{@config['rails'] ? 'enabled' : 'disabled'}"
|
|
28
30
|
check_database_hosts
|
|
29
31
|
@out.puts "SimpleCov: #{defined?(::SimpleCov) ? 'present; start suppressed for this process' : 'not loaded'}"
|
|
30
|
-
|
|
32
|
+
verify_coverage
|
|
33
|
+
check_sandbox
|
|
31
34
|
check_suite(integration, hooks)
|
|
32
35
|
true
|
|
33
36
|
end
|
|
@@ -63,15 +66,51 @@ module Saboteur
|
|
|
63
66
|
"doctor supports the rspec integration; set integration: rspec"
|
|
64
67
|
end
|
|
65
68
|
|
|
69
|
+
# Printed once; verified twice. Boot must not wake coverage up, so the
|
|
70
|
+
# post-boot check re-reads the state without printing a second line.
|
|
66
71
|
def check_coverage
|
|
67
72
|
state = Coverage.state
|
|
68
73
|
@out.puts "Coverage: #{state}"
|
|
74
|
+
verify_coverage(state)
|
|
75
|
+
end
|
|
76
|
+
|
|
77
|
+
def verify_coverage(state = Coverage.state)
|
|
69
78
|
return if Coverage.supported?(:lines) && Coverage.supported?(:methods) && state == :suspended
|
|
70
79
|
|
|
71
80
|
raise Error, "Coverage must be suspended with line and method support before boot; use the saboteur " \
|
|
72
81
|
"executable, disable SimpleCov and remove other Coverage.start calls"
|
|
73
82
|
end
|
|
74
83
|
|
|
84
|
+
# A kernel capability probe, not verified isolation: what the kernel
|
|
85
|
+
# would let mutant children do, and what the run does with that - the
|
|
86
|
+
# engine's sandbox policy owns the refusal or degradation itself.
|
|
87
|
+
def check_sandbox
|
|
88
|
+
if @config["sandbox"] == false
|
|
89
|
+
@out.puts "Sandbox: off (c.sandbox = false)"
|
|
90
|
+
return
|
|
91
|
+
end
|
|
92
|
+
abi = sandbox_abi
|
|
93
|
+
if abi >= 3
|
|
94
|
+
@out.puts "Sandbox: kernel Landlock ABI #{abi} - capability probe only, not verified isolation"
|
|
95
|
+
elsif @config["sandbox"] == "strict"
|
|
96
|
+
@out.puts "Sandbox: #{Engine::SandboxPolicy.sandbox_gap(abi)}; " \
|
|
97
|
+
'c.sandbox = "strict" refuses to run - children never run degraded'
|
|
98
|
+
else
|
|
99
|
+
@out.puts "Sandbox: #{Engine::SandboxPolicy.sandbox_gap(abi)}; " \
|
|
100
|
+
"children would run with the hazard guard and canary only"
|
|
101
|
+
end
|
|
102
|
+
end
|
|
103
|
+
|
|
104
|
+
# The probe contract the engine's policy already keeps: an errored probe
|
|
105
|
+
# (EPERM under a seccomp policy, ...) is an actionable tool error, not a
|
|
106
|
+
# CLI internal error - doctor stays read-only.
|
|
107
|
+
def sandbox_abi
|
|
108
|
+
Execution::Sandbox.abi_version
|
|
109
|
+
rescue SystemCallError => e
|
|
110
|
+
raise Error, "sandbox is on but the Landlock probe failed (#{e.message}); a run would refuse the same way - " \
|
|
111
|
+
"set c.sandbox = false to check without the sandbox"
|
|
112
|
+
end
|
|
113
|
+
|
|
75
114
|
# Match against all included source definitions, not the current diff or
|
|
76
115
|
# selected tests. A limited run must not make a valid ignore look stale.
|
|
77
116
|
def check_ignore_subjects
|
|
@@ -3,6 +3,7 @@
|
|
|
3
3
|
|
|
4
4
|
require "json"
|
|
5
5
|
require "fileutils"
|
|
6
|
+
require_relative "../text"
|
|
6
7
|
|
|
7
8
|
module Saboteur
|
|
8
9
|
class Engine
|
|
@@ -10,16 +11,28 @@ module Saboteur
|
|
|
10
11
|
# or mutation source is used. This catches observed instability, not all
|
|
11
12
|
# possible flaky behaviour.
|
|
12
13
|
class BaselineGuard
|
|
14
|
+
# How many unchanged-suite checks actually completed and validated:
|
|
15
|
+
# the number of order comparisons a run can claim is one less.
|
|
16
|
+
attr_reader :completed_passes
|
|
17
|
+
|
|
13
18
|
EXCERPT_LINES = 10
|
|
14
19
|
EXCERPT_WIDTH = 200
|
|
15
20
|
EXCERPT_SCAN_LINES = 5000
|
|
16
21
|
|
|
17
|
-
|
|
22
|
+
# full_passes: every full-suite pass the run will make, including the
|
|
23
|
+
# coverage build when the cache is cold, so the first timed pass can
|
|
24
|
+
# price the rest (#260).
|
|
25
|
+
def initialize(config:, integration:, hooks:, supervisor:, diagnostics:, full_passes: 2, between_passes: nil,
|
|
26
|
+
timeout_cap: nil)
|
|
18
27
|
@config = config
|
|
19
28
|
@integration = integration
|
|
20
29
|
@hooks = hooks
|
|
21
30
|
@supervisor = supervisor
|
|
22
31
|
@diagnostics = diagnostics
|
|
32
|
+
@full_passes = full_passes
|
|
33
|
+
@between_passes = between_passes
|
|
34
|
+
@timeout_cap = timeout_cap
|
|
35
|
+
@completed_passes = 0
|
|
23
36
|
end
|
|
24
37
|
|
|
25
38
|
def call
|
|
@@ -27,11 +40,16 @@ module Saboteur
|
|
|
27
40
|
raise Error, "baseline has no tests. Check rspec_args and the suite filters before running saboteur again." if
|
|
28
41
|
expected.empty?
|
|
29
42
|
|
|
30
|
-
|
|
43
|
+
passes = @config["baseline_passes"]
|
|
44
|
+
@diagnostics.inform(announce(passes))
|
|
31
45
|
previous = nil
|
|
32
|
-
|
|
46
|
+
passes.times do |index|
|
|
47
|
+
break if budget_spent?(index)
|
|
48
|
+
|
|
33
49
|
number = index + 1
|
|
50
|
+
started = Process.clock_gettime(Process::CLOCK_MONOTONIC)
|
|
34
51
|
data = check(number)
|
|
52
|
+
announce_first_pass_cost(number, started)
|
|
35
53
|
unless data["passed"] && data["failed"].empty?
|
|
36
54
|
refuse(number,
|
|
37
55
|
"the suite failed: #{Array(data['failed']).join(', ')}")
|
|
@@ -41,18 +59,73 @@ module Saboteur
|
|
|
41
59
|
executed.sort == expected.sort
|
|
42
60
|
refuse(number, "test order changed between baseline checks") if previous && executed != previous
|
|
43
61
|
previous = executed
|
|
62
|
+
@completed_passes += 1
|
|
44
63
|
end
|
|
45
64
|
nil
|
|
46
65
|
end
|
|
47
66
|
|
|
48
67
|
private
|
|
49
68
|
|
|
69
|
+
# A spent --max-duration budget skips the remaining passes - including
|
|
70
|
+
# the first, when boot alone already spent it (#245).
|
|
71
|
+
def budget_spent?(_index)
|
|
72
|
+
!@between_passes.nil? && @between_passes.call
|
|
73
|
+
end
|
|
74
|
+
|
|
75
|
+
def announce(passes)
|
|
76
|
+
case passes
|
|
77
|
+
when 1
|
|
78
|
+
"checking the unchanged suite once (seed #{@config['seed']}); " \
|
|
79
|
+
"order stability between passes is not checked (baseline_passes: 1)"
|
|
80
|
+
when 2
|
|
81
|
+
"checking the unchanged suite twice (seed #{@config['seed']})"
|
|
82
|
+
else
|
|
83
|
+
"checking the unchanged suite #{passes} times (seed #{@config['seed']})"
|
|
84
|
+
end
|
|
85
|
+
end
|
|
86
|
+
|
|
87
|
+
# Once the first pass is timed, the total cost stops being a guess:
|
|
88
|
+
# the remaining full passes cost roughly the same each.
|
|
89
|
+
def announce_first_pass_cost(number, started)
|
|
90
|
+
return unless number == 1
|
|
91
|
+
|
|
92
|
+
remaining = @full_passes - number
|
|
93
|
+
return if remaining.zero?
|
|
94
|
+
|
|
95
|
+
elapsed = (Process.clock_gettime(Process::CLOCK_MONOTONIC) - started).round
|
|
96
|
+
@diagnostics.inform("baseline check #{number} took #{elapsed} s; about #{elapsed * remaining} s left for " \
|
|
97
|
+
"the remaining #{remaining} full #{remaining == 1 ? 'pass' : 'passes'} at that rate")
|
|
98
|
+
end
|
|
99
|
+
|
|
100
|
+
# An admitted pass still gets only the time left in the run budget:
|
|
101
|
+
# without the cap it would overrun a spent --max-duration by its
|
|
102
|
+
# whole coverage_timeout.
|
|
103
|
+
def phase_timeout
|
|
104
|
+
@timeout_cap ? @timeout_cap.call(@config["coverage_timeout"]) : @config["coverage_timeout"]
|
|
105
|
+
end
|
|
106
|
+
|
|
107
|
+
# A pass that hit its deadline only because --max-duration shortened
|
|
108
|
+
# the admission below the configured phase timeout: the budget ended
|
|
109
|
+
# the run, the suite is not broken (#296).
|
|
110
|
+
def budget_stopped?(result, admitted)
|
|
111
|
+
result.stopped_by == :timeout && admitted < @config["coverage_timeout"]
|
|
112
|
+
end
|
|
113
|
+
|
|
114
|
+
def refuse_budget_stop(number, admitted)
|
|
115
|
+
raise BudgetPhaseTimeout, "the --max-duration budget stopped baseline check #{number} " \
|
|
116
|
+
"(admitted #{admitted.round(1)}s of the #{@config['coverage_timeout']}s " \
|
|
117
|
+
"phase timeout)"
|
|
118
|
+
end
|
|
119
|
+
|
|
50
120
|
def check(number)
|
|
51
121
|
Execution::ForkReadiness.prepare_heap
|
|
52
122
|
heartbeat = @diagnostics.progress_heartbeat("baseline check #{number}").watch(log_path(number))
|
|
53
|
-
|
|
123
|
+
admitted = phase_timeout
|
|
124
|
+
result = @supervisor.run(timeout: admitted, on_tick: heartbeat.method(:tick),
|
|
125
|
+
label: "baseline child died during setup/run") do |writer|
|
|
54
126
|
record(writer, number)
|
|
55
127
|
end
|
|
128
|
+
refuse_budget_stop(number, admitted) if budget_stopped?(result, admitted)
|
|
56
129
|
refuse(number, "the suite stopped: #{result.stopped_by} (#{result.detail})") if result.stopped_by
|
|
57
130
|
# Before the payload checks: a child killed by a signal (the OOM
|
|
58
131
|
# killer, say) writes nothing, and the signal is the explanation. A
|
|
@@ -129,7 +202,8 @@ module Saboteur
|
|
|
129
202
|
end
|
|
130
203
|
|
|
131
204
|
def log_lines(number)
|
|
132
|
-
File.foreach(log_path(number), chomp: true).first(EXCERPT_SCAN_LINES)
|
|
205
|
+
File.foreach(log_path(number), chomp: true).first(EXCERPT_SCAN_LINES)
|
|
206
|
+
.map { |line| Text.utf8(line) }.reject { |line| line.strip.empty? }
|
|
133
207
|
rescue SystemCallError
|
|
134
208
|
[]
|
|
135
209
|
end
|
data/lib/saboteur/engine/boot.rb
CHANGED
|
@@ -83,10 +83,21 @@ module Saboteur
|
|
|
83
83
|
|
|
84
84
|
def load_hooks
|
|
85
85
|
Saboteur.current_hooks = @hooks
|
|
86
|
-
@config["hooks"].each { |file| load
|
|
86
|
+
@config["hooks"].each { |file| load hook_path(file) }
|
|
87
87
|
ensure
|
|
88
88
|
Saboteur.current_hooks = nil
|
|
89
89
|
end
|
|
90
|
+
|
|
91
|
+
# Kernel#load does not append .rb the way Kernel#require does, and
|
|
92
|
+
# doctor accepts the extensionless form (#257), so resolve it the same
|
|
93
|
+
# way here: the exact path first, then path.rb.
|
|
94
|
+
def hook_path(file)
|
|
95
|
+
path = File.expand_path(file, @root)
|
|
96
|
+
rb = "#{path}.rb"
|
|
97
|
+
return rb if !File.exist?(path) && File.extname(path).empty? && File.file?(rb)
|
|
98
|
+
|
|
99
|
+
path
|
|
100
|
+
end
|
|
90
101
|
end
|
|
91
102
|
end
|
|
92
103
|
end
|
|
@@ -2,6 +2,7 @@
|
|
|
2
2
|
# frozen_string_literal: true
|
|
3
3
|
|
|
4
4
|
require "open3"
|
|
5
|
+
require_relative "../text"
|
|
5
6
|
|
|
6
7
|
module Saboteur
|
|
7
8
|
class Engine
|
|
@@ -15,7 +16,9 @@ module Saboteur
|
|
|
15
16
|
GIT_DIR = ".git"
|
|
16
17
|
|
|
17
18
|
def initialize(root, cache_dir: ".saboteur")
|
|
18
|
-
|
|
19
|
+
# Retag losslessly: path bytes are identity here, and later
|
|
20
|
+
# File.join calls must not mix encodings.
|
|
21
|
+
@root = Text.utf8_identity(root.to_s)
|
|
19
22
|
@cache_dir = cache_dir.to_s
|
|
20
23
|
@snapshot = inventory
|
|
21
24
|
end
|
|
@@ -29,16 +32,40 @@ module Saboteur
|
|
|
29
32
|
|
|
30
33
|
of = " of #{subject.id}" if subject
|
|
31
34
|
culprit = digest ? " The most recent mutant, #{digest}#{of}, is the likely cause." : ""
|
|
32
|
-
raise Saboteur::CanaryTripped,
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
35
|
+
raise Saboteur::CanaryTripped, message(missing, culprit)
|
|
36
|
+
end
|
|
37
|
+
|
|
38
|
+
# The cheap per-mutant probe (#235): stats the snapshot paths only -
|
|
39
|
+
# no git, no glob - so it can run after every mutant.
|
|
40
|
+
def probe
|
|
41
|
+
@snapshot.reject { |rel| present?(rel) }
|
|
42
|
+
end
|
|
43
|
+
|
|
44
|
+
# Like check!, but for the per-mutant probe: the mutant whose run just
|
|
45
|
+
# ended is named, never the subject's most recent mutant.
|
|
46
|
+
def probe!(subject: nil, digest: nil)
|
|
47
|
+
missing = probe
|
|
48
|
+
return if missing.empty?
|
|
49
|
+
|
|
50
|
+
of = " of #{subject.id}" if subject
|
|
51
|
+
culprit = digest ? " The mutant that just ran, #{digest}#{of}, is the likely cause." : ""
|
|
52
|
+
raise Saboteur::CanaryTripped.new(message(missing, culprit), culprit_digest: digest)
|
|
38
53
|
end
|
|
39
54
|
|
|
40
55
|
private
|
|
41
56
|
|
|
57
|
+
def present?(rel)
|
|
58
|
+
File.exist?(File.join(@root, rel)) || File.symlink?(File.join(@root, rel))
|
|
59
|
+
end
|
|
60
|
+
|
|
61
|
+
def message(missing, culprit)
|
|
62
|
+
"canary: #{missing.size} watched file(s) vanished from the working tree while mutants ran " \
|
|
63
|
+
"(first: #{Text.utf8(missing.first)}).#{culprit} The canary watches tracked files and .git presence " \
|
|
64
|
+
"and only detects the loss after the fact; it cannot restore anything. Recover from your " \
|
|
65
|
+
"own backup, a fresh checkout, or a reclone - a deleted .git or uncommitted work is gone " \
|
|
66
|
+
"unless you have another copy."
|
|
67
|
+
end
|
|
68
|
+
|
|
42
69
|
def inventory
|
|
43
70
|
files = git? ? git_tracked_files : walk_files
|
|
44
71
|
files << GIT_DIR if File.exist?(File.join(@root, GIT_DIR))
|
|
@@ -54,15 +81,19 @@ module Saboteur
|
|
|
54
81
|
return walk_files unless status.success?
|
|
55
82
|
|
|
56
83
|
# ls-files reads the index, which still names a deleted file; keep
|
|
57
|
-
# only paths present in the worktree so a deletion shows up.
|
|
58
|
-
|
|
84
|
+
# only paths present in the worktree so a deletion shows up. Split
|
|
85
|
+
# the raw bytes first and retag each path losslessly: scrubbing a
|
|
86
|
+
# raw-byte name would make the File.exist? probe fail and drop the
|
|
87
|
+
# file from the watch, so its deletion would go unseen.
|
|
88
|
+
out.b.split("\0".b).map { |rel| Text.utf8_identity(rel) }
|
|
89
|
+
.reject { |rel| cache_path?(rel) }
|
|
59
90
|
.select { |rel| File.exist?(File.join(@root, rel)) || File.symlink?(File.join(@root, rel)) }
|
|
60
91
|
end
|
|
61
92
|
|
|
62
93
|
def walk_files
|
|
63
94
|
Dir.glob(File.join(@root, "**", "*"), File::FNM_DOTMATCH)
|
|
64
95
|
.select { |path| File.file?(path) }
|
|
65
|
-
.map { |path| path.delete_prefix("#{@root}/") }
|
|
96
|
+
.map { |path| Text.utf8_identity(path).delete_prefix("#{@root}/") }
|
|
66
97
|
.reject { |rel| rel.start_with?("#{GIT_DIR}/") || cache_path?(rel) }
|
|
67
98
|
end
|
|
68
99
|
|
|
@@ -6,7 +6,7 @@ module Saboteur
|
|
|
6
6
|
# Selects a bounded list for each covered subject before workers fork.
|
|
7
7
|
# Subject skips are not mutants and do not enter the mutation score.
|
|
8
8
|
class CandidateSelection
|
|
9
|
-
attr_reader :skipped_subjects, :selected
|
|
9
|
+
attr_reader :skipped_subjects, :selected, :unscoped_digests
|
|
10
10
|
|
|
11
11
|
def initialize(candidates:, map:, limit:, diagnostics:)
|
|
12
12
|
@candidates = candidates
|
|
@@ -15,6 +15,7 @@ module Saboteur
|
|
|
15
15
|
@diagnostics = diagnostics
|
|
16
16
|
@skipped_subjects = {}
|
|
17
17
|
@selected = {}
|
|
18
|
+
@unscoped_digests = {}
|
|
18
19
|
end
|
|
19
20
|
|
|
20
21
|
def for(subject, scope: nil)
|
|
@@ -25,6 +26,13 @@ module Saboteur
|
|
|
25
26
|
end
|
|
26
27
|
|
|
27
28
|
list = @candidates.for(subject)
|
|
29
|
+
# The full pre-scope digests let IgnoreList#apply tell "mutation gone"
|
|
30
|
+
# from "mutation scoped out of this run" (#284).
|
|
31
|
+
if scope
|
|
32
|
+
@unscoped_digests[subject.id] = list.map do |mutation, _|
|
|
33
|
+
mutation.digest(subject_start: subject.start_byte, original_source: subject.file_source)
|
|
34
|
+
end
|
|
35
|
+
end
|
|
28
36
|
list = scope.restrict(subject, list) if scope
|
|
29
37
|
omitted = list.size - @limit
|
|
30
38
|
if omitted.positive?
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
# SPDX-License-Identifier: MIT
|
|
2
|
+
# frozen_string_literal: true
|
|
3
|
+
|
|
4
|
+
module Saboteur
|
|
5
|
+
class Engine
|
|
6
|
+
# Owns the host process's Coverage state across a run: suspends after
|
|
7
|
+
# boot, and at the end resumes only its own suspension. A CLI process
|
|
8
|
+
# exits right after, so resume matters to hosts that embed the engine -
|
|
9
|
+
# a spec, or the dogfood's own coverage-map build (#254).
|
|
10
|
+
class CoverageGuard
|
|
11
|
+
def initialize(diagnostics)
|
|
12
|
+
@diagnostics = diagnostics
|
|
13
|
+
end
|
|
14
|
+
|
|
15
|
+
def suspend
|
|
16
|
+
@was_running = Coverage.state == :running
|
|
17
|
+
Coverage.suspend if @was_running
|
|
18
|
+
end
|
|
19
|
+
|
|
20
|
+
# Best-effort: a host that disturbed Coverage mid-run gets a warning,
|
|
21
|
+
# never a masked run outcome.
|
|
22
|
+
def resume
|
|
23
|
+
Coverage.resume if @was_running && Coverage.state == :suspended
|
|
24
|
+
rescue StandardError => e
|
|
25
|
+
@diagnostics.warn("coverage resume failed: #{e.class}: #{e.message}")
|
|
26
|
+
end
|
|
27
|
+
end
|
|
28
|
+
end
|
|
29
|
+
end
|
|
@@ -15,19 +15,29 @@ module Saboteur
|
|
|
15
15
|
LOCKFILES = %w[Gemfile.lock gems.locked].freeze
|
|
16
16
|
LOG = "coverage-build.log"
|
|
17
17
|
|
|
18
|
-
def initialize(config:, integration:, hooks:, supervisor:, diagnostics:, load_lines: {})
|
|
18
|
+
def initialize(config:, integration:, hooks:, supervisor:, diagnostics:, load_lines: {}, timeout_cap: nil)
|
|
19
19
|
@config = config
|
|
20
20
|
@integration = integration
|
|
21
21
|
@hooks = hooks
|
|
22
22
|
@supervisor = supervisor
|
|
23
23
|
@diagnostics = diagnostics
|
|
24
|
+
@timeout_cap = timeout_cap
|
|
24
25
|
@load_lines = load_lines
|
|
25
26
|
@root = File.realpath(config.root)
|
|
26
27
|
end
|
|
27
28
|
|
|
29
|
+
# True when fetch would reuse the cached map instead of running the
|
|
30
|
+
# suite pass that builds it: lets the run announce the real number of
|
|
31
|
+
# full passes up front (#260). Loads the map with the same validation
|
|
32
|
+
# fetch uses and retains it, so a corrupt or truncated cache announces
|
|
33
|
+
# a build pass, not a reuse it cannot keep.
|
|
34
|
+
def reusable?(files)
|
|
35
|
+
!cached_map(files).nil?
|
|
36
|
+
end
|
|
37
|
+
|
|
28
38
|
def fetch(files)
|
|
29
|
-
path =
|
|
30
|
-
cached =
|
|
39
|
+
path = cache_path(files)
|
|
40
|
+
cached = cached_map(files)
|
|
31
41
|
return refuse_empty_map(cached, files) if cached
|
|
32
42
|
|
|
33
43
|
map = refuse_empty_map(build(files), files)
|
|
@@ -39,6 +49,16 @@ module Saboteur
|
|
|
39
49
|
|
|
40
50
|
private
|
|
41
51
|
|
|
52
|
+
def cache_path(files)
|
|
53
|
+
File.join(@config.path("cache_dir"), "coverage-#{cache_key(files)}.json")
|
|
54
|
+
end
|
|
55
|
+
|
|
56
|
+
def cached_map(files)
|
|
57
|
+
return nil unless @config["cache"]
|
|
58
|
+
|
|
59
|
+
@cached_map ||= Selection::CoverageMap.load(cache_path(files))
|
|
60
|
+
end
|
|
61
|
+
|
|
42
62
|
def cache_key(files)
|
|
43
63
|
inputs = JSON.generate(CACHE_INPUTS.to_h { |key| [key, @config[key]] })
|
|
44
64
|
all = files + @integration.test_files + SuiteFiles.new(@config).files + lockfiles
|
|
@@ -49,6 +69,12 @@ module Saboteur
|
|
|
49
69
|
LOCKFILES.map { |file| File.join(@root, file) }.select { |file| File.file?(file) }
|
|
50
70
|
end
|
|
51
71
|
|
|
72
|
+
# An admitted build still gets only the time left in the run budget
|
|
73
|
+
# (see BaselineGuard#phase_timeout).
|
|
74
|
+
def build_timeout
|
|
75
|
+
@timeout_cap ? @timeout_cap.call(@config["coverage_timeout"]) : @config["coverage_timeout"]
|
|
76
|
+
end
|
|
77
|
+
|
|
52
78
|
def build(files)
|
|
53
79
|
unless Coverage.respond_to?(:state) && Coverage.state == :suspended
|
|
54
80
|
raise Error, "Coverage is #{Coverage.state}; saboteur must set it up before the app loads. " \
|
|
@@ -61,13 +87,16 @@ module Saboteur
|
|
|
61
87
|
# reduce later copying of shared pages.
|
|
62
88
|
Execution::ForkReadiness.prepare_heap
|
|
63
89
|
heartbeat = @diagnostics.progress_heartbeat("building coverage map").watch(log_path)
|
|
64
|
-
|
|
90
|
+
admitted = build_timeout
|
|
91
|
+
result = @supervisor.run(timeout: admitted, on_tick: heartbeat.method(:tick),
|
|
92
|
+
label: "coverage child died during the map build") do |writer|
|
|
65
93
|
record(writer, files)
|
|
66
94
|
end
|
|
67
|
-
map_from(result)
|
|
95
|
+
map_from(result, admitted)
|
|
68
96
|
end
|
|
69
97
|
|
|
70
|
-
def map_from(result)
|
|
98
|
+
def map_from(result, admitted = @config["coverage_timeout"])
|
|
99
|
+
raise_budget_stop(admitted) if budget_stopped?(result, admitted)
|
|
71
100
|
raise Error, "coverage map build stopped: #{result.detail}" if result.stopped_by
|
|
72
101
|
raise Error, "coverage map build failed: #{died(result.exit_status)}" if result.payload.empty?
|
|
73
102
|
|
|
@@ -77,6 +106,19 @@ module Saboteur
|
|
|
77
106
|
with_parent_load_lines(Selection::CoverageMap.from_json(result.payload))
|
|
78
107
|
end
|
|
79
108
|
|
|
109
|
+
# A build that hit its deadline only because --max-duration shortened
|
|
110
|
+
# the admission below the configured phase timeout: the budget ended
|
|
111
|
+
# the run, the suite is not broken (see BaselineGuard#budget_stopped?).
|
|
112
|
+
def budget_stopped?(result, admitted)
|
|
113
|
+
result.stopped_by == :timeout && admitted < @config["coverage_timeout"]
|
|
114
|
+
end
|
|
115
|
+
|
|
116
|
+
def raise_budget_stop(admitted)
|
|
117
|
+
raise BudgetPhaseTimeout, "the --max-duration budget stopped the coverage map build " \
|
|
118
|
+
"(admitted #{admitted.round(1)}s of the #{@config['coverage_timeout']}s " \
|
|
119
|
+
"phase timeout)"
|
|
120
|
+
end
|
|
121
|
+
|
|
80
122
|
# The parent's load lines merge into the freshly built map: the forked
|
|
81
123
|
# build child cannot inherit the parent's Coverage counters.
|
|
82
124
|
def with_parent_load_lines(map)
|