saboteur 0.5.0 → 0.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +399 -0
- data/README.md +202 -42
- data/exe/saboteur +34 -7
- data/lib/saboteur/baseline/comparison.rb +1 -0
- data/lib/saboteur/baseline/evidence.rb +11 -1
- data/lib/saboteur/baseline/run_evidence.rb +1 -1
- data/lib/saboteur/baseline/workflow.rb +2 -1
- data/lib/saboteur/baseline_health.rb +40 -6
- data/lib/saboteur/cli/exit_status.rb +49 -4
- data/lib/saboteur/cli/help_options.rb +16 -0
- data/lib/saboteur/cli/presets.rb +36 -0
- data/lib/saboteur/cli/run_budget_options.rb +37 -0
- data/lib/saboteur/cli/run_reporting.rb +75 -6
- data/lib/saboteur/cli.rb +80 -48
- data/lib/saboteur/config.rb +88 -13
- data/lib/saboteur/crash_report.rb +154 -0
- data/lib/saboteur/custom_map.rb +3 -1
- data/lib/saboteur/doctor.rb +64 -2
- data/lib/saboteur/engine/baseline_guard.rb +79 -5
- data/lib/saboteur/engine/boot.rb +12 -1
- data/lib/saboteur/engine/canary.rb +41 -10
- data/lib/saboteur/engine/candidate_selection.rb +9 -1
- data/lib/saboteur/engine/candidates.rb +4 -1
- data/lib/saboteur/engine/coverage_guard.rb +29 -0
- data/lib/saboteur/engine/coverage_store.rb +48 -6
- data/lib/saboteur/engine/diagnostics.rb +23 -2
- data/lib/saboteur/engine/diff_scope.rb +24 -5
- data/lib/saboteur/engine/distribution.rb +124 -8
- data/lib/saboteur/engine/early_stop_recording.rb +46 -0
- data/lib/saboteur/engine/result_cache.rb +2 -1
- data/lib/saboteur/engine/run_budget.rb +63 -0
- data/lib/saboteur/engine/run_fingerprint.rb +5 -1
- data/lib/saboteur/engine/sandbox_policy.rb +18 -11
- data/lib/saboteur/engine/scheduler.rb +114 -18
- data/lib/saboteur/engine/subject_selection.rb +84 -5
- data/lib/saboteur/engine/suite_files.rb +3 -1
- data/lib/saboteur/engine/test_guess.rb +183 -23
- data/lib/saboteur/engine/thread_warnings.rb +53 -0
- data/lib/saboteur/engine/unscoped_suite_warning.rb +41 -0
- data/lib/saboteur/engine/verdict_check.rb +54 -18
- data/lib/saboteur/engine.rb +175 -96
- data/lib/saboteur/execution/protocol.rb +2 -1
- data/lib/saboteur/execution/runner.rb +179 -17
- data/lib/saboteur/execution/sandbox.rb +6 -1
- data/lib/saboteur/execution/supervisor.rb +18 -3
- data/lib/saboteur/execution/worker_pool.rb +87 -7
- data/lib/saboteur/frameworks/database_hosts.rb +30 -0
- data/lib/saboteur/frameworks/rails.rb +8 -10
- data/lib/saboteur/help.rb +55 -10
- data/lib/saboteur/ignore_list.rb +11 -1
- data/lib/saboteur/install.rb +4 -3
- data/lib/saboteur/integration/base.rb +7 -2
- data/lib/saboteur/integration/rspec.rb +40 -4
- data/lib/saboteur/mutation_source/builtin.rb +22 -13
- data/lib/saboteur/mutation_source/hazard_guard.rb +21 -3
- data/lib/saboteur/mutation_source/interpolation_removal.rb +41 -0
- data/lib/saboteur/mutation_source/method_swap.rb +45 -0
- data/lib/saboteur/mutation_source/mutation.rb +5 -3
- data/lib/saboteur/mutation_source/operand_replacement.rb +54 -0
- data/lib/saboteur/mutation_source/operator_examples.rb +38 -0
- data/lib/saboteur/mutation_source/operator_registry.rb +36 -8
- data/lib/saboteur/mutation_source/operator_visit.rb +39 -0
- data/lib/saboteur/preview.rb +12 -2
- data/lib/saboteur/reporter/agent_commands.rb +49 -16
- data/lib/saboteur/reporter/agent_incomplete.rb +27 -2
- data/lib/saboteur/reporter/agent_summary.rb +1 -1
- data/lib/saboteur/reporter/history_policy.rb +35 -0
- data/lib/saboteur/reporter/history_record.rb +4 -5
- data/lib/saboteur/reporter/next_steps.rb +14 -4
- data/lib/saboteur/reporter/score_history.rb +19 -3
- data/lib/saboteur/reporter/stryker.rb +19 -8
- data/lib/saboteur/reporter/survivor_hints.rb +3 -0
- data/lib/saboteur/reporter/terminal.rb +43 -10
- data/lib/saboteur/result.rb +25 -7
- data/lib/saboteur/selection/coverage_map.rb +2 -1
- data/lib/saboteur/selection/git_diff.rb +15 -5
- data/lib/saboteur/selection/untracked_changes.rb +126 -0
- data/lib/saboteur/subject.rb +11 -0
- data/lib/saboteur/text.rb +42 -0
- data/lib/saboteur/version.rb +1 -1
- data/lib/saboteur.rb +16 -1
- metadata +20 -2
data/lib/saboteur/config.rb
CHANGED
|
@@ -1,6 +1,8 @@
|
|
|
1
1
|
# SPDX-License-Identifier: MIT
|
|
2
2
|
# frozen_string_literal: true
|
|
3
3
|
|
|
4
|
+
require_relative "text"
|
|
5
|
+
|
|
4
6
|
require_relative "mutation_source/builtin"
|
|
5
7
|
require_relative "mutation_source/operator_registry"
|
|
6
8
|
require_relative "mutation_source/hazard_guard"
|
|
@@ -39,8 +41,11 @@ module Saboteur
|
|
|
39
41
|
"custom_map" => "config/saboteur_custom_map.rb",
|
|
40
42
|
"sources" => ["builtin"],
|
|
41
43
|
"operators" => [],
|
|
44
|
+
"exclude_operators" => [],
|
|
42
45
|
"hooks" => [],
|
|
43
46
|
"rails" => false,
|
|
47
|
+
# Hosts the user asserts are disposable test databases (issue #258).
|
|
48
|
+
"allow_db_hosts" => [],
|
|
44
49
|
"reeval" => "nesting",
|
|
45
50
|
"cache_dir" => ".saboteur",
|
|
46
51
|
"json_report" => ".saboteur/report.json",
|
|
@@ -54,8 +59,13 @@ module Saboteur
|
|
|
54
59
|
"allow_focus" => false,
|
|
55
60
|
"fail_fast" => false,
|
|
56
61
|
"guess_tests" => false,
|
|
62
|
+
"all" => false,
|
|
63
|
+
"test_selection" => "auto",
|
|
57
64
|
"matrix" => false,
|
|
58
65
|
"seed" => 1,
|
|
66
|
+
"baseline_passes" => 2,
|
|
67
|
+
"max_duration" => nil,
|
|
68
|
+
"max_duration_ok" => false,
|
|
59
69
|
"tests" => [],
|
|
60
70
|
"fingerprint_extra" => [],
|
|
61
71
|
"threshold" => nil,
|
|
@@ -63,12 +73,13 @@ module Saboteur
|
|
|
63
73
|
"max_errors" => nil
|
|
64
74
|
}.freeze
|
|
65
75
|
|
|
66
|
-
INTEGER_KEYS = %w[jobs max_memory_mb max_processes max_mutants_per_method seed].freeze
|
|
76
|
+
INTEGER_KEYS = %w[jobs max_memory_mb max_processes max_mutants_per_method seed baseline_passes].freeze
|
|
67
77
|
FLOAT_KEYS = %w[timeout_factor timeout_min baseline_timeout coverage_timeout].freeze
|
|
78
|
+
NILABLE_FLOAT_KEYS = %w[threshold max_duration].freeze
|
|
68
79
|
THRESHOLD_RANGE = (0.0..100.0)
|
|
69
80
|
|
|
70
81
|
LIST_KEYS = %w[require include exclude arid hazard sources hooks rspec_args subjects tests
|
|
71
|
-
fingerprint_extra sandbox_writable operators whole_file].freeze
|
|
82
|
+
fingerprint_extra sandbox_writable operators exclude_operators whole_file allow_db_hosts].freeze
|
|
72
83
|
|
|
73
84
|
attr_reader :root
|
|
74
85
|
|
|
@@ -93,11 +104,19 @@ module Saboteur
|
|
|
93
104
|
hash.to_h { |key, value| [key.to_s, value] }.compact
|
|
94
105
|
end
|
|
95
106
|
|
|
96
|
-
|
|
97
|
-
|
|
107
|
+
# Deprecations found while loading: printed by the CLI, never fatal.
|
|
108
|
+
attr_reader :deprecation_warnings
|
|
109
|
+
|
|
110
|
+
def initialize(values = {}, root: Dir.pwd, explicit: nil)
|
|
111
|
+
# Retag the root losslessly: under a non-UTF-8 locale it arrives
|
|
112
|
+
# locale-tagged, and every glob, join and expand below would mix
|
|
113
|
+
# encodings with the UTF-8 patterns from the config file (#231).
|
|
114
|
+
@root = Text.utf8_identity(File.expand_path(root))
|
|
98
115
|
unknown = values.keys - DEFAULTS.keys
|
|
99
116
|
raise ConfigError, "unknown config key(s): #{unknown.join(', ')}" if unknown.any?
|
|
100
117
|
|
|
118
|
+
@deprecation_warnings = deprecations(values)
|
|
119
|
+
@explicit_keys = (explicit || values.keys).map(&:to_s).freeze
|
|
101
120
|
@values = DEFAULTS.merge(values)
|
|
102
121
|
coerce!
|
|
103
122
|
validate!
|
|
@@ -112,6 +131,26 @@ module Saboteur
|
|
|
112
131
|
@values.fetch(key.to_s, default)
|
|
113
132
|
end
|
|
114
133
|
|
|
134
|
+
# Keys the user set (config file or CLI flag) rather than relying on
|
|
135
|
+
# the default; carried through Engine's config clone so provenance
|
|
136
|
+
# survives (#296).
|
|
137
|
+
attr_reader :explicit_keys
|
|
138
|
+
|
|
139
|
+
# A copy with some values replaced or dropped, keeping which keys the
|
|
140
|
+
# user set explicitly: the replacements count as explicit, the dropped
|
|
141
|
+
# keys lose it (#296).
|
|
142
|
+
def with(values, except: [])
|
|
143
|
+
merged = to_h.except(*except).merge(values)
|
|
144
|
+
Config.new(merged, root: root, explicit: (explicit_keys - except.map(&:to_s)) | values.keys.map(&:to_s))
|
|
145
|
+
end
|
|
146
|
+
|
|
147
|
+
# Whether the user set a key (config file or CLI flag) rather than
|
|
148
|
+
# relying on the default. An explicitly requested report artifact is
|
|
149
|
+
# completion evidence, so its write failure is a hard error (#296).
|
|
150
|
+
def explicit?(key)
|
|
151
|
+
@explicit_keys.include?(key.to_s)
|
|
152
|
+
end
|
|
153
|
+
|
|
115
154
|
def to_h
|
|
116
155
|
@values.dup
|
|
117
156
|
end
|
|
@@ -121,7 +160,8 @@ module Saboteur
|
|
|
121
160
|
matched = self["include"].flat_map { |pattern| Dir.glob(pattern, base: root) }
|
|
122
161
|
excluded = self["exclude"].flat_map { |pattern| Dir.glob(pattern, base: root) }
|
|
123
162
|
(matched - excluded).select { |file| file.end_with?(".rb") }
|
|
124
|
-
.map { |file| File.realpath(File.expand_path(file, root)) }
|
|
163
|
+
.map { |file| File.realpath(File.expand_path(Text.utf8_identity(file), root)) }
|
|
164
|
+
.uniq.sort
|
|
125
165
|
end
|
|
126
166
|
|
|
127
167
|
# Patterns that match no file. A pattern goes stale when files move;
|
|
@@ -148,11 +188,23 @@ module Saboteur
|
|
|
148
188
|
|
|
149
189
|
private
|
|
150
190
|
|
|
191
|
+
# An explicit guess_tests = false (config file or --no-guess-tests) used
|
|
192
|
+
# to keep the whole suite; inference is the default now, so the opt-out
|
|
193
|
+
# is --all / c.test_selection = :all. guess_tests = true is the default
|
|
194
|
+
# behavior spelled out, so it stays silent.
|
|
195
|
+
def deprecations(values)
|
|
196
|
+
return [] unless values["guess_tests"] == false
|
|
197
|
+
|
|
198
|
+
["guess_tests = false is deprecated: inferred specs are the default and it no longer restores the " \
|
|
199
|
+
"whole suite; use --all or c.test_selection = :all to run every spec"]
|
|
200
|
+
end
|
|
201
|
+
|
|
151
202
|
def coerce!
|
|
152
203
|
INTEGER_KEYS.each { |key| @values[key] = Integer(@values[key]) }
|
|
153
204
|
FLOAT_KEYS.each { |key| @values[key] = Float(@values[key]) }
|
|
154
205
|
LIST_KEYS.each { |key| @values[key] = Array(@values[key]).map(&:to_s) }
|
|
155
|
-
@values[
|
|
206
|
+
NILABLE_FLOAT_KEYS.each { |key| @values[key] = Float(@values[key]) unless @values[key].nil? }
|
|
207
|
+
@values["test_selection"] = @values["test_selection"].to_s
|
|
156
208
|
rescue ArgumentError, TypeError => e
|
|
157
209
|
raise ConfigError, "invalid config value: #{e.message}"
|
|
158
210
|
end
|
|
@@ -161,6 +213,13 @@ module Saboteur
|
|
|
161
213
|
raise ConfigError, "max_mutants_per_method must be > 0" unless self["max_mutants_per_method"].positive?
|
|
162
214
|
end
|
|
163
215
|
|
|
216
|
+
# The unchanged-suite checks: how many, and the whole run's clock (#55, #245).
|
|
217
|
+
def validate_baseline!
|
|
218
|
+
raise ConfigError, "seed must be >= 0" if self["seed"].negative?
|
|
219
|
+
raise ConfigError, "baseline_passes must be >= 1" if self["baseline_passes"] < 1
|
|
220
|
+
raise ConfigError, "max_duration must be > 0" if self["max_duration"] && !self["max_duration"].positive?
|
|
221
|
+
end
|
|
222
|
+
|
|
164
223
|
def validate_report_paths!
|
|
165
224
|
json = path("json_report")
|
|
166
225
|
if json && json == path("html_report")
|
|
@@ -176,7 +235,7 @@ module Saboteur
|
|
|
176
235
|
def validate_modes!
|
|
177
236
|
validate_fail_fast!
|
|
178
237
|
validate_matrix!
|
|
179
|
-
|
|
238
|
+
validate_test_selection!
|
|
180
239
|
validate_gates!
|
|
181
240
|
end
|
|
182
241
|
|
|
@@ -205,11 +264,10 @@ module Saboteur
|
|
|
205
264
|
raise ConfigError, "threshold must be between 0 and 100"
|
|
206
265
|
end
|
|
207
266
|
|
|
208
|
-
def
|
|
209
|
-
return
|
|
267
|
+
def validate_test_selection!
|
|
268
|
+
return if %w[auto all].include?(self["test_selection"])
|
|
210
269
|
|
|
211
|
-
raise ConfigError, "
|
|
212
|
-
raise ConfigError, "guess_tests cannot be combined with tests" if self["tests"].any?
|
|
270
|
+
raise ConfigError, "test_selection must be one of: auto, all"
|
|
213
271
|
end
|
|
214
272
|
|
|
215
273
|
def validate_reeval!
|
|
@@ -226,7 +284,7 @@ module Saboteur
|
|
|
226
284
|
validate_mutant_limit!
|
|
227
285
|
validate_threshold!
|
|
228
286
|
validate_modes!
|
|
229
|
-
|
|
287
|
+
validate_baseline!
|
|
230
288
|
raise ConfigError, "max_memory_mb must be > 0" unless self["max_memory_mb"].positive?
|
|
231
289
|
raise ConfigError, "max_processes must be > 0" unless self["max_processes"].positive?
|
|
232
290
|
raise ConfigError, "timeout_factor must be > 0" unless self["timeout_factor"].positive?
|
|
@@ -236,8 +294,25 @@ module Saboteur
|
|
|
236
294
|
validate_operator_config!
|
|
237
295
|
end
|
|
238
296
|
|
|
297
|
+
public
|
|
298
|
+
|
|
299
|
+
def operator_policy
|
|
300
|
+
MutationSource::OperatorRegistry.policy(self["operators"], self["exclude_operators"])
|
|
301
|
+
end
|
|
302
|
+
|
|
303
|
+
# Versioned policy travels with every identity, not with mutation digests.
|
|
304
|
+
def selection_settings
|
|
305
|
+
to_h.merge("operator_policy" => operator_policy)
|
|
306
|
+
end
|
|
307
|
+
|
|
308
|
+
private
|
|
309
|
+
|
|
239
310
|
def validate_operator_config!
|
|
240
|
-
MutationSource::OperatorRegistry.
|
|
311
|
+
effective = MutationSource::OperatorRegistry.select(self["operators"], self["exclude_operators"])
|
|
312
|
+
if effective.empty?
|
|
313
|
+
raise ConfigError,
|
|
314
|
+
"operator selection is empty: enable at least one operator or clear exclusions"
|
|
315
|
+
end
|
|
241
316
|
|
|
242
317
|
self["arid"].each { |pattern| Regexp.new(pattern) }
|
|
243
318
|
rescue RegexpError => e
|
|
@@ -0,0 +1,154 @@
|
|
|
1
|
+
# SPDX-License-Identifier: MIT
|
|
2
|
+
# frozen_string_literal: true
|
|
3
|
+
|
|
4
|
+
require "fileutils"
|
|
5
|
+
require "json"
|
|
6
|
+
require "tmpdir"
|
|
7
|
+
require "pathname"
|
|
8
|
+
|
|
9
|
+
module Saboteur
|
|
10
|
+
# Last-resort crash reporting, shared by the CLI's StandardError rescue
|
|
11
|
+
# and the exe-level wrapper: one bounded line on stderr, the full
|
|
12
|
+
# backtrace in a log file the line names, the backtrace on stderr with
|
|
13
|
+
# --verbose, and the same failure as agent JSON when an agent listens.
|
|
14
|
+
# Always returns EXIT_TOOL.
|
|
15
|
+
module CrashReport
|
|
16
|
+
ISSUES_URL = "https://github.com/vamsipavanmahesh/saboteur/issues"
|
|
17
|
+
|
|
18
|
+
# Same value as CLI::EXIT_TOOL, kept local so the reporter still works
|
|
19
|
+
# when the CLI itself never finished loading.
|
|
20
|
+
EXIT_TOOL = 3
|
|
21
|
+
|
|
22
|
+
# The crash log's file name under the run's cache dir; the runner reads
|
|
23
|
+
# it back to explain a nested run's crash in the failure detail (#296).
|
|
24
|
+
LOG_NAME = "internal-error.log"
|
|
25
|
+
|
|
26
|
+
module_function
|
|
27
|
+
|
|
28
|
+
def call(error, verbose:, agent:, err:, out:, log_path: nil)
|
|
29
|
+
summary = "internal error (#{error.class}: #{one_line(error.message)})"
|
|
30
|
+
line = "saboteur: #{summary} - please report it: #{ISSUES_URL}"
|
|
31
|
+
written = write_log(log_path, error)
|
|
32
|
+
line = "#{line}; full backtrace: #{display_path(written)}" if written
|
|
33
|
+
err.puts(line)
|
|
34
|
+
err.puts(full_text(error)) if verbose
|
|
35
|
+
out.print(agent_json(summary)) if agent
|
|
36
|
+
EXIT_TOOL
|
|
37
|
+
end
|
|
38
|
+
|
|
39
|
+
# A crash message can span lines or carry bytes that are invalid in
|
|
40
|
+
# UTF-8 (a LANG=C failure, say); the one-line diagnostic must survive
|
|
41
|
+
# both, and stay one bounded line.
|
|
42
|
+
def one_line(message)
|
|
43
|
+
text = scrub(message).gsub(/\s+/, " ").strip
|
|
44
|
+
text.length > 160 ? "#{text[0...160]}..." : text
|
|
45
|
+
end
|
|
46
|
+
|
|
47
|
+
# Class, full message and backtrace, scrubbed so locale-invalid bytes
|
|
48
|
+
# cannot corrupt the log or the terminal.
|
|
49
|
+
def full_text(error)
|
|
50
|
+
(["#{error.class}: #{scrub(error.message)}"] + Array(error.backtrace).map { |frame| scrub(frame) }).join("\n")
|
|
51
|
+
end
|
|
52
|
+
|
|
53
|
+
# Writes the log next to the other run state; returns the written path,
|
|
54
|
+
# or nil when the log cannot be written - the crash itself must still
|
|
55
|
+
# be reported, so an unwritable directory is not a second failure.
|
|
56
|
+
# Reporter::Agent.failure's payload, reproduced minimally for crashes
|
|
57
|
+
# before the CLI's reporter files load: same schema and pretty rendering;
|
|
58
|
+
# the summary is already normalized, so no message cap is needed here.
|
|
59
|
+
def agent_json(summary)
|
|
60
|
+
payload =
|
|
61
|
+
if defined?(Reporter::Agent)
|
|
62
|
+
Reporter::Agent.failure(summary, EXIT_TOOL)
|
|
63
|
+
else
|
|
64
|
+
{ version: 1, status: "error", verdict: "tool_failure", exitCode: EXIT_TOOL,
|
|
65
|
+
message: summary, next: { help: "saboteur help run" } }
|
|
66
|
+
end
|
|
67
|
+
"#{JSON.pretty_generate(payload)}\n"
|
|
68
|
+
end
|
|
69
|
+
|
|
70
|
+
def write_log(log_path, error)
|
|
71
|
+
return nil unless log_path
|
|
72
|
+
|
|
73
|
+
# The primary path is user-configured but still predictable: without
|
|
74
|
+
# O_NOFOLLOW a planted symlink would redirect the truncate/chmod, and
|
|
75
|
+
# without O_NONBLOCK a planted FIFO would block the open. Where the
|
|
76
|
+
# platform lacks either, only the EXCL-created temp fallback is safe
|
|
77
|
+
# (#296).
|
|
78
|
+
if safe_log_io?
|
|
79
|
+
begin
|
|
80
|
+
FileUtils.mkdir_p(File.dirname(log_path))
|
|
81
|
+
# Private like the temp fallback: the runner only reads logs with
|
|
82
|
+
# no group/other bits, from the same no-follow fd (#296).
|
|
83
|
+
flags = File::CREAT | File::WRONLY | File::TRUNC | nofollow_flag | nonblock_flag
|
|
84
|
+
File.open(log_path, flags, 0o600) do |file|
|
|
85
|
+
file.chmod(0o600)
|
|
86
|
+
file.write("#{full_text(error)}\n")
|
|
87
|
+
end
|
|
88
|
+
return log_path
|
|
89
|
+
rescue SystemCallError, IOError
|
|
90
|
+
# Fall through to the temp fallback.
|
|
91
|
+
end
|
|
92
|
+
end
|
|
93
|
+
write_tmp_log(error)
|
|
94
|
+
end
|
|
95
|
+
|
|
96
|
+
# The fallback is computed lazily, only once the primary write failed:
|
|
97
|
+
# an eager tmp_log_path call could itself raise (Dir.tmpdir finds no
|
|
98
|
+
# usable dir) and mask the crash it was meant to log (#296).
|
|
99
|
+
#
|
|
100
|
+
# The fallback path is predictable and lives in a shared directory, so
|
|
101
|
+
# it is created privately: CREAT|EXCL|WRONLY|NOFOLLOW never follows or
|
|
102
|
+
# overwrites a pre-existing file or symlink, and the mode pins the log
|
|
103
|
+
# to 0600. A taken or hostile path means no log - the crash report
|
|
104
|
+
# itself must still go out, so every failure here degrades to nil.
|
|
105
|
+
def write_tmp_log(error)
|
|
106
|
+
File.open(tmp_log_path, File::CREAT | File::EXCL | File::WRONLY | nofollow_flag, 0o600) do |file|
|
|
107
|
+
file.write("#{full_text(error)}\n")
|
|
108
|
+
end
|
|
109
|
+
tmp_log_path
|
|
110
|
+
rescue SystemCallError, IOError, ArgumentError
|
|
111
|
+
nil
|
|
112
|
+
end
|
|
113
|
+
|
|
114
|
+
# O_NOFOLLOW where the platform has it (every CI target does).
|
|
115
|
+
def nofollow_flag
|
|
116
|
+
File.const_defined?(:NOFOLLOW) ? File::NOFOLLOW : 0
|
|
117
|
+
end
|
|
118
|
+
|
|
119
|
+
# O_NONBLOCK where the platform has it (every CI target does).
|
|
120
|
+
def nonblock_flag
|
|
121
|
+
File.const_defined?(:NONBLOCK) ? File::NONBLOCK : 0
|
|
122
|
+
end
|
|
123
|
+
|
|
124
|
+
# The private crash-log write/read needs both flags: without O_NOFOLLOW
|
|
125
|
+
# a planted symlink redirects the open, without O_NONBLOCK a planted
|
|
126
|
+
# FIFO blocks it. Where either is missing, the log is omitted rather
|
|
127
|
+
# than opened unsafely (#296).
|
|
128
|
+
def safe_log_io?
|
|
129
|
+
File.const_defined?(:NOFOLLOW) && File.const_defined?(:NONBLOCK)
|
|
130
|
+
end
|
|
131
|
+
|
|
132
|
+
# Where the log lands when a configured cache dir is not writable - a
|
|
133
|
+
# sandboxed child holds the project read-only, but the system temp dir is
|
|
134
|
+
# granted (#296). Only a configured log path falls back: cache_dir = nil
|
|
135
|
+
# still means no log at all. Keyed by pid so parallel children never
|
|
136
|
+
# share a file; the runner reads the same path back in the same process.
|
|
137
|
+
def tmp_log_path(pid: Process.pid)
|
|
138
|
+
File.join(Dir.tmpdir, "saboteur-#{pid}-#{LOG_NAME}")
|
|
139
|
+
end
|
|
140
|
+
|
|
141
|
+
# The message names the log; relative to the working directory when it
|
|
142
|
+
# sits inside it, so the line stays short.
|
|
143
|
+
def display_path(path)
|
|
144
|
+
relative = Pathname(path).relative_path_from(Pathname(Dir.pwd)).to_s
|
|
145
|
+
relative.start_with?("..") ? path : relative
|
|
146
|
+
rescue ArgumentError
|
|
147
|
+
path
|
|
148
|
+
end
|
|
149
|
+
|
|
150
|
+
def scrub(string)
|
|
151
|
+
string.to_s.dup.force_encoding(Encoding::UTF_8).scrub("?")
|
|
152
|
+
end
|
|
153
|
+
end
|
|
154
|
+
end
|
data/lib/saboteur/custom_map.rb
CHANGED
|
@@ -1,6 +1,8 @@
|
|
|
1
1
|
# SPDX-License-Identifier: MIT
|
|
2
2
|
# frozen_string_literal: true
|
|
3
3
|
|
|
4
|
+
require_relative "text"
|
|
5
|
+
|
|
4
6
|
module Saboteur
|
|
5
7
|
# Explicit source-file -> spec-file mapping from config/saboteur_custom_map.rb:
|
|
6
8
|
#
|
|
@@ -109,7 +111,7 @@ module Saboteur
|
|
|
109
111
|
def duplicate_keys(path)
|
|
110
112
|
seen = Hash.new(0)
|
|
111
113
|
File.readlines(path).each do |line|
|
|
112
|
-
seen[::Regexp.last_match(1)] += 1 if line =~ /^\s*"(.+?)"\s*=>/
|
|
114
|
+
seen[::Regexp.last_match(1)] += 1 if Text.utf8(line) =~ /^\s*"(.+?)"\s*=>/
|
|
113
115
|
end
|
|
114
116
|
seen.select { |_, count| count > 1 }.keys
|
|
115
117
|
end
|
data/lib/saboteur/doctor.rb
CHANGED
|
@@ -3,6 +3,8 @@
|
|
|
3
3
|
|
|
4
4
|
require "coverage"
|
|
5
5
|
require_relative "baseline_health"
|
|
6
|
+
require_relative "engine/sandbox_policy"
|
|
7
|
+
require_relative "execution/sandbox"
|
|
6
8
|
|
|
7
9
|
module Saboteur
|
|
8
10
|
# Reports the current environment, never mutations. The two unchanged suite
|
|
@@ -25,8 +27,10 @@ module Saboteur
|
|
|
25
27
|
integration = boot_target(hooks)
|
|
26
28
|
@out.puts "Integration: #{@config['integration']} (loaded)"
|
|
27
29
|
@out.puts "Rails: #{@config['rails'] ? 'enabled' : 'disabled'}"
|
|
30
|
+
check_database_hosts
|
|
28
31
|
@out.puts "SimpleCov: #{defined?(::SimpleCov) ? 'present; start suppressed for this process' : 'not loaded'}"
|
|
29
|
-
|
|
32
|
+
verify_coverage
|
|
33
|
+
check_sandbox
|
|
30
34
|
check_suite(integration, hooks)
|
|
31
35
|
true
|
|
32
36
|
end
|
|
@@ -62,15 +66,51 @@ module Saboteur
|
|
|
62
66
|
"doctor supports the rspec integration; set integration: rspec"
|
|
63
67
|
end
|
|
64
68
|
|
|
69
|
+
# Printed once; verified twice. Boot must not wake coverage up, so the
|
|
70
|
+
# post-boot check re-reads the state without printing a second line.
|
|
65
71
|
def check_coverage
|
|
66
72
|
state = Coverage.state
|
|
67
73
|
@out.puts "Coverage: #{state}"
|
|
74
|
+
verify_coverage(state)
|
|
75
|
+
end
|
|
76
|
+
|
|
77
|
+
def verify_coverage(state = Coverage.state)
|
|
68
78
|
return if Coverage.supported?(:lines) && Coverage.supported?(:methods) && state == :suspended
|
|
69
79
|
|
|
70
80
|
raise Error, "Coverage must be suspended with line and method support before boot; use the saboteur " \
|
|
71
81
|
"executable, disable SimpleCov and remove other Coverage.start calls"
|
|
72
82
|
end
|
|
73
83
|
|
|
84
|
+
# A kernel capability probe, not verified isolation: what the kernel
|
|
85
|
+
# would let mutant children do, and what the run does with that - the
|
|
86
|
+
# engine's sandbox policy owns the refusal or degradation itself.
|
|
87
|
+
def check_sandbox
|
|
88
|
+
if @config["sandbox"] == false
|
|
89
|
+
@out.puts "Sandbox: off (c.sandbox = false)"
|
|
90
|
+
return
|
|
91
|
+
end
|
|
92
|
+
abi = sandbox_abi
|
|
93
|
+
if abi >= 3
|
|
94
|
+
@out.puts "Sandbox: kernel Landlock ABI #{abi} - capability probe only, not verified isolation"
|
|
95
|
+
elsif @config["sandbox"] == "strict"
|
|
96
|
+
@out.puts "Sandbox: #{Engine::SandboxPolicy.sandbox_gap(abi)}; " \
|
|
97
|
+
'c.sandbox = "strict" refuses to run - children never run degraded'
|
|
98
|
+
else
|
|
99
|
+
@out.puts "Sandbox: #{Engine::SandboxPolicy.sandbox_gap(abi)}; " \
|
|
100
|
+
"children would run with the hazard guard and canary only"
|
|
101
|
+
end
|
|
102
|
+
end
|
|
103
|
+
|
|
104
|
+
# The probe contract the engine's policy already keeps: an errored probe
|
|
105
|
+
# (EPERM under a seccomp policy, ...) is an actionable tool error, not a
|
|
106
|
+
# CLI internal error - doctor stays read-only.
|
|
107
|
+
def sandbox_abi
|
|
108
|
+
Execution::Sandbox.abi_version
|
|
109
|
+
rescue SystemCallError => e
|
|
110
|
+
raise Error, "sandbox is on but the Landlock probe failed (#{e.message}); a run would refuse the same way - " \
|
|
111
|
+
"set c.sandbox = false to check without the sandbox"
|
|
112
|
+
end
|
|
113
|
+
|
|
74
114
|
# Match against all included source definitions, not the current diff or
|
|
75
115
|
# selected tests. A limited run must not make a valid ignore look stale.
|
|
76
116
|
def check_ignore_subjects
|
|
@@ -105,11 +145,33 @@ module Saboteur
|
|
|
105
145
|
@out.puts "Custom map: #{map.entries.size} entries (loaded)"
|
|
106
146
|
end
|
|
107
147
|
|
|
148
|
+
# Prints the test database hosts the booted app reports, so a non-local
|
|
149
|
+
# refusal names the host here instead of only at target boot (issue #258).
|
|
150
|
+
def check_database_hosts
|
|
151
|
+
return unless @config["rails"]
|
|
152
|
+
return @out.puts("Databases: ActiveRecord did not load; host check skipped") unless defined?(::ActiveRecord::Base)
|
|
153
|
+
|
|
154
|
+
configs = ::ActiveRecord::Base.configurations.configs_for(env_name: "test")
|
|
155
|
+
configs.each do |db|
|
|
156
|
+
host = db.configuration_hash[:host]
|
|
157
|
+
@out.puts "Database: #{db.name} (host #{host.to_s.empty? ? 'local socket' : host})"
|
|
158
|
+
end
|
|
159
|
+
remote = Frameworks::Rails.remote_databases(configs, @config["allow_db_hosts"])
|
|
160
|
+
return if remote.empty?
|
|
161
|
+
|
|
162
|
+
raise ConfigError, "non-local test databases: #{remote.join(', ')}; if they are disposable test databases, " \
|
|
163
|
+
"allow them with c.allow_db_hosts in #{Config::FILE}"
|
|
164
|
+
end
|
|
165
|
+
|
|
108
166
|
def check_boot_files
|
|
109
167
|
%w[require hooks].each do |key|
|
|
110
168
|
@config[key].each do |file|
|
|
111
169
|
path = File.expand_path(file, @config.root)
|
|
112
|
-
|
|
170
|
+
# Resolve the way Kernel#require does: the path as given, or with
|
|
171
|
+
# .rb appended (the Rails quickstart sets ./config/environment, issue #257).
|
|
172
|
+
unless File.file?(path) || File.file?("#{path}.rb")
|
|
173
|
+
raise ConfigError, "#{key} file missing: #{file}; fix config/saboteur.rb"
|
|
174
|
+
end
|
|
113
175
|
|
|
114
176
|
@out.puts "#{key}: #{file} (found)"
|
|
115
177
|
end
|
|
@@ -3,6 +3,7 @@
|
|
|
3
3
|
|
|
4
4
|
require "json"
|
|
5
5
|
require "fileutils"
|
|
6
|
+
require_relative "../text"
|
|
6
7
|
|
|
7
8
|
module Saboteur
|
|
8
9
|
class Engine
|
|
@@ -10,16 +11,28 @@ module Saboteur
|
|
|
10
11
|
# or mutation source is used. This catches observed instability, not all
|
|
11
12
|
# possible flaky behaviour.
|
|
12
13
|
class BaselineGuard
|
|
14
|
+
# How many unchanged-suite checks actually completed and validated:
|
|
15
|
+
# the number of order comparisons a run can claim is one less.
|
|
16
|
+
attr_reader :completed_passes
|
|
17
|
+
|
|
13
18
|
EXCERPT_LINES = 10
|
|
14
19
|
EXCERPT_WIDTH = 200
|
|
15
20
|
EXCERPT_SCAN_LINES = 5000
|
|
16
21
|
|
|
17
|
-
|
|
22
|
+
# full_passes: every full-suite pass the run will make, including the
|
|
23
|
+
# coverage build when the cache is cold, so the first timed pass can
|
|
24
|
+
# price the rest (#260).
|
|
25
|
+
def initialize(config:, integration:, hooks:, supervisor:, diagnostics:, full_passes: 2, between_passes: nil,
|
|
26
|
+
timeout_cap: nil)
|
|
18
27
|
@config = config
|
|
19
28
|
@integration = integration
|
|
20
29
|
@hooks = hooks
|
|
21
30
|
@supervisor = supervisor
|
|
22
31
|
@diagnostics = diagnostics
|
|
32
|
+
@full_passes = full_passes
|
|
33
|
+
@between_passes = between_passes
|
|
34
|
+
@timeout_cap = timeout_cap
|
|
35
|
+
@completed_passes = 0
|
|
23
36
|
end
|
|
24
37
|
|
|
25
38
|
def call
|
|
@@ -27,11 +40,16 @@ module Saboteur
|
|
|
27
40
|
raise Error, "baseline has no tests. Check rspec_args and the suite filters before running saboteur again." if
|
|
28
41
|
expected.empty?
|
|
29
42
|
|
|
30
|
-
|
|
43
|
+
passes = @config["baseline_passes"]
|
|
44
|
+
@diagnostics.inform(announce(passes))
|
|
31
45
|
previous = nil
|
|
32
|
-
|
|
46
|
+
passes.times do |index|
|
|
47
|
+
break if budget_spent?(index)
|
|
48
|
+
|
|
33
49
|
number = index + 1
|
|
50
|
+
started = Process.clock_gettime(Process::CLOCK_MONOTONIC)
|
|
34
51
|
data = check(number)
|
|
52
|
+
announce_first_pass_cost(number, started)
|
|
35
53
|
unless data["passed"] && data["failed"].empty?
|
|
36
54
|
refuse(number,
|
|
37
55
|
"the suite failed: #{Array(data['failed']).join(', ')}")
|
|
@@ -41,18 +59,73 @@ module Saboteur
|
|
|
41
59
|
executed.sort == expected.sort
|
|
42
60
|
refuse(number, "test order changed between baseline checks") if previous && executed != previous
|
|
43
61
|
previous = executed
|
|
62
|
+
@completed_passes += 1
|
|
44
63
|
end
|
|
45
64
|
nil
|
|
46
65
|
end
|
|
47
66
|
|
|
48
67
|
private
|
|
49
68
|
|
|
69
|
+
# A spent --max-duration budget skips the remaining passes - including
|
|
70
|
+
# the first, when boot alone already spent it (#245).
|
|
71
|
+
def budget_spent?(_index)
|
|
72
|
+
!@between_passes.nil? && @between_passes.call
|
|
73
|
+
end
|
|
74
|
+
|
|
75
|
+
def announce(passes)
|
|
76
|
+
case passes
|
|
77
|
+
when 1
|
|
78
|
+
"checking the unchanged suite once (seed #{@config['seed']}); " \
|
|
79
|
+
"order stability between passes is not checked (baseline_passes: 1)"
|
|
80
|
+
when 2
|
|
81
|
+
"checking the unchanged suite twice (seed #{@config['seed']})"
|
|
82
|
+
else
|
|
83
|
+
"checking the unchanged suite #{passes} times (seed #{@config['seed']})"
|
|
84
|
+
end
|
|
85
|
+
end
|
|
86
|
+
|
|
87
|
+
# Once the first pass is timed, the total cost stops being a guess:
|
|
88
|
+
# the remaining full passes cost roughly the same each.
|
|
89
|
+
def announce_first_pass_cost(number, started)
|
|
90
|
+
return unless number == 1
|
|
91
|
+
|
|
92
|
+
remaining = @full_passes - number
|
|
93
|
+
return if remaining.zero?
|
|
94
|
+
|
|
95
|
+
elapsed = (Process.clock_gettime(Process::CLOCK_MONOTONIC) - started).round
|
|
96
|
+
@diagnostics.inform("baseline check #{number} took #{elapsed} s; about #{elapsed * remaining} s left for " \
|
|
97
|
+
"the remaining #{remaining} full #{remaining == 1 ? 'pass' : 'passes'} at that rate")
|
|
98
|
+
end
|
|
99
|
+
|
|
100
|
+
# An admitted pass still gets only the time left in the run budget:
|
|
101
|
+
# without the cap it would overrun a spent --max-duration by its
|
|
102
|
+
# whole coverage_timeout.
|
|
103
|
+
def phase_timeout
|
|
104
|
+
@timeout_cap ? @timeout_cap.call(@config["coverage_timeout"]) : @config["coverage_timeout"]
|
|
105
|
+
end
|
|
106
|
+
|
|
107
|
+
# A pass that hit its deadline only because --max-duration shortened
|
|
108
|
+
# the admission below the configured phase timeout: the budget ended
|
|
109
|
+
# the run, the suite is not broken (#296).
|
|
110
|
+
def budget_stopped?(result, admitted)
|
|
111
|
+
result.stopped_by == :timeout && admitted < @config["coverage_timeout"]
|
|
112
|
+
end
|
|
113
|
+
|
|
114
|
+
def refuse_budget_stop(number, admitted)
|
|
115
|
+
raise BudgetPhaseTimeout, "the --max-duration budget stopped baseline check #{number} " \
|
|
116
|
+
"(admitted #{admitted.round(1)}s of the #{@config['coverage_timeout']}s " \
|
|
117
|
+
"phase timeout)"
|
|
118
|
+
end
|
|
119
|
+
|
|
50
120
|
def check(number)
|
|
51
121
|
Execution::ForkReadiness.prepare_heap
|
|
52
122
|
heartbeat = @diagnostics.progress_heartbeat("baseline check #{number}").watch(log_path(number))
|
|
53
|
-
|
|
123
|
+
admitted = phase_timeout
|
|
124
|
+
result = @supervisor.run(timeout: admitted, on_tick: heartbeat.method(:tick),
|
|
125
|
+
label: "baseline child died during setup/run") do |writer|
|
|
54
126
|
record(writer, number)
|
|
55
127
|
end
|
|
128
|
+
refuse_budget_stop(number, admitted) if budget_stopped?(result, admitted)
|
|
56
129
|
refuse(number, "the suite stopped: #{result.stopped_by} (#{result.detail})") if result.stopped_by
|
|
57
130
|
# Before the payload checks: a child killed by a signal (the OOM
|
|
58
131
|
# killer, say) writes nothing, and the signal is the explanation. A
|
|
@@ -129,7 +202,8 @@ module Saboteur
|
|
|
129
202
|
end
|
|
130
203
|
|
|
131
204
|
def log_lines(number)
|
|
132
|
-
File.foreach(log_path(number), chomp: true).first(EXCERPT_SCAN_LINES)
|
|
205
|
+
File.foreach(log_path(number), chomp: true).first(EXCERPT_SCAN_LINES)
|
|
206
|
+
.map { |line| Text.utf8(line) }.reject { |line| line.strip.empty? }
|
|
133
207
|
rescue SystemCallError
|
|
134
208
|
[]
|
|
135
209
|
end
|
data/lib/saboteur/engine/boot.rb
CHANGED
|
@@ -83,10 +83,21 @@ module Saboteur
|
|
|
83
83
|
|
|
84
84
|
def load_hooks
|
|
85
85
|
Saboteur.current_hooks = @hooks
|
|
86
|
-
@config["hooks"].each { |file| load
|
|
86
|
+
@config["hooks"].each { |file| load hook_path(file) }
|
|
87
87
|
ensure
|
|
88
88
|
Saboteur.current_hooks = nil
|
|
89
89
|
end
|
|
90
|
+
|
|
91
|
+
# Kernel#load does not append .rb the way Kernel#require does, and
|
|
92
|
+
# doctor accepts the extensionless form (#257), so resolve it the same
|
|
93
|
+
# way here: the exact path first, then path.rb.
|
|
94
|
+
def hook_path(file)
|
|
95
|
+
path = File.expand_path(file, @root)
|
|
96
|
+
rb = "#{path}.rb"
|
|
97
|
+
return rb if !File.exist?(path) && File.extname(path).empty? && File.file?(rb)
|
|
98
|
+
|
|
99
|
+
path
|
|
100
|
+
end
|
|
90
101
|
end
|
|
91
102
|
end
|
|
92
103
|
end
|