saboteur 0.5.0 → 0.7.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (83) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +399 -0
  3. data/README.md +202 -42
  4. data/exe/saboteur +34 -7
  5. data/lib/saboteur/baseline/comparison.rb +1 -0
  6. data/lib/saboteur/baseline/evidence.rb +11 -1
  7. data/lib/saboteur/baseline/run_evidence.rb +1 -1
  8. data/lib/saboteur/baseline/workflow.rb +2 -1
  9. data/lib/saboteur/baseline_health.rb +40 -6
  10. data/lib/saboteur/cli/exit_status.rb +49 -4
  11. data/lib/saboteur/cli/help_options.rb +16 -0
  12. data/lib/saboteur/cli/presets.rb +36 -0
  13. data/lib/saboteur/cli/run_budget_options.rb +37 -0
  14. data/lib/saboteur/cli/run_reporting.rb +75 -6
  15. data/lib/saboteur/cli.rb +80 -48
  16. data/lib/saboteur/config.rb +88 -13
  17. data/lib/saboteur/crash_report.rb +154 -0
  18. data/lib/saboteur/custom_map.rb +3 -1
  19. data/lib/saboteur/doctor.rb +64 -2
  20. data/lib/saboteur/engine/baseline_guard.rb +79 -5
  21. data/lib/saboteur/engine/boot.rb +12 -1
  22. data/lib/saboteur/engine/canary.rb +41 -10
  23. data/lib/saboteur/engine/candidate_selection.rb +9 -1
  24. data/lib/saboteur/engine/candidates.rb +4 -1
  25. data/lib/saboteur/engine/coverage_guard.rb +29 -0
  26. data/lib/saboteur/engine/coverage_store.rb +48 -6
  27. data/lib/saboteur/engine/diagnostics.rb +23 -2
  28. data/lib/saboteur/engine/diff_scope.rb +24 -5
  29. data/lib/saboteur/engine/distribution.rb +124 -8
  30. data/lib/saboteur/engine/early_stop_recording.rb +46 -0
  31. data/lib/saboteur/engine/result_cache.rb +2 -1
  32. data/lib/saboteur/engine/run_budget.rb +63 -0
  33. data/lib/saboteur/engine/run_fingerprint.rb +5 -1
  34. data/lib/saboteur/engine/sandbox_policy.rb +18 -11
  35. data/lib/saboteur/engine/scheduler.rb +114 -18
  36. data/lib/saboteur/engine/subject_selection.rb +84 -5
  37. data/lib/saboteur/engine/suite_files.rb +3 -1
  38. data/lib/saboteur/engine/test_guess.rb +183 -23
  39. data/lib/saboteur/engine/thread_warnings.rb +53 -0
  40. data/lib/saboteur/engine/unscoped_suite_warning.rb +41 -0
  41. data/lib/saboteur/engine/verdict_check.rb +54 -18
  42. data/lib/saboteur/engine.rb +175 -96
  43. data/lib/saboteur/execution/protocol.rb +2 -1
  44. data/lib/saboteur/execution/runner.rb +179 -17
  45. data/lib/saboteur/execution/sandbox.rb +6 -1
  46. data/lib/saboteur/execution/supervisor.rb +18 -3
  47. data/lib/saboteur/execution/worker_pool.rb +87 -7
  48. data/lib/saboteur/frameworks/database_hosts.rb +30 -0
  49. data/lib/saboteur/frameworks/rails.rb +8 -10
  50. data/lib/saboteur/help.rb +55 -10
  51. data/lib/saboteur/ignore_list.rb +11 -1
  52. data/lib/saboteur/install.rb +4 -3
  53. data/lib/saboteur/integration/base.rb +7 -2
  54. data/lib/saboteur/integration/rspec.rb +40 -4
  55. data/lib/saboteur/mutation_source/builtin.rb +22 -13
  56. data/lib/saboteur/mutation_source/hazard_guard.rb +21 -3
  57. data/lib/saboteur/mutation_source/interpolation_removal.rb +41 -0
  58. data/lib/saboteur/mutation_source/method_swap.rb +45 -0
  59. data/lib/saboteur/mutation_source/mutation.rb +5 -3
  60. data/lib/saboteur/mutation_source/operand_replacement.rb +54 -0
  61. data/lib/saboteur/mutation_source/operator_examples.rb +38 -0
  62. data/lib/saboteur/mutation_source/operator_registry.rb +36 -8
  63. data/lib/saboteur/mutation_source/operator_visit.rb +39 -0
  64. data/lib/saboteur/preview.rb +12 -2
  65. data/lib/saboteur/reporter/agent_commands.rb +49 -16
  66. data/lib/saboteur/reporter/agent_incomplete.rb +27 -2
  67. data/lib/saboteur/reporter/agent_summary.rb +1 -1
  68. data/lib/saboteur/reporter/history_policy.rb +35 -0
  69. data/lib/saboteur/reporter/history_record.rb +4 -5
  70. data/lib/saboteur/reporter/next_steps.rb +14 -4
  71. data/lib/saboteur/reporter/score_history.rb +19 -3
  72. data/lib/saboteur/reporter/stryker.rb +19 -8
  73. data/lib/saboteur/reporter/survivor_hints.rb +3 -0
  74. data/lib/saboteur/reporter/terminal.rb +43 -10
  75. data/lib/saboteur/result.rb +25 -7
  76. data/lib/saboteur/selection/coverage_map.rb +2 -1
  77. data/lib/saboteur/selection/git_diff.rb +15 -5
  78. data/lib/saboteur/selection/untracked_changes.rb +126 -0
  79. data/lib/saboteur/subject.rb +11 -0
  80. data/lib/saboteur/text.rb +42 -0
  81. data/lib/saboteur/version.rb +1 -1
  82. data/lib/saboteur.rb +16 -1
  83. metadata +20 -2
@@ -1,6 +1,8 @@
1
1
  # SPDX-License-Identifier: MIT
2
2
  # frozen_string_literal: true
3
3
 
4
+ require_relative "text"
5
+
4
6
  require_relative "mutation_source/builtin"
5
7
  require_relative "mutation_source/operator_registry"
6
8
  require_relative "mutation_source/hazard_guard"
@@ -39,8 +41,11 @@ module Saboteur
39
41
  "custom_map" => "config/saboteur_custom_map.rb",
40
42
  "sources" => ["builtin"],
41
43
  "operators" => [],
44
+ "exclude_operators" => [],
42
45
  "hooks" => [],
43
46
  "rails" => false,
47
+ # Hosts the user asserts are disposable test databases (issue #258).
48
+ "allow_db_hosts" => [],
44
49
  "reeval" => "nesting",
45
50
  "cache_dir" => ".saboteur",
46
51
  "json_report" => ".saboteur/report.json",
@@ -54,8 +59,13 @@ module Saboteur
54
59
  "allow_focus" => false,
55
60
  "fail_fast" => false,
56
61
  "guess_tests" => false,
62
+ "all" => false,
63
+ "test_selection" => "auto",
57
64
  "matrix" => false,
58
65
  "seed" => 1,
66
+ "baseline_passes" => 2,
67
+ "max_duration" => nil,
68
+ "max_duration_ok" => false,
59
69
  "tests" => [],
60
70
  "fingerprint_extra" => [],
61
71
  "threshold" => nil,
@@ -63,12 +73,13 @@ module Saboteur
63
73
  "max_errors" => nil
64
74
  }.freeze
65
75
 
66
- INTEGER_KEYS = %w[jobs max_memory_mb max_processes max_mutants_per_method seed].freeze
76
+ INTEGER_KEYS = %w[jobs max_memory_mb max_processes max_mutants_per_method seed baseline_passes].freeze
67
77
  FLOAT_KEYS = %w[timeout_factor timeout_min baseline_timeout coverage_timeout].freeze
78
+ NILABLE_FLOAT_KEYS = %w[threshold max_duration].freeze
68
79
  THRESHOLD_RANGE = (0.0..100.0)
69
80
 
70
81
  LIST_KEYS = %w[require include exclude arid hazard sources hooks rspec_args subjects tests
71
- fingerprint_extra sandbox_writable operators whole_file].freeze
82
+ fingerprint_extra sandbox_writable operators exclude_operators whole_file allow_db_hosts].freeze
72
83
 
73
84
  attr_reader :root
74
85
 
@@ -93,11 +104,19 @@ module Saboteur
93
104
  hash.to_h { |key, value| [key.to_s, value] }.compact
94
105
  end
95
106
 
96
- def initialize(values = {}, root: Dir.pwd)
97
- @root = File.expand_path(root)
107
+ # Deprecations found while loading: printed by the CLI, never fatal.
108
+ attr_reader :deprecation_warnings
109
+
110
+ def initialize(values = {}, root: Dir.pwd, explicit: nil)
111
+ # Retag the root losslessly: under a non-UTF-8 locale it arrives
112
+ # locale-tagged, and every glob, join and expand below would mix
113
+ # encodings with the UTF-8 patterns from the config file (#231).
114
+ @root = Text.utf8_identity(File.expand_path(root))
98
115
  unknown = values.keys - DEFAULTS.keys
99
116
  raise ConfigError, "unknown config key(s): #{unknown.join(', ')}" if unknown.any?
100
117
 
118
+ @deprecation_warnings = deprecations(values)
119
+ @explicit_keys = (explicit || values.keys).map(&:to_s).freeze
101
120
  @values = DEFAULTS.merge(values)
102
121
  coerce!
103
122
  validate!
@@ -112,6 +131,26 @@ module Saboteur
112
131
  @values.fetch(key.to_s, default)
113
132
  end
114
133
 
134
+ # Keys the user set (config file or CLI flag) rather than relying on
135
+ # the default; carried through Engine's config clone so provenance
136
+ # survives (#296).
137
+ attr_reader :explicit_keys
138
+
139
+ # A copy with some values replaced or dropped, keeping which keys the
140
+ # user set explicitly: the replacements count as explicit, the dropped
141
+ # keys lose it (#296).
142
+ def with(values, except: [])
143
+ merged = to_h.except(*except).merge(values)
144
+ Config.new(merged, root: root, explicit: (explicit_keys - except.map(&:to_s)) | values.keys.map(&:to_s))
145
+ end
146
+
147
+ # Whether the user set a key (config file or CLI flag) rather than
148
+ # relying on the default. An explicitly requested report artifact is
149
+ # completion evidence, so its write failure is a hard error (#296).
150
+ def explicit?(key)
151
+ @explicit_keys.include?(key.to_s)
152
+ end
153
+
115
154
  def to_h
116
155
  @values.dup
117
156
  end
@@ -121,7 +160,8 @@ module Saboteur
121
160
  matched = self["include"].flat_map { |pattern| Dir.glob(pattern, base: root) }
122
161
  excluded = self["exclude"].flat_map { |pattern| Dir.glob(pattern, base: root) }
123
162
  (matched - excluded).select { |file| file.end_with?(".rb") }
124
- .map { |file| File.realpath(File.expand_path(file, root)) }.uniq.sort
163
+ .map { |file| File.realpath(File.expand_path(Text.utf8_identity(file), root)) }
164
+ .uniq.sort
125
165
  end
126
166
 
127
167
  # Patterns that match no file. A pattern goes stale when files move;
@@ -148,11 +188,23 @@ module Saboteur
148
188
 
149
189
  private
150
190
 
191
+ # An explicit guess_tests = false (config file or --no-guess-tests) used
192
+ # to keep the whole suite; inference is the default now, so the opt-out
193
+ # is --all / c.test_selection = :all. guess_tests = true is the default
194
+ # behavior spelled out, so it stays silent.
195
+ def deprecations(values)
196
+ return [] unless values["guess_tests"] == false
197
+
198
+ ["guess_tests = false is deprecated: inferred specs are the default and it no longer restores the " \
199
+ "whole suite; use --all or c.test_selection = :all to run every spec"]
200
+ end
201
+
151
202
  def coerce!
152
203
  INTEGER_KEYS.each { |key| @values[key] = Integer(@values[key]) }
153
204
  FLOAT_KEYS.each { |key| @values[key] = Float(@values[key]) }
154
205
  LIST_KEYS.each { |key| @values[key] = Array(@values[key]).map(&:to_s) }
155
- @values["threshold"] = Float(@values["threshold"]) unless @values["threshold"].nil?
206
+ NILABLE_FLOAT_KEYS.each { |key| @values[key] = Float(@values[key]) unless @values[key].nil? }
207
+ @values["test_selection"] = @values["test_selection"].to_s
156
208
  rescue ArgumentError, TypeError => e
157
209
  raise ConfigError, "invalid config value: #{e.message}"
158
210
  end
@@ -161,6 +213,13 @@ module Saboteur
161
213
  raise ConfigError, "max_mutants_per_method must be > 0" unless self["max_mutants_per_method"].positive?
162
214
  end
163
215
 
216
+ # The unchanged-suite checks: how many, and the whole run's clock (#55, #245).
217
+ def validate_baseline!
218
+ raise ConfigError, "seed must be >= 0" if self["seed"].negative?
219
+ raise ConfigError, "baseline_passes must be >= 1" if self["baseline_passes"] < 1
220
+ raise ConfigError, "max_duration must be > 0" if self["max_duration"] && !self["max_duration"].positive?
221
+ end
222
+
164
223
  def validate_report_paths!
165
224
  json = path("json_report")
166
225
  if json && json == path("html_report")
@@ -176,7 +235,7 @@ module Saboteur
176
235
  def validate_modes!
177
236
  validate_fail_fast!
178
237
  validate_matrix!
179
- validate_guess_tests!
238
+ validate_test_selection!
180
239
  validate_gates!
181
240
  end
182
241
 
@@ -205,11 +264,10 @@ module Saboteur
205
264
  raise ConfigError, "threshold must be between 0 and 100"
206
265
  end
207
266
 
208
- def validate_guess_tests!
209
- return unless self["guess_tests"]
267
+ def validate_test_selection!
268
+ return if %w[auto all].include?(self["test_selection"])
210
269
 
211
- raise ConfigError, "guess_tests needs since" unless self["since"]
212
- raise ConfigError, "guess_tests cannot be combined with tests" if self["tests"].any?
270
+ raise ConfigError, "test_selection must be one of: auto, all"
213
271
  end
214
272
 
215
273
  def validate_reeval!
@@ -226,7 +284,7 @@ module Saboteur
226
284
  validate_mutant_limit!
227
285
  validate_threshold!
228
286
  validate_modes!
229
- raise ConfigError, "seed must be >= 0" if self["seed"].negative?
287
+ validate_baseline!
230
288
  raise ConfigError, "max_memory_mb must be > 0" unless self["max_memory_mb"].positive?
231
289
  raise ConfigError, "max_processes must be > 0" unless self["max_processes"].positive?
232
290
  raise ConfigError, "timeout_factor must be > 0" unless self["timeout_factor"].positive?
@@ -236,8 +294,25 @@ module Saboteur
236
294
  validate_operator_config!
237
295
  end
238
296
 
297
+ public
298
+
299
+ def operator_policy
300
+ MutationSource::OperatorRegistry.policy(self["operators"], self["exclude_operators"])
301
+ end
302
+
303
+ # Versioned policy travels with every identity, not with mutation digests.
304
+ def selection_settings
305
+ to_h.merge("operator_policy" => operator_policy)
306
+ end
307
+
308
+ private
309
+
239
310
  def validate_operator_config!
240
- MutationSource::OperatorRegistry.resolve(self["operators"])
311
+ effective = MutationSource::OperatorRegistry.select(self["operators"], self["exclude_operators"])
312
+ if effective.empty?
313
+ raise ConfigError,
314
+ "operator selection is empty: enable at least one operator or clear exclusions"
315
+ end
241
316
 
242
317
  self["arid"].each { |pattern| Regexp.new(pattern) }
243
318
  rescue RegexpError => e
@@ -0,0 +1,154 @@
1
+ # SPDX-License-Identifier: MIT
2
+ # frozen_string_literal: true
3
+
4
+ require "fileutils"
5
+ require "json"
6
+ require "tmpdir"
7
+ require "pathname"
8
+
9
+ module Saboteur
10
+ # Last-resort crash reporting, shared by the CLI's StandardError rescue
11
+ # and the exe-level wrapper: one bounded line on stderr, the full
12
+ # backtrace in a log file the line names, the backtrace on stderr with
13
+ # --verbose, and the same failure as agent JSON when an agent listens.
14
+ # Always returns EXIT_TOOL.
15
+ module CrashReport
16
+ ISSUES_URL = "https://github.com/vamsipavanmahesh/saboteur/issues"
17
+
18
+ # Same value as CLI::EXIT_TOOL, kept local so the reporter still works
19
+ # when the CLI itself never finished loading.
20
+ EXIT_TOOL = 3
21
+
22
+ # The crash log's file name under the run's cache dir; the runner reads
23
+ # it back to explain a nested run's crash in the failure detail (#296).
24
+ LOG_NAME = "internal-error.log"
25
+
26
+ module_function
27
+
28
+ def call(error, verbose:, agent:, err:, out:, log_path: nil)
29
+ summary = "internal error (#{error.class}: #{one_line(error.message)})"
30
+ line = "saboteur: #{summary} - please report it: #{ISSUES_URL}"
31
+ written = write_log(log_path, error)
32
+ line = "#{line}; full backtrace: #{display_path(written)}" if written
33
+ err.puts(line)
34
+ err.puts(full_text(error)) if verbose
35
+ out.print(agent_json(summary)) if agent
36
+ EXIT_TOOL
37
+ end
38
+
39
+ # A crash message can span lines or carry bytes that are invalid in
40
+ # UTF-8 (a LANG=C failure, say); the one-line diagnostic must survive
41
+ # both, and stay one bounded line.
42
+ def one_line(message)
43
+ text = scrub(message).gsub(/\s+/, " ").strip
44
+ text.length > 160 ? "#{text[0...160]}..." : text
45
+ end
46
+
47
+ # Class, full message and backtrace, scrubbed so locale-invalid bytes
48
+ # cannot corrupt the log or the terminal.
49
+ def full_text(error)
50
+ (["#{error.class}: #{scrub(error.message)}"] + Array(error.backtrace).map { |frame| scrub(frame) }).join("\n")
51
+ end
52
+
53
+ # Writes the log next to the other run state; returns the written path,
54
+ # or nil when the log cannot be written - the crash itself must still
55
+ # be reported, so an unwritable directory is not a second failure.
56
+ # Reporter::Agent.failure's payload, reproduced minimally for crashes
57
+ # before the CLI's reporter files load: same schema and pretty rendering;
58
+ # the summary is already normalized, so no message cap is needed here.
59
+ def agent_json(summary)
60
+ payload =
61
+ if defined?(Reporter::Agent)
62
+ Reporter::Agent.failure(summary, EXIT_TOOL)
63
+ else
64
+ { version: 1, status: "error", verdict: "tool_failure", exitCode: EXIT_TOOL,
65
+ message: summary, next: { help: "saboteur help run" } }
66
+ end
67
+ "#{JSON.pretty_generate(payload)}\n"
68
+ end
69
+
70
+ def write_log(log_path, error)
71
+ return nil unless log_path
72
+
73
+ # The primary path is user-configured but still predictable: without
74
+ # O_NOFOLLOW a planted symlink would redirect the truncate/chmod, and
75
+ # without O_NONBLOCK a planted FIFO would block the open. Where the
76
+ # platform lacks either, only the EXCL-created temp fallback is safe
77
+ # (#296).
78
+ if safe_log_io?
79
+ begin
80
+ FileUtils.mkdir_p(File.dirname(log_path))
81
+ # Private like the temp fallback: the runner only reads logs with
82
+ # no group/other bits, from the same no-follow fd (#296).
83
+ flags = File::CREAT | File::WRONLY | File::TRUNC | nofollow_flag | nonblock_flag
84
+ File.open(log_path, flags, 0o600) do |file|
85
+ file.chmod(0o600)
86
+ file.write("#{full_text(error)}\n")
87
+ end
88
+ return log_path
89
+ rescue SystemCallError, IOError
90
+ # Fall through to the temp fallback.
91
+ end
92
+ end
93
+ write_tmp_log(error)
94
+ end
95
+
96
+ # The fallback is computed lazily, only once the primary write failed:
97
+ # an eager tmp_log_path call could itself raise (Dir.tmpdir finds no
98
+ # usable dir) and mask the crash it was meant to log (#296).
99
+ #
100
+ # The fallback path is predictable and lives in a shared directory, so
101
+ # it is created privately: CREAT|EXCL|WRONLY|NOFOLLOW never follows or
102
+ # overwrites a pre-existing file or symlink, and the mode pins the log
103
+ # to 0600. A taken or hostile path means no log - the crash report
104
+ # itself must still go out, so every failure here degrades to nil.
105
+ def write_tmp_log(error)
106
+ File.open(tmp_log_path, File::CREAT | File::EXCL | File::WRONLY | nofollow_flag, 0o600) do |file|
107
+ file.write("#{full_text(error)}\n")
108
+ end
109
+ tmp_log_path
110
+ rescue SystemCallError, IOError, ArgumentError
111
+ nil
112
+ end
113
+
114
+ # O_NOFOLLOW where the platform has it (every CI target does).
115
+ def nofollow_flag
116
+ File.const_defined?(:NOFOLLOW) ? File::NOFOLLOW : 0
117
+ end
118
+
119
+ # O_NONBLOCK where the platform has it (every CI target does).
120
+ def nonblock_flag
121
+ File.const_defined?(:NONBLOCK) ? File::NONBLOCK : 0
122
+ end
123
+
124
+ # The private crash-log write/read needs both flags: without O_NOFOLLOW
125
+ # a planted symlink redirects the open, without O_NONBLOCK a planted
126
+ # FIFO blocks it. Where either is missing, the log is omitted rather
127
+ # than opened unsafely (#296).
128
+ def safe_log_io?
129
+ File.const_defined?(:NOFOLLOW) && File.const_defined?(:NONBLOCK)
130
+ end
131
+
132
+ # Where the log lands when a configured cache dir is not writable - a
133
+ # sandboxed child holds the project read-only, but the system temp dir is
134
+ # granted (#296). Only a configured log path falls back: cache_dir = nil
135
+ # still means no log at all. Keyed by pid so parallel children never
136
+ # share a file; the runner reads the same path back in the same process.
137
+ def tmp_log_path(pid: Process.pid)
138
+ File.join(Dir.tmpdir, "saboteur-#{pid}-#{LOG_NAME}")
139
+ end
140
+
141
+ # The message names the log; relative to the working directory when it
142
+ # sits inside it, so the line stays short.
143
+ def display_path(path)
144
+ relative = Pathname(path).relative_path_from(Pathname(Dir.pwd)).to_s
145
+ relative.start_with?("..") ? path : relative
146
+ rescue ArgumentError
147
+ path
148
+ end
149
+
150
+ def scrub(string)
151
+ string.to_s.dup.force_encoding(Encoding::UTF_8).scrub("?")
152
+ end
153
+ end
154
+ end
@@ -1,6 +1,8 @@
1
1
  # SPDX-License-Identifier: MIT
2
2
  # frozen_string_literal: true
3
3
 
4
+ require_relative "text"
5
+
4
6
  module Saboteur
5
7
  # Explicit source-file -> spec-file mapping from config/saboteur_custom_map.rb:
6
8
  #
@@ -109,7 +111,7 @@ module Saboteur
109
111
  def duplicate_keys(path)
110
112
  seen = Hash.new(0)
111
113
  File.readlines(path).each do |line|
112
- seen[::Regexp.last_match(1)] += 1 if line =~ /^\s*"(.+?)"\s*=>/
114
+ seen[::Regexp.last_match(1)] += 1 if Text.utf8(line) =~ /^\s*"(.+?)"\s*=>/
113
115
  end
114
116
  seen.select { |_, count| count > 1 }.keys
115
117
  end
@@ -3,6 +3,8 @@
3
3
 
4
4
  require "coverage"
5
5
  require_relative "baseline_health"
6
+ require_relative "engine/sandbox_policy"
7
+ require_relative "execution/sandbox"
6
8
 
7
9
  module Saboteur
8
10
  # Reports the current environment, never mutations. The two unchanged suite
@@ -25,8 +27,10 @@ module Saboteur
25
27
  integration = boot_target(hooks)
26
28
  @out.puts "Integration: #{@config['integration']} (loaded)"
27
29
  @out.puts "Rails: #{@config['rails'] ? 'enabled' : 'disabled'}"
30
+ check_database_hosts
28
31
  @out.puts "SimpleCov: #{defined?(::SimpleCov) ? 'present; start suppressed for this process' : 'not loaded'}"
29
- check_coverage
32
+ verify_coverage
33
+ check_sandbox
30
34
  check_suite(integration, hooks)
31
35
  true
32
36
  end
@@ -62,15 +66,51 @@ module Saboteur
62
66
  "doctor supports the rspec integration; set integration: rspec"
63
67
  end
64
68
 
69
+ # Printed once; verified twice. Boot must not wake coverage up, so the
70
+ # post-boot check re-reads the state without printing a second line.
65
71
  def check_coverage
66
72
  state = Coverage.state
67
73
  @out.puts "Coverage: #{state}"
74
+ verify_coverage(state)
75
+ end
76
+
77
+ def verify_coverage(state = Coverage.state)
68
78
  return if Coverage.supported?(:lines) && Coverage.supported?(:methods) && state == :suspended
69
79
 
70
80
  raise Error, "Coverage must be suspended with line and method support before boot; use the saboteur " \
71
81
  "executable, disable SimpleCov and remove other Coverage.start calls"
72
82
  end
73
83
 
84
+ # A kernel capability probe, not verified isolation: what the kernel
85
+ # would let mutant children do, and what the run does with that - the
86
+ # engine's sandbox policy owns the refusal or degradation itself.
87
+ def check_sandbox
88
+ if @config["sandbox"] == false
89
+ @out.puts "Sandbox: off (c.sandbox = false)"
90
+ return
91
+ end
92
+ abi = sandbox_abi
93
+ if abi >= 3
94
+ @out.puts "Sandbox: kernel Landlock ABI #{abi} - capability probe only, not verified isolation"
95
+ elsif @config["sandbox"] == "strict"
96
+ @out.puts "Sandbox: #{Engine::SandboxPolicy.sandbox_gap(abi)}; " \
97
+ 'c.sandbox = "strict" refuses to run - children never run degraded'
98
+ else
99
+ @out.puts "Sandbox: #{Engine::SandboxPolicy.sandbox_gap(abi)}; " \
100
+ "children would run with the hazard guard and canary only"
101
+ end
102
+ end
103
+
104
+ # The probe contract the engine's policy already keeps: an errored probe
105
+ # (EPERM under a seccomp policy, ...) is an actionable tool error, not a
106
+ # CLI internal error - doctor stays read-only.
107
+ def sandbox_abi
108
+ Execution::Sandbox.abi_version
109
+ rescue SystemCallError => e
110
+ raise Error, "sandbox is on but the Landlock probe failed (#{e.message}); a run would refuse the same way - " \
111
+ "set c.sandbox = false to check without the sandbox"
112
+ end
113
+
74
114
  # Match against all included source definitions, not the current diff or
75
115
  # selected tests. A limited run must not make a valid ignore look stale.
76
116
  def check_ignore_subjects
@@ -105,11 +145,33 @@ module Saboteur
105
145
  @out.puts "Custom map: #{map.entries.size} entries (loaded)"
106
146
  end
107
147
 
148
+ # Prints the test database hosts the booted app reports, so a non-local
149
+ # refusal names the host here instead of only at target boot (issue #258).
150
+ def check_database_hosts
151
+ return unless @config["rails"]
152
+ return @out.puts("Databases: ActiveRecord did not load; host check skipped") unless defined?(::ActiveRecord::Base)
153
+
154
+ configs = ::ActiveRecord::Base.configurations.configs_for(env_name: "test")
155
+ configs.each do |db|
156
+ host = db.configuration_hash[:host]
157
+ @out.puts "Database: #{db.name} (host #{host.to_s.empty? ? 'local socket' : host})"
158
+ end
159
+ remote = Frameworks::Rails.remote_databases(configs, @config["allow_db_hosts"])
160
+ return if remote.empty?
161
+
162
+ raise ConfigError, "non-local test databases: #{remote.join(', ')}; if they are disposable test databases, " \
163
+ "allow them with c.allow_db_hosts in #{Config::FILE}"
164
+ end
165
+
108
166
  def check_boot_files
109
167
  %w[require hooks].each do |key|
110
168
  @config[key].each do |file|
111
169
  path = File.expand_path(file, @config.root)
112
- raise ConfigError, "#{key} file missing: #{file}; fix config/saboteur.rb" unless File.file?(path)
170
+ # Resolve the way Kernel#require does: the path as given, or with
171
+ # .rb appended (the Rails quickstart sets ./config/environment, issue #257).
172
+ unless File.file?(path) || File.file?("#{path}.rb")
173
+ raise ConfigError, "#{key} file missing: #{file}; fix config/saboteur.rb"
174
+ end
113
175
 
114
176
  @out.puts "#{key}: #{file} (found)"
115
177
  end
@@ -3,6 +3,7 @@
3
3
 
4
4
  require "json"
5
5
  require "fileutils"
6
+ require_relative "../text"
6
7
 
7
8
  module Saboteur
8
9
  class Engine
@@ -10,16 +11,28 @@ module Saboteur
10
11
  # or mutation source is used. This catches observed instability, not all
11
12
  # possible flaky behaviour.
12
13
  class BaselineGuard
14
+ # How many unchanged-suite checks actually completed and validated:
15
+ # the number of order comparisons a run can claim is one less.
16
+ attr_reader :completed_passes
17
+
13
18
  EXCERPT_LINES = 10
14
19
  EXCERPT_WIDTH = 200
15
20
  EXCERPT_SCAN_LINES = 5000
16
21
 
17
- def initialize(config:, integration:, hooks:, supervisor:, diagnostics:)
22
+ # full_passes: every full-suite pass the run will make, including the
23
+ # coverage build when the cache is cold, so the first timed pass can
24
+ # price the rest (#260).
25
+ def initialize(config:, integration:, hooks:, supervisor:, diagnostics:, full_passes: 2, between_passes: nil,
26
+ timeout_cap: nil)
18
27
  @config = config
19
28
  @integration = integration
20
29
  @hooks = hooks
21
30
  @supervisor = supervisor
22
31
  @diagnostics = diagnostics
32
+ @full_passes = full_passes
33
+ @between_passes = between_passes
34
+ @timeout_cap = timeout_cap
35
+ @completed_passes = 0
23
36
  end
24
37
 
25
38
  def call
@@ -27,11 +40,16 @@ module Saboteur
27
40
  raise Error, "baseline has no tests. Check rspec_args and the suite filters before running saboteur again." if
28
41
  expected.empty?
29
42
 
30
- @diagnostics.inform("checking the unchanged suite twice (seed #{@config['seed']})")
43
+ passes = @config["baseline_passes"]
44
+ @diagnostics.inform(announce(passes))
31
45
  previous = nil
32
- 2.times do |index|
46
+ passes.times do |index|
47
+ break if budget_spent?(index)
48
+
33
49
  number = index + 1
50
+ started = Process.clock_gettime(Process::CLOCK_MONOTONIC)
34
51
  data = check(number)
52
+ announce_first_pass_cost(number, started)
35
53
  unless data["passed"] && data["failed"].empty?
36
54
  refuse(number,
37
55
  "the suite failed: #{Array(data['failed']).join(', ')}")
@@ -41,18 +59,73 @@ module Saboteur
41
59
  executed.sort == expected.sort
42
60
  refuse(number, "test order changed between baseline checks") if previous && executed != previous
43
61
  previous = executed
62
+ @completed_passes += 1
44
63
  end
45
64
  nil
46
65
  end
47
66
 
48
67
  private
49
68
 
69
+ # A spent --max-duration budget skips the remaining passes - including
70
+ # the first, when boot alone already spent it (#245).
71
+ def budget_spent?(_index)
72
+ !@between_passes.nil? && @between_passes.call
73
+ end
74
+
75
+ def announce(passes)
76
+ case passes
77
+ when 1
78
+ "checking the unchanged suite once (seed #{@config['seed']}); " \
79
+ "order stability between passes is not checked (baseline_passes: 1)"
80
+ when 2
81
+ "checking the unchanged suite twice (seed #{@config['seed']})"
82
+ else
83
+ "checking the unchanged suite #{passes} times (seed #{@config['seed']})"
84
+ end
85
+ end
86
+
87
+ # Once the first pass is timed, the total cost stops being a guess:
88
+ # the remaining full passes cost roughly the same each.
89
+ def announce_first_pass_cost(number, started)
90
+ return unless number == 1
91
+
92
+ remaining = @full_passes - number
93
+ return if remaining.zero?
94
+
95
+ elapsed = (Process.clock_gettime(Process::CLOCK_MONOTONIC) - started).round
96
+ @diagnostics.inform("baseline check #{number} took #{elapsed} s; about #{elapsed * remaining} s left for " \
97
+ "the remaining #{remaining} full #{remaining == 1 ? 'pass' : 'passes'} at that rate")
98
+ end
99
+
100
+ # An admitted pass still gets only the time left in the run budget:
101
+ # without the cap it would overrun a spent --max-duration by its
102
+ # whole coverage_timeout.
103
+ def phase_timeout
104
+ @timeout_cap ? @timeout_cap.call(@config["coverage_timeout"]) : @config["coverage_timeout"]
105
+ end
106
+
107
+ # A pass that hit its deadline only because --max-duration shortened
108
+ # the admission below the configured phase timeout: the budget ended
109
+ # the run, the suite is not broken (#296).
110
+ def budget_stopped?(result, admitted)
111
+ result.stopped_by == :timeout && admitted < @config["coverage_timeout"]
112
+ end
113
+
114
+ def refuse_budget_stop(number, admitted)
115
+ raise BudgetPhaseTimeout, "the --max-duration budget stopped baseline check #{number} " \
116
+ "(admitted #{admitted.round(1)}s of the #{@config['coverage_timeout']}s " \
117
+ "phase timeout)"
118
+ end
119
+
50
120
  def check(number)
51
121
  Execution::ForkReadiness.prepare_heap
52
122
  heartbeat = @diagnostics.progress_heartbeat("baseline check #{number}").watch(log_path(number))
53
- result = @supervisor.run(timeout: @config["coverage_timeout"], on_tick: heartbeat.method(:tick)) do |writer|
123
+ admitted = phase_timeout
124
+ result = @supervisor.run(timeout: admitted, on_tick: heartbeat.method(:tick),
125
+ label: "baseline child died during setup/run") do |writer|
54
126
  record(writer, number)
55
127
  end
128
+ refuse_budget_stop(number, admitted) if budget_stopped?(result, admitted)
56
129
  refuse(number, "the suite stopped: #{result.stopped_by} (#{result.detail})") if result.stopped_by
57
130
  # Before the payload checks: a child killed by a signal (the OOM
58
131
  # killer, say) writes nothing, and the signal is the explanation. A
@@ -129,7 +202,8 @@ module Saboteur
129
202
  end
130
203
 
131
204
  def log_lines(number)
132
- File.foreach(log_path(number), chomp: true).first(EXCERPT_SCAN_LINES).reject { |line| line.strip.empty? }
205
+ File.foreach(log_path(number), chomp: true).first(EXCERPT_SCAN_LINES)
206
+ .map { |line| Text.utf8(line) }.reject { |line| line.strip.empty? }
133
207
  rescue SystemCallError
134
208
  []
135
209
  end
@@ -83,10 +83,21 @@ module Saboteur
83
83
 
84
84
  def load_hooks
85
85
  Saboteur.current_hooks = @hooks
86
- @config["hooks"].each { |file| load File.expand_path(file, @root) }
86
+ @config["hooks"].each { |file| load hook_path(file) }
87
87
  ensure
88
88
  Saboteur.current_hooks = nil
89
89
  end
90
+
91
+ # Kernel#load does not append .rb the way Kernel#require does, and
92
+ # doctor accepts the extensionless form (#257), so resolve it the same
93
+ # way here: the exact path first, then path.rb.
94
+ def hook_path(file)
95
+ path = File.expand_path(file, @root)
96
+ rb = "#{path}.rb"
97
+ return rb if !File.exist?(path) && File.extname(path).empty? && File.file?(rb)
98
+
99
+ path
100
+ end
90
101
  end
91
102
  end
92
103
  end