saboteur 0.6.0 → 0.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +1024 -0
- data/README.md +199 -952
- data/exe/saboteur +34 -7
- data/lib/saboteur/baseline/workflow.rb +2 -1
- data/lib/saboteur/baseline_health.rb +40 -6
- data/lib/saboteur/cli/exit_status.rb +80 -4
- data/lib/saboteur/cli/help_options.rb +17 -0
- data/lib/saboteur/cli/history_command.rb +135 -0
- data/lib/saboteur/cli/presets.rb +40 -0
- data/lib/saboteur/cli/run_budget_options.rb +37 -0
- data/lib/saboteur/cli/run_reporting.rb +74 -5
- data/lib/saboteur/cli/selection_options.rb +31 -0
- data/lib/saboteur/cli/shard_options.rb +17 -0
- data/lib/saboteur/cli/source_set.rb +53 -0
- data/lib/saboteur/cli/survivors_command.rb +157 -0
- data/lib/saboteur/cli.rb +136 -98
- data/lib/saboteur/config.rb +163 -15
- data/lib/saboteur/config_file.rb +8 -0
- data/lib/saboteur/constant_reads.rb +182 -0
- data/lib/saboteur/constant_subject.rb +70 -0
- data/lib/saboteur/constant_subject_finder.rb +281 -0
- data/lib/saboteur/coverage_reads.rb +29 -0
- data/lib/saboteur/crash_report.rb +154 -0
- data/lib/saboteur/custom_map.rb +3 -1
- data/lib/saboteur/dirty_tree_warning.rb +128 -0
- data/lib/saboteur/doctor.rb +49 -1
- data/lib/saboteur/engine/baseline_guard.rb +136 -14
- data/lib/saboteur/engine/boot.rb +16 -1
- data/lib/saboteur/engine/canary.rb +41 -10
- data/lib/saboteur/engine/candidate_selection.rb +28 -1
- data/lib/saboteur/engine/candidates.rb +43 -3
- data/lib/saboteur/engine/coverage_guard.rb +29 -0
- data/lib/saboteur/engine/coverage_store.rb +79 -15
- data/lib/saboteur/engine/defined_order_prefix.rb +43 -0
- data/lib/saboteur/engine/diagnostics.rb +41 -2
- data/lib/saboteur/engine/diff_scope.rb +24 -5
- data/lib/saboteur/engine/distribution.rb +134 -8
- data/lib/saboteur/engine/early_stop_recording.rb +47 -0
- data/lib/saboteur/engine/gated_run.rb +35 -0
- data/lib/saboteur/engine/load_coverage.rb +45 -0
- data/lib/saboteur/engine/out_of_scope_guard.rb +21 -0
- data/lib/saboteur/engine/pipelines.rb +107 -0
- data/lib/saboteur/engine/result_cache.rb +67 -9
- data/lib/saboteur/engine/run_budget.rb +90 -0
- data/lib/saboteur/engine/run_fingerprint.rb +4 -0
- data/lib/saboteur/engine/sandbox_policy.rb +56 -11
- data/lib/saboteur/engine/scheduler.rb +222 -20
- data/lib/saboteur/engine/selection_announcement.rb +54 -0
- data/lib/saboteur/engine/shard.rb +69 -0
- data/lib/saboteur/engine/subject_selection.rb +373 -19
- data/lib/saboteur/engine/suite_files.rb +3 -1
- data/lib/saboteur/engine/test_guess.rb +438 -25
- data/lib/saboteur/engine/thread_warnings.rb +53 -0
- data/lib/saboteur/engine/unchanged_syntax.rb +17 -2
- data/lib/saboteur/engine/unscoped_suite_warning.rb +41 -0
- data/lib/saboteur/engine/verdict_check.rb +54 -18
- data/lib/saboteur/engine.rb +202 -158
- data/lib/saboteur/execution/external_runner.rb +471 -0
- data/lib/saboteur/execution/marker_sweep.rb +79 -0
- data/lib/saboteur/execution/peak_memory.rb +28 -0
- data/lib/saboteur/execution/process_table.rb +12 -0
- data/lib/saboteur/execution/process_tree.rb +65 -5
- data/lib/saboteur/execution/protocol.rb +2 -1
- data/lib/saboteur/execution/reevaluator.rb +68 -12
- data/lib/saboteur/execution/runner.rb +225 -24
- data/lib/saboteur/execution/sandbox.rb +21 -3
- data/lib/saboteur/execution/seatbelt.rb +226 -0
- data/lib/saboteur/execution/supervisor.rb +45 -13
- data/lib/saboteur/execution/worker_pool.rb +110 -8
- data/lib/saboteur/frameworks/sqlite_memory_database.rb +49 -0
- data/lib/saboteur/gates.rb +43 -2
- data/lib/saboteur/help.rb +108 -13
- data/lib/saboteur/ignore_list.rb +40 -7
- data/lib/saboteur/install.rb +4 -3
- data/lib/saboteur/integration/base.rb +7 -2
- data/lib/saboteur/integration/rspec.rb +175 -9
- data/lib/saboteur/mutation_source/argument_removal.rb +1 -1
- data/lib/saboteur/mutation_source/builtin.rb +44 -14
- data/lib/saboteur/mutation_source/chain_removal.rb +1 -1
- data/lib/saboteur/mutation_source/condition_negation.rb +1 -1
- data/lib/saboteur/mutation_source/constant_literals.rb +66 -0
- data/lib/saboteur/mutation_source/element_removal.rb +73 -0
- data/lib/saboteur/mutation_source/endpoint_replacement.rb +1 -1
- data/lib/saboteur/mutation_source/entry_edits.rb +44 -0
- data/lib/saboteur/mutation_source/filter_replacement.rb +1 -1
- data/lib/saboteur/mutation_source/hazard_guard.rb +129 -3
- data/lib/saboteur/mutation_source/interpolation_removal.rb +41 -0
- data/lib/saboteur/mutation_source/literal_replacement.rb +1 -1
- data/lib/saboteur/mutation_source/method_swap.rb +193 -0
- data/lib/saboteur/mutation_source/mutation.rb +5 -3
- data/lib/saboteur/mutation_source/operand_replacement.rb +54 -0
- data/lib/saboteur/mutation_source/operator_assignment.rb +1 -1
- data/lib/saboteur/mutation_source/operator_examples.rb +6 -1
- data/lib/saboteur/mutation_source/operator_registry.rb +7 -2
- data/lib/saboteur/mutation_source/operator_visit.rb +39 -0
- data/lib/saboteur/mutation_source/pair_removal.rb +13 -31
- data/lib/saboteur/mutation_source/range_replacement.rb +1 -1
- data/lib/saboteur/mutation_source/regex_replacement.rb +93 -0
- data/lib/saboteur/preview.rb +122 -13
- data/lib/saboteur/refinement_activations.rb +60 -0
- data/lib/saboteur/reporter/agent.rb +9 -1
- data/lib/saboteur/reporter/agent_commands.rb +47 -15
- data/lib/saboteur/reporter/agent_incomplete.rb +38 -2
- data/lib/saboteur/reporter/agent_summary.rb +4 -3
- data/lib/saboteur/reporter/findings.rb +5 -2
- data/lib/saboteur/reporter/history_policy.rb +11 -0
- data/lib/saboteur/reporter/history_reader.rb +47 -0
- data/lib/saboteur/reporter/html.rb +26 -4
- data/lib/saboteur/reporter/mutation_text.rb +6 -0
- data/lib/saboteur/reporter/next_steps.rb +14 -4
- data/lib/saboteur/reporter/score_history.rb +5 -1
- data/lib/saboteur/reporter/stryker.rb +40 -7
- data/lib/saboteur/reporter/survivor_classification.rb +101 -0
- data/lib/saboteur/reporter/survivor_hints.rb +3 -0
- data/lib/saboteur/reporter/survivor_list.rb +15 -9
- data/lib/saboteur/reporter/terminal.rb +74 -17
- data/lib/saboteur/result.rb +38 -7
- data/lib/saboteur/selection/coverage_map.rb +107 -11
- data/lib/saboteur/selection/dependency_shield.rb +80 -0
- data/lib/saboteur/selection/external_test_map.rb +27 -0
- data/lib/saboteur/selection/git_diff.rb +105 -13
- data/lib/saboteur/selection/layout_only_lines.rb +80 -0
- data/lib/saboteur/selection/recorder.rb +22 -4
- data/lib/saboteur/selection/source_scope.rb +22 -0
- data/lib/saboteur/selection/untracked_changes.rb +126 -0
- data/lib/saboteur/subject.rb +53 -7
- data/lib/saboteur/subject_finder.rb +173 -29
- data/lib/saboteur/text.rb +42 -0
- data/lib/saboteur/version.rb +1 -1
- data/lib/saboteur/working_tree.rb +38 -15
- data/lib/saboteur.rb +29 -1
- metadata +50 -2
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
# SPDX-License-Identifier: MIT
|
|
2
|
+
# frozen_string_literal: true
|
|
3
|
+
|
|
4
|
+
module Saboteur
|
|
5
|
+
class Engine
|
|
6
|
+
# The --since stop whose changed source files all sit outside the
|
|
7
|
+
# include scope (#483), kept apart from the guess flow it guards.
|
|
8
|
+
module OutOfScopeGuard
|
|
9
|
+
private
|
|
10
|
+
|
|
11
|
+
# Every changed source file sits outside the include scope (#483): the
|
|
12
|
+
# whole-suite fallback would run the slowest possible suite against
|
|
13
|
+
# zero subjects, so stop and name what the scope hid.
|
|
14
|
+
def refuse_out_of_scope_changes(guess)
|
|
15
|
+
return unless guess.changed.zero? && guess.out_of_scope.any?
|
|
16
|
+
|
|
17
|
+
raise ConfigError, guess.out_of_scope_message
|
|
18
|
+
end
|
|
19
|
+
end
|
|
20
|
+
end
|
|
21
|
+
end
|
|
@@ -0,0 +1,107 @@
|
|
|
1
|
+
# SPDX-License-Identifier: MIT
|
|
2
|
+
# frozen_string_literal: true
|
|
3
|
+
|
|
4
|
+
module Saboteur
|
|
5
|
+
class Engine
|
|
6
|
+
# The two run pipelines: in-process (boot, coverage, workers) and the
|
|
7
|
+
# --test-command external mode (#140).
|
|
8
|
+
module Pipelines
|
|
9
|
+
# The in-process pipeline: boot the target, mutate under coverage, and
|
|
10
|
+
# clean the workers up however the run ends.
|
|
11
|
+
def run_pipeline(ignore_list, started)
|
|
12
|
+
return call_external(ignore_list, started) if external_command?
|
|
13
|
+
|
|
14
|
+
boot = Boot.new(config, hooks)
|
|
15
|
+
begin
|
|
16
|
+
integration, files, load_lines, load_methods = boot_target(boot)
|
|
17
|
+
@rails = boot.rails
|
|
18
|
+
subjects, scope = select_subjects(files, test_files: integration.test_files)
|
|
19
|
+
results = if subjects.empty?
|
|
20
|
+
[]
|
|
21
|
+
else
|
|
22
|
+
mutate(subjects, files, integration, scope, ignore_list,
|
|
23
|
+
load_lines:, load_methods:)
|
|
24
|
+
end
|
|
25
|
+
@diagnostics.warnings.concat(ignore_list.apply(results, unscoped_digests: @unscoped_digests))
|
|
26
|
+
build_report(results, subjects, scope, started)
|
|
27
|
+
ensure
|
|
28
|
+
# Cleanup runs however the call ends - a partway boot failure, no
|
|
29
|
+
# subjects, no runnable work - so a host embedding Engine never
|
|
30
|
+
# holds worker copies until process exit. Idempotent; the Rails
|
|
31
|
+
# profile's at_exit handler stays the fallback for abrupt exit.
|
|
32
|
+
begin
|
|
33
|
+
@coverage_guard&.resume # an embedded host gets its coverage back (#254)
|
|
34
|
+
ensure
|
|
35
|
+
safely_cleanup_workers(boot.rails)
|
|
36
|
+
end
|
|
37
|
+
end
|
|
38
|
+
end
|
|
39
|
+
|
|
40
|
+
# The --test-command pipeline (#140): no boot, no coverage, no
|
|
41
|
+
# baseline - every mutant runs the command on the mutated tree,
|
|
42
|
+
# serially, with no sandbox. The command is arbitrary user code.
|
|
43
|
+
def call_external(ignore_list, started)
|
|
44
|
+
@diagnostics.inform("external test command: #{config['test_command']} " \
|
|
45
|
+
"(serial, no coverage, unsandboxed, mutants applied in a disposable copy)")
|
|
46
|
+
files = config.included_files
|
|
47
|
+
subjects, scope = select_subjects(files)
|
|
48
|
+
results = subjects.empty? ? [] : mutate_external(subjects, scope, ignore_list)
|
|
49
|
+
@diagnostics.warnings.concat(ignore_list.apply(results, unscoped_digests: @unscoped_digests))
|
|
50
|
+
build_report(results, subjects, scope, started)
|
|
51
|
+
end
|
|
52
|
+
|
|
53
|
+
def mutate_external(subjects, scope, ignore_list)
|
|
54
|
+
runner = external_runner
|
|
55
|
+
# One neutral run before any mutant exists: a command that fails on
|
|
56
|
+
# the unchanged tree can never claim kills (#305). It answers to the
|
|
57
|
+
# run budget with an ABSOLUTE deadline, admitted after the tree copy:
|
|
58
|
+
# a budget already spent is a budget stop, not a command error.
|
|
59
|
+
begin
|
|
60
|
+
runner.preflight!(deadline: preflight_deadline, budget_deadline: budget_deadline)
|
|
61
|
+
rescue BudgetPhaseTimeout => e
|
|
62
|
+
return budget_phase_stop(e, subjects)
|
|
63
|
+
end
|
|
64
|
+
map = Selection::ExternalTestMap.new
|
|
65
|
+
scheduler = Scheduler.new(config:, runner:, map:,
|
|
66
|
+
reevaluator: Execution::Reevaluator.new(config["reeval"],
|
|
67
|
+
whole_file: config["whole_file"]),
|
|
68
|
+
diagnostics: @diagnostics, deadline: budget_deadline, verify_loaded: false)
|
|
69
|
+
lists = candidate_lists(subjects, scope, map)
|
|
70
|
+
work = subjects.zip(lists).reject { |_, list| list.empty? }
|
|
71
|
+
return [] if work.empty?
|
|
72
|
+
|
|
73
|
+
runnable_subjects, runnable_lists = work.transpose
|
|
74
|
+
begin
|
|
75
|
+
warn_unless_mutants_visible(runner, runnable_subjects.first)
|
|
76
|
+
rescue BudgetPhaseTimeout => e
|
|
77
|
+
return budget_phase_stop(e, subjects)
|
|
78
|
+
end
|
|
79
|
+
run_candidates(runnable_subjects, runnable_lists, scheduler, nil, ignore_list)
|
|
80
|
+
ensure
|
|
81
|
+
runner.close
|
|
82
|
+
end
|
|
83
|
+
|
|
84
|
+
# Warn loudly when the command cannot see mutants at all: an
|
|
85
|
+
# anchored or never-loaded file reads as 100 % survivors (#441).
|
|
86
|
+
def warn_unless_mutants_visible(runner, subject)
|
|
87
|
+
warning = runner.visibility_check(subject: subject, deadline: preflight_deadline,
|
|
88
|
+
budget_deadline: budget_deadline)
|
|
89
|
+
@diagnostics.warn(warning) if warning
|
|
90
|
+
end
|
|
91
|
+
|
|
92
|
+
def external_command? = !config["test_command"].to_s.empty?
|
|
93
|
+
|
|
94
|
+
# The preflight's absolute monotonic deadline: its own
|
|
95
|
+
# baseline_timeout slice, or the run budget's end, whichever is
|
|
96
|
+
# sooner. Absolute, so the tree copy before admission cannot eat the
|
|
97
|
+
# budget while the command still gets a full timeout (#305 Architect).
|
|
98
|
+
def preflight_deadline
|
|
99
|
+
own = Process.clock_gettime(Process::CLOCK_MONOTONIC) + config["baseline_timeout"]
|
|
100
|
+
deadline = budget_deadline
|
|
101
|
+
deadline ? [own, deadline].min : own
|
|
102
|
+
end
|
|
103
|
+
|
|
104
|
+
def external_runner = Execution::ExternalRunner.new(command: config["test_command"], root: @root)
|
|
105
|
+
end
|
|
106
|
+
end
|
|
107
|
+
end
|
|
@@ -4,6 +4,7 @@
|
|
|
4
4
|
require "digest"
|
|
5
5
|
require "fileutils"
|
|
6
6
|
require "json"
|
|
7
|
+
require_relative "../text"
|
|
7
8
|
|
|
8
9
|
module Saboteur
|
|
9
10
|
class Engine
|
|
@@ -14,25 +15,36 @@ module Saboteur
|
|
|
14
15
|
# same: the method's source, the whole file the method lives in (so a
|
|
15
16
|
# constant or helper changed elsewhere in that file is seen), the mutant,
|
|
16
17
|
# the set of covering tests, the content of those tests' files, the
|
|
17
|
-
# lockfile, the run's inputs (config, saboteur version)
|
|
18
|
-
#
|
|
18
|
+
# lockfile, the run's inputs (config, saboteur version), and every source
|
|
19
|
+
# file a covering test touched (a method it calls, a constant defined
|
|
20
|
+
# elsewhere - #197). Only killed, survived and timeout are stored;
|
|
19
21
|
# errors and skips always run again.
|
|
20
22
|
#
|
|
21
|
-
#
|
|
22
|
-
#
|
|
23
|
-
#
|
|
23
|
+
# When the dependency set is unknown - no coverage map, or an empty one -
|
|
24
|
+
# every source file is watched instead: reuse turns conservative, never
|
|
25
|
+
# stale.
|
|
24
26
|
class ResultCache
|
|
25
27
|
FORMAT = 1
|
|
26
28
|
STORED = %i[killed survived timeout].freeze
|
|
27
29
|
LOCKFILES = %w[Gemfile.lock gems.locked].freeze
|
|
30
|
+
# Dependency declarations themselves, so the no-lockfile library
|
|
31
|
+
# shape (bundler's own convention: libraries gitignore the lockfile)
|
|
32
|
+
# still invalidates on a dependency edit (#408).
|
|
33
|
+
MANIFESTS = %w[Gemfile gems.rb].freeze
|
|
34
|
+
|
|
35
|
+
SOURCE_EXCLUDES = %w[.git vendor node_modules].freeze
|
|
28
36
|
|
|
29
37
|
# suite_digest: SuiteFiles#digest, so an edit to a spec helper or
|
|
30
38
|
# support file invalidates every verdict.
|
|
31
|
-
def initialize(path:, root:, inputs:, suite_digest: "")
|
|
39
|
+
def initialize(path:, root:, inputs:, suite_digest: "", coverage: nil, fallback_files: [])
|
|
32
40
|
@path = path
|
|
33
41
|
@root = root
|
|
34
42
|
@inputs = inputs
|
|
35
43
|
@suite_digest = suite_digest
|
|
44
|
+
# Engine always passes the built map; a missing one behaves as an
|
|
45
|
+
# empty one, which watches every fallback and child-loaded source (#197).
|
|
46
|
+
@coverage = coverage || Selection::CoverageMap.new(lines: {}, tests: [])
|
|
47
|
+
@fallback_files = fallback_files.map { |file| File.expand_path(file, root) }.sort
|
|
36
48
|
@entries = load
|
|
37
49
|
@file_digests = {}
|
|
38
50
|
end
|
|
@@ -64,8 +76,18 @@ module Saboteur
|
|
|
64
76
|
|
|
65
77
|
private
|
|
66
78
|
|
|
79
|
+
# The current project source files, relative and sorted. A file added
|
|
80
|
+
# after the map was built changes the set and invalidates every stored
|
|
81
|
+
# verdict, whether or not any watched file references it yet.
|
|
82
|
+
def source_set
|
|
83
|
+
@source_set ||= Dir.glob("**/*.rb", base: @root)
|
|
84
|
+
.reject { |rel| SOURCE_EXCLUDES.include?(rel.split("/").first) }
|
|
85
|
+
.sort
|
|
86
|
+
.join("\0")
|
|
87
|
+
end
|
|
88
|
+
|
|
67
89
|
def load
|
|
68
|
-
data = JSON.parse(File.read(@path))
|
|
90
|
+
data = JSON.parse(Text.utf8_strict(File.read(@path)))
|
|
69
91
|
data["format"] == FORMAT ? data["entries"] : {}
|
|
70
92
|
rescue SystemCallError, JSON::ParserError
|
|
71
93
|
{}
|
|
@@ -77,12 +99,33 @@ module Saboteur
|
|
|
77
99
|
|
|
78
100
|
def fingerprint(subject, tests)
|
|
79
101
|
digest = Digest::SHA256.new
|
|
80
|
-
digest << VERSION
|
|
102
|
+
digest << [VERSION, @inputs, @suite_digest, source_set, subject.source].join("\0") << "\0"
|
|
81
103
|
# A hex digest has a fixed width, so it needs no separator after it.
|
|
82
104
|
digest << Digest::SHA256.hexdigest(subject.file_source)
|
|
105
|
+
digest_tests(digest, tests)
|
|
106
|
+
hash_dependency_inputs(digest)
|
|
107
|
+
digest.hexdigest
|
|
108
|
+
end
|
|
109
|
+
|
|
110
|
+
def digest_tests(digest, tests)
|
|
83
111
|
tests.sort.each { |test| digest << test << "\0" << file_digest(test_file(test)) << "\0" }
|
|
112
|
+
dependencies(tests).each { |dep| digest << dep << "\0" << file_digest(dep) << "\0" }
|
|
84
113
|
LOCKFILES.each { |name| digest << file_digest(File.join(@root, name)) << "\0" }
|
|
85
|
-
|
|
114
|
+
end
|
|
115
|
+
|
|
116
|
+
# Files a change in which can flip the verdict (#197): every source
|
|
117
|
+
# file a covering test touched, from the run's coverage map, plus the
|
|
118
|
+
# loaded sources the map never observed (#344, #353).
|
|
119
|
+
def dependencies(tests)
|
|
120
|
+
(@coverage.files_for(tests) | @coverage.boot_files | unobserved_files).sort
|
|
121
|
+
end
|
|
122
|
+
|
|
123
|
+
# Loaded sources the map never observed (no line, method, or load
|
|
124
|
+
# evidence): coverage cannot say whether they matter, so a change in
|
|
125
|
+
# any of them must invalidate (#344).
|
|
126
|
+
def unobserved_files
|
|
127
|
+
@unobserved_files ||=
|
|
128
|
+
(@fallback_files | @coverage.loaded_files).select { |file| @coverage.empty_for?([file]) }
|
|
86
129
|
end
|
|
87
130
|
|
|
88
131
|
# RSpec ids name their file: "./spec/a_spec.rb[1:2]".
|
|
@@ -90,6 +133,21 @@ module Saboteur
|
|
|
90
133
|
File.expand_path(test[/\A(.+?\.rb)(?:\[|:|\z)/, 1] || test, @root)
|
|
91
134
|
end
|
|
92
135
|
|
|
136
|
+
def hash_dependency_inputs(digest)
|
|
137
|
+
dependency_files.each { |path| digest << path.delete_prefix("#{@root}/") << "\0" << file_digest(path) << "\0" }
|
|
138
|
+
end
|
|
139
|
+
|
|
140
|
+
# Manifests and gemspecs anywhere below the root, since a monorepo
|
|
141
|
+
# sub-gem carries its own dependency declarations (#561); keyed
|
|
142
|
+
# root-relative so same-named sub-gem files cannot collide. Vendored
|
|
143
|
+
# installs and dependencies' own checkouts hold gemspecs the project
|
|
144
|
+
# does not declare, so the scan mirrors RunFingerprint's exclusions.
|
|
145
|
+
def dependency_files
|
|
146
|
+
@dependency_files ||= Dir[File.join(@root, "**", "{#{MANIFESTS.join(',')},*.gemspec}")].reject do |path|
|
|
147
|
+
path.delete_prefix("#{@root}/").split("/").any? { |part| RunFingerprint::EXCLUDED_DIRECTORIES.include?(part) }
|
|
148
|
+
end
|
|
149
|
+
end
|
|
150
|
+
|
|
93
151
|
def file_digest(path)
|
|
94
152
|
@file_digests[path] ||= File.file?(path) ? Digest::SHA256.file(path).hexdigest : "missing"
|
|
95
153
|
end
|
|
@@ -0,0 +1,90 @@
|
|
|
1
|
+
# SPDX-License-Identifier: MIT
|
|
2
|
+
# frozen_string_literal: true
|
|
3
|
+
|
|
4
|
+
module Saboteur
|
|
5
|
+
class Engine
|
|
6
|
+
# The --max-duration wall-clock budget (#245). The clock anchors at
|
|
7
|
+
# engine entry, so boot, baselines and the coverage map spend it too;
|
|
8
|
+
# schedulers get an absolute deadline so a spent budget also stops a
|
|
9
|
+
# long subject between mutants.
|
|
10
|
+
module RunBudget
|
|
11
|
+
private
|
|
12
|
+
|
|
13
|
+
def start_budget_clock(started)
|
|
14
|
+
@run_started = started
|
|
15
|
+
end
|
|
16
|
+
|
|
17
|
+
# The whole run is budgeted, baselines included: an oversized
|
|
18
|
+
# baseline spends the budget by itself.
|
|
19
|
+
def budget_exceeded?
|
|
20
|
+
return false unless config["max_duration"]
|
|
21
|
+
|
|
22
|
+
exceeded = Process.clock_gettime(Process::CLOCK_MONOTONIC) - @run_started >= config["max_duration"]
|
|
23
|
+
announce_budget_once if exceeded
|
|
24
|
+
exceeded
|
|
25
|
+
end
|
|
26
|
+
|
|
27
|
+
# A phase admitted inside the budget still gets only the time left
|
|
28
|
+
# in it: baseline passes and coverage builds cap at the deadline.
|
|
29
|
+
def budget_capped_timeout(timeout)
|
|
30
|
+
deadline = budget_deadline
|
|
31
|
+
return timeout unless deadline
|
|
32
|
+
|
|
33
|
+
(deadline - Process.clock_gettime(Process::CLOCK_MONOTONIC)).clamp(0.0, timeout)
|
|
34
|
+
end
|
|
35
|
+
|
|
36
|
+
# Absolute monotonic deadline shared with the schedulers (forked
|
|
37
|
+
# workers inherit it): a subject's remaining mutants stop once it
|
|
38
|
+
# passes, so the overshoot after a spent budget stays under one
|
|
39
|
+
# mutant deadline.
|
|
40
|
+
def budget_deadline
|
|
41
|
+
config["max_duration"] && (@run_started + config["max_duration"])
|
|
42
|
+
end
|
|
43
|
+
|
|
44
|
+
# Once the baselines are timed, a cold coverage build is the last
|
|
45
|
+
# unpriced phase before the mutants: it costs about one more full pass
|
|
46
|
+
# and keeps nothing when the budget cuts it off, so a resume starts it
|
|
47
|
+
# from zero again. Refuse up front when the remaining budget cannot
|
|
48
|
+
# cover that pass, naming the shortfall and the remedies (#427).
|
|
49
|
+
def refuse_unaffordable_coverage_build(store, files, baseline_elapsed = @baseline_elapsed,
|
|
50
|
+
completed_passes = @completed_baseline_passes)
|
|
51
|
+
return unless config["max_duration"]
|
|
52
|
+
return if store.reusable?(files)
|
|
53
|
+
return if completed_passes.to_i.zero?
|
|
54
|
+
|
|
55
|
+
per_pass = baseline_elapsed / completed_passes
|
|
56
|
+
remaining = config["max_duration"] - (Process.clock_gettime(Process::CLOCK_MONOTONIC) - @run_started)
|
|
57
|
+
# A spent budget is the existing stop path's news, and an untimed
|
|
58
|
+
# (zero-second) baseline carries no estimate to project from.
|
|
59
|
+
return if remaining <= 0 || per_pass <= 0
|
|
60
|
+
return if remaining >= per_pass
|
|
61
|
+
|
|
62
|
+
needed = per_pass - remaining
|
|
63
|
+
raise BudgetPhaseTimeout,
|
|
64
|
+
"the --max-duration budget cannot cover the coverage map build: baseline passes took about " \
|
|
65
|
+
"#{per_pass.round}s each, only #{remaining.round(1)}s remain, and a cold build needs about one " \
|
|
66
|
+
"more full pass and keeps nothing when it is cut off. Re-run with --max-duration about " \
|
|
67
|
+
"#{needed.round}s higher, fewer --baseline-passes, or once without --max-duration so the " \
|
|
68
|
+
"coverage cache is built for later --incremental runs"
|
|
69
|
+
end
|
|
70
|
+
|
|
71
|
+
# Boot and baselines can spend the budget by themselves: stop before
|
|
72
|
+
# an expensive cold coverage build and mark every selected subject
|
|
73
|
+
# unrun.
|
|
74
|
+
def abandon_for_budget?(subjects)
|
|
75
|
+
return false unless budget_exceeded?
|
|
76
|
+
|
|
77
|
+
@early_stop = EarlyStop.new(reason: "max_duration", unreached: subjects.map(&:id), truncated: [],
|
|
78
|
+
unverified: [])
|
|
79
|
+
true
|
|
80
|
+
end
|
|
81
|
+
|
|
82
|
+
def announce_budget_once
|
|
83
|
+
return if @budget_announced
|
|
84
|
+
|
|
85
|
+
@budget_announced = true
|
|
86
|
+
@diagnostics.inform("--max-duration reached: stopping before the next subject; the report marks it partial")
|
|
87
|
+
end
|
|
88
|
+
end
|
|
89
|
+
end
|
|
90
|
+
end
|
|
@@ -3,6 +3,7 @@
|
|
|
3
3
|
|
|
4
4
|
require "digest"
|
|
5
5
|
require "json"
|
|
6
|
+
require_relative "../text"
|
|
6
7
|
require_relative "data_inputs"
|
|
7
8
|
|
|
8
9
|
module Saboteur
|
|
@@ -252,6 +253,9 @@ module Saboteur
|
|
|
252
253
|
# undeclared dependency boundary, not a silently omitted source tree.
|
|
253
254
|
def project_files(issues, directory = @root)
|
|
254
255
|
Dir.children(directory).sort.flat_map do |name|
|
|
256
|
+
# Child names arrive locale-tagged; retag losslessly before
|
|
257
|
+
# joining with the UTF-8 root (#231).
|
|
258
|
+
name = Text.utf8_identity(name)
|
|
255
259
|
path = File.join(directory, name)
|
|
256
260
|
next [] if excluded?(path, name)
|
|
257
261
|
|
|
@@ -16,26 +16,31 @@ module Saboteur
|
|
|
16
16
|
end
|
|
17
17
|
|
|
18
18
|
# The third C1 layer (#152): each mutant child makes the project tree
|
|
19
|
-
# read-only except tmp
|
|
19
|
+
# read-only except tmp/ and the system temp dir - the cache dir stays
|
|
20
|
+
# read-only too, so a mutant cannot delete the baseline, the reports
|
|
21
|
+
# or other children's logs; its own log arrives as an inherited open
|
|
22
|
+
# descriptor (#234). The
|
|
20
23
|
# claim needs Landlock ABI 3 (Linux 6.2+): earlier ABIs cannot restrict
|
|
21
24
|
# truncate(), so the run degrades honestly - children run with the
|
|
22
25
|
# guard and canary only, noted once - unless c.sandbox = "strict",
|
|
23
26
|
# which refuses instead.
|
|
24
27
|
def hook
|
|
25
28
|
return nil unless @config["sandbox"]
|
|
29
|
+
return seatbelt_hook if @config["sandbox"] == "seatbelt" && Execution::Seatbelt.darwin?
|
|
26
30
|
|
|
27
31
|
abi = sandbox_abi
|
|
28
32
|
return restrict_hook if abi >= 3
|
|
29
33
|
|
|
30
34
|
if @config["sandbox"] == "strict"
|
|
31
|
-
raise Error, "sandbox is strict but #{sandbox_gap(abi)}; refusing to run without the promised " \
|
|
35
|
+
raise Error, "sandbox is strict but #{self.class.sandbox_gap(abi)}; refusing to run without the promised " \
|
|
32
36
|
"sandbox - set c.sandbox = true to run degraded, or false to opt out"
|
|
33
37
|
end
|
|
34
38
|
# Once, through the report's warnings only: warn + inform printed the
|
|
35
39
|
# same line twice (D's review, #178).
|
|
36
40
|
unless @sandbox_note_given
|
|
37
41
|
@sandbox_note_given = true
|
|
38
|
-
@diagnostics.warn("sandbox is on but #{sandbox_gap(abi)}; children run with the hazard guard
|
|
42
|
+
@diagnostics.warn("sandbox is on but #{self.class.sandbox_gap(abi)}; children run with the hazard guard " \
|
|
43
|
+
"and canary only")
|
|
39
44
|
end
|
|
40
45
|
nil
|
|
41
46
|
end
|
|
@@ -49,7 +54,7 @@ module Saboteur
|
|
|
49
54
|
"unsandboxed - set c.sandbox = false to proceed without it"
|
|
50
55
|
end
|
|
51
56
|
|
|
52
|
-
def sandbox_gap(abi)
|
|
57
|
+
def self.sandbox_gap(abi)
|
|
53
58
|
return "this kernel lacks Landlock (needs Linux 6.2+)" if abi.zero?
|
|
54
59
|
|
|
55
60
|
"this kernel's Landlock ABI is #{abi} but ABI 3 (Linux 6.2+) is needed to restrict truncate()"
|
|
@@ -60,22 +65,62 @@ module Saboteur
|
|
|
60
65
|
->(keep_fds) { Execution::Sandbox.restrict!(writable: writable, keep_fds: keep_fds) }
|
|
61
66
|
end
|
|
62
67
|
|
|
68
|
+
# c.sandbox = "seatbelt" (the macOS opt-in, #436): the probe decides.
|
|
69
|
+
# Confine when Seatbelt truly enforces; a clean "no Seatbelt here"
|
|
70
|
+
# degrades to the guard and canary with a one-time note; anything
|
|
71
|
+
# else - an errored probe, or an API claiming success its profile
|
|
72
|
+
# cannot prove - refuses the run, exactly like the Landlock probe
|
|
73
|
+
# (#178): opting into a sandbox must never buy silent confinement
|
|
74
|
+
# theatre. Fail-closed lives one layer down too: a child whose
|
|
75
|
+
# profile refuses to apply raises instead of running the mutant.
|
|
76
|
+
def seatbelt_hook
|
|
77
|
+
case (verdict = Execution::Seatbelt.probe)
|
|
78
|
+
when :ok
|
|
79
|
+
writable = sandbox_writable_dirs
|
|
80
|
+
->(keep_fds) { Execution::Seatbelt.restrict!(writable: writable, keep_fds: keep_fds) }
|
|
81
|
+
when :unavailable
|
|
82
|
+
seatbelt_note("this macOS lacks Seatbelt")
|
|
83
|
+
nil
|
|
84
|
+
else
|
|
85
|
+
raise Error, "sandbox is \"seatbelt\" but the Seatbelt probe #{verdict_description(verdict)}; refusing to " \
|
|
86
|
+
"run unsandboxed - set c.sandbox = true to run with the hazard guard and canary only, " \
|
|
87
|
+
"or false to opt out"
|
|
88
|
+
end
|
|
89
|
+
end
|
|
90
|
+
|
|
91
|
+
def verdict_description(verdict)
|
|
92
|
+
case verdict
|
|
93
|
+
when :unenforced then "reported a profile that did not enforce - a macOS update may have changed it (#436)"
|
|
94
|
+
else "failed (#{verdict}) - #436 tracks the deprecation"
|
|
95
|
+
end
|
|
96
|
+
end
|
|
97
|
+
|
|
98
|
+
def seatbelt_note(reason)
|
|
99
|
+
return if @sandbox_note_given
|
|
100
|
+
|
|
101
|
+
@sandbox_note_given = true
|
|
102
|
+
@diagnostics.warn(%(sandbox is "seatbelt" but #{reason}; children run with the hazard guard and canary only))
|
|
103
|
+
end
|
|
104
|
+
|
|
63
105
|
# c.sandbox_writable adds the dirs a suite genuinely needs (db/, log/,
|
|
64
106
|
# ...): without a grant, writes there fail with EACCES in the child.
|
|
107
|
+
# Only the user grants are created: the project tmp is granted when it
|
|
108
|
+
# already exists, never made for it (#236).
|
|
65
109
|
def sandbox_writable_dirs
|
|
66
110
|
user_dirs = @config["sandbox_writable"].map { |dir| File.expand_path(dir, @root) }
|
|
67
|
-
|
|
68
|
-
|
|
111
|
+
user_dirs.each { |dir| FileUtils.mkdir_p(dir) }
|
|
112
|
+
tmp = File.join(@root, "tmp")
|
|
113
|
+
dirs = Dir.exist?(tmp) ? [tmp] + user_dirs : user_dirs
|
|
69
114
|
real = (dirs.map { |dir| File.realpath(dir) } + [File.realpath(Dir.tmpdir)]).uniq
|
|
70
115
|
check_sandbox_write_set(real, user_dirs.map { |dir| File.realpath(dir) })
|
|
71
116
|
real
|
|
72
117
|
end
|
|
73
118
|
|
|
74
|
-
# The effective write set - every grant realpath'd,
|
|
75
|
-
#
|
|
76
|
-
#
|
|
77
|
-
#
|
|
78
|
-
#
|
|
119
|
+
# The effective write set - every grant realpath'd, the project tmp
|
|
120
|
+
# and the system temp dir included - must not contain the project
|
|
121
|
+
# root: a tmp symlink or a checkout inside the system temp dir would
|
|
122
|
+
# otherwise leave the whole tree writable without a word.
|
|
123
|
+
# c.sandbox = "strict" refuses: strict guards the effective
|
|
79
124
|
# policy, not only the kernel ABI. A user grant containing another
|
|
80
125
|
# grant only makes the narrower one redundant, so that stays a warning.
|
|
81
126
|
def check_sandbox_write_set(all_dirs, user_dirs)
|