saboteur 0.8.0 → 0.9.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +215 -0
- data/README.md +44 -6
- data/exe/saboteur +6 -2
- data/lib/saboteur/attribute_finder.rb +116 -0
- data/lib/saboteur/baseline/run_evidence.rb +2 -1
- data/lib/saboteur/cli/agent_provenance.rb +59 -0
- data/lib/saboteur/cli/dirty_tree_notice.rb +28 -0
- data/lib/saboteur/cli/doctor_command.rb +41 -0
- data/lib/saboteur/cli/error_cause.rb +40 -0
- data/lib/saboteur/cli/execution_options.rb +46 -0
- data/lib/saboteur/cli/exit_status.rb +12 -14
- data/lib/saboteur/cli/explicit_file_options.rb +21 -0
- data/lib/saboteur/cli/explicit_sources.rb +46 -0
- data/lib/saboteur/cli/failure_output.rb +35 -0
- data/lib/saboteur/cli/help_command.rb +23 -0
- data/lib/saboteur/cli/matrix_option.rb +17 -0
- data/lib/saboteur/cli/output_options.rb +27 -0
- data/lib/saboteur/cli/presets.rb +41 -1
- data/lib/saboteur/cli/preview_command.rb +26 -0
- data/lib/saboteur/cli/run_budget_options.rb +6 -1
- data/lib/saboteur/cli/run_command.rb +33 -0
- data/lib/saboteur/cli/run_output.rb +38 -0
- data/lib/saboteur/cli/run_reporting.rb +6 -5
- data/lib/saboteur/cli/run_validation.rb +54 -0
- data/lib/saboteur/cli/source_set.rb +0 -18
- data/lib/saboteur/cli.rb +29 -305
- data/lib/saboteur/config.rb +76 -7
- data/lib/saboteur/config_file.rb +12 -8
- data/lib/saboteur/constant_subject_finder.rb +3 -0
- data/lib/saboteur/custom_map.rb +24 -5
- data/lib/saboteur/deep_source.rb +49 -0
- data/lib/saboteur/dirty_tree_warning.rb +6 -4
- data/lib/saboteur/doctor.rb +10 -0
- data/lib/saboteur/engine/baseline_guard.rb +118 -15
- data/lib/saboteur/engine/baseline_parts.rb +51 -0
- data/lib/saboteur/engine/boot.rb +11 -3
- data/lib/saboteur/engine/bytecode_equivalence.rb +17 -1
- data/lib/saboteur/engine/candidate_selection.rb +16 -5
- data/lib/saboteur/engine/candidates.rb +4 -1
- data/lib/saboteur/engine/coverage_inputs_snapshot.rb +146 -0
- data/lib/saboteur/engine/coverage_miss_note.rb +86 -0
- data/lib/saboteur/engine/coverage_partial.rb +48 -0
- data/lib/saboteur/engine/coverage_parts.rb +50 -0
- data/lib/saboteur/engine/coverage_shadow.rb +92 -0
- data/lib/saboteur/engine/coverage_store.rb +296 -21
- data/lib/saboteur/engine/diagnostics.rb +24 -7
- data/lib/saboteur/engine/distribution.rb +2 -2
- data/lib/saboteur/engine/gated_run.rb +4 -1
- data/lib/saboteur/engine/load_coverage.rb +72 -8
- data/lib/saboteur/engine/partial_invalidation.rb +115 -0
- data/lib/saboteur/engine/partial_merge.rb +40 -0
- data/lib/saboteur/engine/pipelines.rb +11 -3
- data/lib/saboteur/engine/reference_specs.rb +71 -0
- data/lib/saboteur/engine/result_cache.rb +68 -5
- data/lib/saboteur/engine/resume_manifest.rb +110 -0
- data/lib/saboteur/engine/run_budget.rb +10 -7
- data/lib/saboteur/engine/run_fingerprint.rb +2 -1
- data/lib/saboteur/engine/sandbox_policy.rb +26 -5
- data/lib/saboteur/engine/scheduler.rb +33 -25
- data/lib/saboteur/engine/selection_announcement.rb +9 -3
- data/lib/saboteur/engine/shadow_comparison.rb +121 -0
- data/lib/saboteur/engine/shadow_run.rb +38 -0
- data/lib/saboteur/engine/subject_selection.rb +144 -64
- data/lib/saboteur/engine/suite_partition.rb +22 -0
- data/lib/saboteur/engine/test_guess.rb +186 -32
- data/lib/saboteur/engine/unchanged_syntax.rb +10 -2
- data/lib/saboteur/engine.rb +44 -11
- data/lib/saboteur/environment_hints.rb +55 -0
- data/lib/saboteur/execution/bundle_gemfile_pin.rb +72 -0
- data/lib/saboteur/execution/external_runner.rb +123 -1
- data/lib/saboteur/execution/fiddle_loader.rb +34 -0
- data/lib/saboteur/execution/gemfile_path_sources.rb +166 -0
- data/lib/saboteur/execution/marker_sweep.rb +2 -1
- data/lib/saboteur/execution/process_table.rb +7 -3
- data/lib/saboteur/execution/reevaluator.rb +7 -0
- data/lib/saboteur/execution/sandbox.rb +39 -20
- data/lib/saboteur/execution/seatbelt.rb +10 -3
- data/lib/saboteur/execution/supervisor.rb +42 -8
- data/lib/saboteur/execution/supervisor_batch.rb +105 -0
- data/lib/saboteur/execution/test_command_shape.rb +149 -0
- data/lib/saboteur/execution/worker_pool.rb +48 -4
- data/lib/saboteur/frameworks/rails.rb +2 -5
- data/lib/saboteur/frameworks/sqlite_memory_database.rb +32 -2
- data/lib/saboteur/gates.rb +121 -28
- data/lib/saboteur/install.rb +6 -1
- data/lib/saboteur/integration/base.rb +19 -2
- data/lib/saboteur/integration/rspec.rb +145 -18
- data/lib/saboteur/line_hits.rb +81 -0
- data/lib/saboteur/method_calls.rb +91 -0
- data/lib/saboteur/mutation_source/argument_swap.rb +54 -0
- data/lib/saboteur/mutation_source/builtin.rb +33 -21
- data/lib/saboteur/mutation_source/condition_negation.rb +3 -12
- data/lib/saboteur/mutation_source/hazard_guard.rb +94 -13
- data/lib/saboteur/mutation_source/jump_value.rb +59 -0
- data/lib/saboteur/mutation_source/literal_replacement.rb +6 -23
- data/lib/saboteur/mutation_source/operator_examples.rb +6 -0
- data/lib/saboteur/mutation_source/operator_registry.rb +6 -0
- data/lib/saboteur/mutation_source/operator_visit.rb +7 -1
- data/lib/saboteur/mutation_source/rescue_removal.rb +69 -0
- data/lib/saboteur/mutation_source/unary_removal.rb +47 -0
- data/lib/saboteur/mutation_source/when_removal.rb +51 -0
- data/lib/saboteur/preview.rb +13 -0
- data/lib/saboteur/refinement_activations.rb +22 -10
- data/lib/saboteur/reporter/agent.rb +2 -2
- data/lib/saboteur/reporter/agent_incomplete.rb +31 -5
- data/lib/saboteur/reporter/agent_summary.rb +16 -10
- data/lib/saboteur/reporter/review_comments.rb +151 -0
- data/lib/saboteur/reporter/safe_out.rb +32 -0
- data/lib/saboteur/reporter/stryker.rb +8 -6
- data/lib/saboteur/reporter/survivor_classification.rb +41 -3
- data/lib/saboteur/reporter/survivor_hints.rb +5 -0
- data/lib/saboteur/reporter/terminal.rb +45 -5
- data/lib/saboteur/result.rb +14 -4
- data/lib/saboteur/selection/coverage_map.rb +150 -9
- data/lib/saboteur/selection/external_test_map.rb +4 -0
- data/lib/saboteur/selection/git_diff.rb +24 -2
- data/lib/saboteur/selection/layout_only_lines.rb +45 -1
- data/lib/saboteur/selection/recorder.rb +52 -22
- data/lib/saboteur/selection/trace_recorder.rb +83 -0
- data/lib/saboteur/selection/untracked_changes.rb +1 -2
- data/lib/saboteur/subject.rb +16 -0
- data/lib/saboteur/subject_finder.rb +91 -6
- data/lib/saboteur/text.rb +31 -0
- data/lib/saboteur/version.rb +1 -1
- data/lib/saboteur/working_tree.rb +7 -2
- data/lib/saboteur.rb +9 -0
- metadata +48 -16
|
@@ -0,0 +1,115 @@
|
|
|
1
|
+
# SPDX-License-Identifier: MIT
|
|
2
|
+
# frozen_string_literal: true
|
|
3
|
+
|
|
4
|
+
module Saboteur
|
|
5
|
+
class Engine
|
|
6
|
+
# Which spec files a partial coverage rebuild would have to rerun, or
|
|
7
|
+
# :full when the whole map must be rebuilt (#46, step 2). Pure: it reads
|
|
8
|
+
# two inputs snapshots and a coverage map and touches nothing else.
|
|
9
|
+
#
|
|
10
|
+
# An allowlist, not a blocklist. Coverage never records a test reading a
|
|
11
|
+
# fixture, a YAML file, an ENV value or a template, so no changed file
|
|
12
|
+
# can be proven harmless by absence. Only two kinds of change may take
|
|
13
|
+
# the partial path: a spec file of the suite, and an included source
|
|
14
|
+
# file with recorded coverage evidence that did not run while the app
|
|
15
|
+
# loaded. Every other change, and every added or removed file, is :full,
|
|
16
|
+
# as is a change of root, settings, runtime or snapshot format.
|
|
17
|
+
#
|
|
18
|
+
# The stale files come back in the order the map's full run used, so a
|
|
19
|
+
# rerun keeps the relative order that run had.
|
|
20
|
+
class PartialInvalidation
|
|
21
|
+
FULL = :full
|
|
22
|
+
# Placeholder, to be fitted from measurements; not configuration.
|
|
23
|
+
THRESHOLD = 0.5
|
|
24
|
+
SAME = %w[format root runtime settings].freeze
|
|
25
|
+
|
|
26
|
+
# included: relative paths of the included source files.
|
|
27
|
+
# spec_files: relative paths of the suite's spec files.
|
|
28
|
+
# status_file: whether RSpec picks examples from its generated status
|
|
29
|
+
# file (--only-failures, --next-failure), which no snapshot covers.
|
|
30
|
+
# RSpec ids name their file: "./spec/a_spec.rb[1:2]" is spec/a_spec.rb.
|
|
31
|
+
def self.spec_of(id)
|
|
32
|
+
id.split("[").first.delete_prefix("./")
|
|
33
|
+
end
|
|
34
|
+
|
|
35
|
+
def initialize(map:, root:, included:, spec_files:, status_file: false)
|
|
36
|
+
@status_file = status_file
|
|
37
|
+
@map = map
|
|
38
|
+
@root = Selection::CoverageMap.canonical(root)
|
|
39
|
+
@included = included
|
|
40
|
+
@spec_files = spec_files
|
|
41
|
+
end
|
|
42
|
+
|
|
43
|
+
def stale_specs(base, current)
|
|
44
|
+
return FULL if @status_file
|
|
45
|
+
return FULL unless SAME.all? { |key| base[key] == current[key] }
|
|
46
|
+
|
|
47
|
+
changed = changed_files([base, current].map { |snapshot| snapshot["files"] })
|
|
48
|
+
return FULL unless changed
|
|
49
|
+
|
|
50
|
+
stale = changed.each_with_object([]) do |file, specs|
|
|
51
|
+
found = specs_for(file)
|
|
52
|
+
return FULL unless found
|
|
53
|
+
|
|
54
|
+
specs.concat(found)
|
|
55
|
+
end
|
|
56
|
+
ordered(stale.uniq).then { |specs| heavy?(specs) ? FULL : specs }
|
|
57
|
+
end
|
|
58
|
+
|
|
59
|
+
# The part of the recorded spec time these files hold, 0 when none was recorded.
|
|
60
|
+
def share(specs)
|
|
61
|
+
total = weights.values.sum
|
|
62
|
+
total.zero? ? 0.0 : specs.sum { |file| weights.fetch(file, 0) }.fdiv(total)
|
|
63
|
+
end
|
|
64
|
+
|
|
65
|
+
private
|
|
66
|
+
|
|
67
|
+
# nil when a file was added, removed or unreadable on either side.
|
|
68
|
+
def changed_files(snapshots)
|
|
69
|
+
before, after = snapshots
|
|
70
|
+
return nil unless before.keys.sort == after.keys.sort
|
|
71
|
+
return nil if (before.values + after.values).include?(false)
|
|
72
|
+
|
|
73
|
+
before.keys.reject { |file| before[file] == after[file] }
|
|
74
|
+
end
|
|
75
|
+
|
|
76
|
+
def specs_for(file)
|
|
77
|
+
return [file] if @spec_files.include?(file)
|
|
78
|
+
return nil unless @included.include?(file) && evidence?(File.join(@root, file))
|
|
79
|
+
|
|
80
|
+
touching(File.join(@root, file)).then { |specs| specs if specs.all? { |spec| @spec_files.include?(spec) } }
|
|
81
|
+
end
|
|
82
|
+
|
|
83
|
+
def evidence?(path)
|
|
84
|
+
[@map.lines, @map.methods].any? { |hits| (hits[path] || {}).values.any?(&:any?) } &&
|
|
85
|
+
[@map.load_lines, @map.load_methods].none? { |hits| hits[path]&.any? } &&
|
|
86
|
+
!@map.preloaded?(path)
|
|
87
|
+
end
|
|
88
|
+
|
|
89
|
+
def touching(path)
|
|
90
|
+
ids = [@map.lines, @map.methods].flat_map { |hits| (hits[path] || {}).values.flatten }
|
|
91
|
+
ids.map { |id| spec_of(id) }.uniq
|
|
92
|
+
end
|
|
93
|
+
|
|
94
|
+
def spec_of(id)
|
|
95
|
+
self.class.spec_of(id)
|
|
96
|
+
end
|
|
97
|
+
|
|
98
|
+
def ordered(stale)
|
|
99
|
+
run_order = @map.tests.map { |id| spec_of(id) }.uniq
|
|
100
|
+
run_order.select { |file| stale.include?(file) } + (stale - run_order).sort
|
|
101
|
+
end
|
|
102
|
+
|
|
103
|
+
# Stale share of the recorded time, by spec file, never above THRESHOLD.
|
|
104
|
+
def heavy?(specs)
|
|
105
|
+
total = weights.values.sum
|
|
106
|
+
total.zero? || share(specs) > THRESHOLD
|
|
107
|
+
end
|
|
108
|
+
|
|
109
|
+
def weights
|
|
110
|
+
@map.tests.group_by { |id| spec_of(id) }
|
|
111
|
+
.transform_values { |ids| ids.sum { |id| @map.durations.fetch(id, 0) } }
|
|
112
|
+
end
|
|
113
|
+
end
|
|
114
|
+
end
|
|
115
|
+
end
|
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
# SPDX-License-Identifier: MIT
|
|
2
|
+
# frozen_string_literal: true
|
|
3
|
+
|
|
4
|
+
module Saboteur
|
|
5
|
+
class Engine
|
|
6
|
+
# The map a partial rebuild produces (#46, step 3): the base map with all
|
|
7
|
+
# evidence of the rerun spec files removed, plus the fresh map the child
|
|
8
|
+
# recorded over just those files. Whole spec files are replaced, never
|
|
9
|
+
# single examples, since a file's example ids can shift when it changes.
|
|
10
|
+
# Load-time evidence, loaded files and preloaded files carry over from
|
|
11
|
+
# both: every run boots the same app.
|
|
12
|
+
module PartialMerge
|
|
13
|
+
module_function
|
|
14
|
+
|
|
15
|
+
def call(base, fresh, spec_files)
|
|
16
|
+
specs = spec_files.to_set
|
|
17
|
+
dropped = base.tests.select { |id| specs.include?(PartialInvalidation.spec_of(id)) }.to_set
|
|
18
|
+
trimmed = Selection::CoverageMap.new(
|
|
19
|
+
lines: without(base.lines, dropped), methods: without(base.methods, dropped),
|
|
20
|
+
branches: without(base.branches, dropped), tests: base.tests.reject { |id| dropped.include?(id) },
|
|
21
|
+
durations: base.durations.except(*dropped),
|
|
22
|
+
load_lines: base.load_lines, load_methods: base.load_methods,
|
|
23
|
+
loaded_files: base.loaded_files, preloaded: base.preloaded.to_a, build_seconds: base.build_seconds
|
|
24
|
+
)
|
|
25
|
+
Selection::CoverageMap.combine([trimmed, fresh])
|
|
26
|
+
end
|
|
27
|
+
|
|
28
|
+
# {file => {line-or-span => ids}} without the dropped ids. An entry left
|
|
29
|
+
# with no test is removed, and so is a file left with no entry: a file
|
|
30
|
+
# with no test behind it must read as no evidence, not as covered.
|
|
31
|
+
def without(hits, dropped)
|
|
32
|
+
hits.each_with_object({}) do |(file, by_line), acc|
|
|
33
|
+
kept = by_line.transform_values { |ids| ids.reject { |id| dropped.include?(id) } }
|
|
34
|
+
kept.reject! { |_, ids| ids.empty? }
|
|
35
|
+
acc[file] = kept unless kept.empty?
|
|
36
|
+
end
|
|
37
|
+
end
|
|
38
|
+
end
|
|
39
|
+
end
|
|
40
|
+
end
|
|
@@ -1,6 +1,8 @@
|
|
|
1
1
|
# SPDX-License-Identifier: MIT
|
|
2
2
|
# frozen_string_literal: true
|
|
3
3
|
|
|
4
|
+
require_relative "../execution/test_command_shape"
|
|
5
|
+
|
|
4
6
|
module Saboteur
|
|
5
7
|
class Engine
|
|
6
8
|
# The two run pipelines: in-process (boot, coverage, workers) and the
|
|
@@ -22,7 +24,7 @@ module Saboteur
|
|
|
22
24
|
mutate(subjects, files, integration, scope, ignore_list,
|
|
23
25
|
load_lines:, load_methods:)
|
|
24
26
|
end
|
|
25
|
-
@diagnostics.
|
|
27
|
+
@diagnostics.add_warnings(ignore_list.apply(results, unscoped_digests: @unscoped_digests))
|
|
26
28
|
build_report(results, subjects, scope, started)
|
|
27
29
|
ensure
|
|
28
30
|
# Cleanup runs however the call ends - a partway boot failure, no
|
|
@@ -43,10 +45,16 @@ module Saboteur
|
|
|
43
45
|
def call_external(ignore_list, started)
|
|
44
46
|
@diagnostics.inform("external test command: #{config['test_command']} " \
|
|
45
47
|
"(serial, no coverage, unsandboxed, mutants applied in a disposable copy)")
|
|
48
|
+
if Execution::TestCommandShape.multi_file_ruby?(config["test_command"])
|
|
49
|
+
@diagnostics.warn("use the framework's multi-file form (bin/rails test f1 f2, rspec f1 f2) or a " \
|
|
50
|
+
"shell loop (for f in ...; do ruby \"$f\" || exit 1; done)\n " \
|
|
51
|
+
"the test command hands several test files to one ruby: ruby executes only " \
|
|
52
|
+
"the first, so the rest never run and their mutants silently survive (#611)")
|
|
53
|
+
end
|
|
46
54
|
files = config.included_files
|
|
47
55
|
subjects, scope = select_subjects(files)
|
|
48
56
|
results = subjects.empty? ? [] : mutate_external(subjects, scope, ignore_list)
|
|
49
|
-
@diagnostics.
|
|
57
|
+
@diagnostics.add_warnings(ignore_list.apply(results, unscoped_digests: @unscoped_digests))
|
|
50
58
|
build_report(results, subjects, scope, started)
|
|
51
59
|
end
|
|
52
60
|
|
|
@@ -78,7 +86,7 @@ module Saboteur
|
|
|
78
86
|
end
|
|
79
87
|
run_candidates(runnable_subjects, runnable_lists, scheduler, nil, ignore_list)
|
|
80
88
|
ensure
|
|
81
|
-
runner
|
|
89
|
+
runner&.close
|
|
82
90
|
end
|
|
83
91
|
|
|
84
92
|
# Warn loudly when the command cannot see mutants at all: an
|
|
@@ -0,0 +1,71 @@
|
|
|
1
|
+
# SPDX-License-Identifier: MIT
|
|
2
|
+
# frozen_string_literal: true
|
|
3
|
+
|
|
4
|
+
module Saboteur
|
|
5
|
+
class Engine
|
|
6
|
+
# The specs that read a constant a source file defines, or require it: the
|
|
7
|
+
# cold-start choice for a file with no spec by name. The constant match is
|
|
8
|
+
# the lexical matching #538 uses for changed specs; the require match
|
|
9
|
+
# covers a file that defines only top-level methods. A file no spec reads or requires, or one read by more than
|
|
10
|
+
# CAP specs (a hub: the choice is not selective), gets no entry.
|
|
11
|
+
class ReferenceSpecs
|
|
12
|
+
CAP = 10
|
|
13
|
+
REQUIRE = /^\s*(require_relative|require)[ (]\s*["']([^"']+)["']/
|
|
14
|
+
|
|
15
|
+
# constants: file => the constants it defines, as TestGuess shapes them
|
|
16
|
+
# for ConstantReads.
|
|
17
|
+
def initialize(root, constants)
|
|
18
|
+
@root = root
|
|
19
|
+
@constants = constants
|
|
20
|
+
end
|
|
21
|
+
|
|
22
|
+
# chosen: file => sorted root-relative spec paths; crowded: file => how
|
|
23
|
+
# many specs read it, for the files over the cap (no choice was made).
|
|
24
|
+
Found = Data.define(:chosen, :crowded)
|
|
25
|
+
|
|
26
|
+
# The announcement for the files that got specs by reference.
|
|
27
|
+
def self.note(referenced)
|
|
28
|
+
named = referenced.map { |file, specs| "#{file} (#{specs.join(', ')})" }
|
|
29
|
+
"inferred specs: selected by constant reference or require, not by name: #{named.join(', ')}; " \
|
|
30
|
+
"a spec that reaches the file another way is not found"
|
|
31
|
+
end
|
|
32
|
+
|
|
33
|
+
def self.crowd_note(count) = "#{count} specs reference or require it, over the cap of #{CAP}"
|
|
34
|
+
|
|
35
|
+
# files: root-relative source files; spec_paths: root-relative spec
|
|
36
|
+
# files of the suite.
|
|
37
|
+
def call(files, spec_paths)
|
|
38
|
+
absolute = spec_paths.map { |path| File.join(@root, path) }
|
|
39
|
+
absolute.select! { |path| File.file?(path) && !File.symlink?(path) }
|
|
40
|
+
found = files.to_h { |file| [file, readers_of(file, absolute)] }.reject { |_, readers| readers.empty? }
|
|
41
|
+
crowded, chosen = found.partition { |_, readers| readers.size > CAP }.map(&:to_h)
|
|
42
|
+
Found.new(chosen: chosen, crowded: crowded.transform_values(&:size))
|
|
43
|
+
end
|
|
44
|
+
|
|
45
|
+
private
|
|
46
|
+
|
|
47
|
+
def readers_of(file, absolute_specs)
|
|
48
|
+
read = @constants.call(file).flat_map { |constant| ConstantReads.across(absolute_specs, constant).keys }
|
|
49
|
+
(read + requirers_of(file, absolute_specs)).uniq.sort.map { |path| path.delete_prefix("#{@root}/") }
|
|
50
|
+
end
|
|
51
|
+
|
|
52
|
+
# Specs that require the file: by path from the spec (require_relative)
|
|
53
|
+
# or by load path from the file's source root (require "a/b" for
|
|
54
|
+
# lib/a/b.rb). A bare basename is not a load path, so it matches nothing.
|
|
55
|
+
def requirers_of(file, absolute_specs)
|
|
56
|
+
stem = File.join(@root, file).delete_suffix(".rb")
|
|
57
|
+
loadable = file.delete_suffix(".rb").sub(%r{\A[^/]+/}, "")
|
|
58
|
+
absolute_specs.select { |spec| requires?(spec, stem, loadable) }
|
|
59
|
+
end
|
|
60
|
+
|
|
61
|
+
def requires?(spec, stem, loadable)
|
|
62
|
+
File.read(spec, encoding: "UTF-8").scrub.scan(REQUIRE).any? do |kind, target|
|
|
63
|
+
target = target.delete_suffix(".rb")
|
|
64
|
+
kind == "require_relative" ? File.expand_path(target, File.dirname(spec)) == stem : target == loadable
|
|
65
|
+
end
|
|
66
|
+
rescue SystemCallError
|
|
67
|
+
false
|
|
68
|
+
end
|
|
69
|
+
end
|
|
70
|
+
end
|
|
71
|
+
end
|
|
@@ -4,6 +4,7 @@
|
|
|
4
4
|
require "digest"
|
|
5
5
|
require "fileutils"
|
|
6
6
|
require "json"
|
|
7
|
+
require "securerandom"
|
|
7
8
|
require_relative "../text"
|
|
8
9
|
|
|
9
10
|
module Saboteur
|
|
@@ -25,6 +26,7 @@ module Saboteur
|
|
|
25
26
|
# stale.
|
|
26
27
|
class ResultCache
|
|
27
28
|
FORMAT = 1
|
|
29
|
+
MAX_BYTES = 64 * 1024 * 1024
|
|
28
30
|
STORED = %i[killed survived timeout].freeze
|
|
29
31
|
LOCKFILES = %w[Gemfile.lock gems.locked].freeze
|
|
30
32
|
# Dependency declarations themselves, so the no-lockfile library
|
|
@@ -34,9 +36,15 @@ module Saboteur
|
|
|
34
36
|
|
|
35
37
|
SOURCE_EXCLUDES = %w[.git vendor node_modules].freeze
|
|
36
38
|
|
|
39
|
+
# Seconds between checkpoint saves while verdicts arrive, so a run
|
|
40
|
+
# killed mid-phase (OOM, CI timeout, SIGKILL) keeps what it finished
|
|
41
|
+
# (#471).
|
|
42
|
+
SAVE_INTERVAL = 30
|
|
43
|
+
|
|
37
44
|
# suite_digest: SuiteFiles#digest, so an edit to a spec helper or
|
|
38
45
|
# support file invalidates every verdict.
|
|
39
|
-
def initialize(path:, root:, inputs:, suite_digest: "", coverage: nil, fallback_files: []
|
|
46
|
+
def initialize(path:, root:, inputs:, suite_digest: "", coverage: nil, fallback_files: [],
|
|
47
|
+
save_interval: SAVE_INTERVAL, clock: -> { Process.clock_gettime(Process::CLOCK_MONOTONIC) })
|
|
40
48
|
@path = path
|
|
41
49
|
@root = root
|
|
42
50
|
@inputs = inputs
|
|
@@ -47,6 +55,9 @@ module Saboteur
|
|
|
47
55
|
@fallback_files = fallback_files.map { |file| File.expand_path(file, root) }.sort
|
|
48
56
|
@entries = load
|
|
49
57
|
@file_digests = {}
|
|
58
|
+
@save_interval = save_interval
|
|
59
|
+
@clock = clock
|
|
60
|
+
@last_saved = clock.call
|
|
50
61
|
end
|
|
51
62
|
|
|
52
63
|
# The stored verdict as a hash (status, reason, killed_by, kill_cause,
|
|
@@ -67,15 +78,39 @@ module Saboteur
|
|
|
67
78
|
"reason" => result.reason, "killed_by" => result.killed_by, "kill_cause" => result.kill_cause,
|
|
68
79
|
"runtime" => result.runtime
|
|
69
80
|
}
|
|
81
|
+
checkpoint if @clock.call - @last_saved >= @save_interval
|
|
70
82
|
end
|
|
71
83
|
|
|
84
|
+
# Written to a temp file and renamed, so a kill mid-write leaves the
|
|
85
|
+
# previous complete file, never a truncated one. A failed write or
|
|
86
|
+
# rename removes the temp file and raises.
|
|
72
87
|
def save
|
|
73
88
|
FileUtils.mkdir_p(File.dirname(@path))
|
|
74
|
-
|
|
89
|
+
temp = "#{@path}.#{SecureRandom.hex(8)}.tmp"
|
|
90
|
+
begin
|
|
91
|
+
# "wx": a planted file or symlink at the temp name is never followed.
|
|
92
|
+
File.open(temp, "wx", 0o600) { |out| out.write(JSON.generate(format: FORMAT, entries: @entries)) }
|
|
93
|
+
File.rename(temp, @path)
|
|
94
|
+
rescue StandardError
|
|
95
|
+
FileUtils.rm_f(temp)
|
|
96
|
+
raise
|
|
97
|
+
end
|
|
98
|
+
@last_saved = @clock.call
|
|
75
99
|
end
|
|
76
100
|
|
|
77
101
|
private
|
|
78
102
|
|
|
103
|
+
# A mid-run save is a safety net, so it is best effort: a full disk or
|
|
104
|
+
# an unwritable directory must not abort a run that is still producing
|
|
105
|
+
# verdicts. The wait restarts either way, so a broken disk is tried
|
|
106
|
+
# once per interval, not once per verdict. The final save is not
|
|
107
|
+
# swallowed (Engine calls #save).
|
|
108
|
+
def checkpoint
|
|
109
|
+
save
|
|
110
|
+
rescue SystemCallError
|
|
111
|
+
@last_saved = @clock.call
|
|
112
|
+
end
|
|
113
|
+
|
|
79
114
|
# The current project source files, relative and sorted. A file added
|
|
80
115
|
# after the map was built changes the set and invalidates every stored
|
|
81
116
|
# verdict, whether or not any watched file references it yet.
|
|
@@ -86,13 +121,41 @@ module Saboteur
|
|
|
86
121
|
.join("\0")
|
|
87
122
|
end
|
|
88
123
|
|
|
124
|
+
# The file is untrusted input (a restored cache, a --resume directory):
|
|
125
|
+
# read only as a regular file, never a symlink or FIFO, under a size
|
|
126
|
+
# cap, and kept only when entries is a Hash of Hashes.
|
|
89
127
|
def load
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
128
|
+
return {} unless File.lstat(@path).file?
|
|
129
|
+
|
|
130
|
+
text = slurp
|
|
131
|
+
return {} unless text
|
|
132
|
+
|
|
133
|
+
data = JSON.parse(Text.utf8_strict(text))
|
|
134
|
+
entries = data["entries"] if data.is_a?(Hash) && data["format"] == FORMAT
|
|
135
|
+
well_formed?(entries) ? entries : {}
|
|
136
|
+
rescue SystemCallError, JSON::ParserError, ArgumentError, EncodingError
|
|
93
137
|
{}
|
|
94
138
|
end
|
|
95
139
|
|
|
140
|
+
def slurp
|
|
141
|
+
File.open(@path, File::RDONLY | File::NONBLOCK | File::NOFOLLOW) do |io|
|
|
142
|
+
text = io.read(MAX_BYTES)
|
|
143
|
+
text if io.eof?
|
|
144
|
+
end
|
|
145
|
+
end
|
|
146
|
+
|
|
147
|
+
def well_formed?(entries)
|
|
148
|
+
entries.is_a?(Hash) && entries.values.all? { |entry| entry.is_a?(Hash) && typed?(entry) }
|
|
149
|
+
end
|
|
150
|
+
|
|
151
|
+
# The fields the reports read, typed so a crafted entry cannot crash
|
|
152
|
+
# them.
|
|
153
|
+
def typed?(entry)
|
|
154
|
+
STORED.map(&:to_s).include?(entry["status"]) && (entry["runtime"].nil? || entry["runtime"].is_a?(Numeric)) &&
|
|
155
|
+
%w[reason kill_cause].all? { |key| entry[key].nil? || entry[key].is_a?(String) } &&
|
|
156
|
+
entry["killed_by"].is_a?(Array) && entry["killed_by"].all?(String)
|
|
157
|
+
end
|
|
158
|
+
|
|
96
159
|
def key(subject, mutation)
|
|
97
160
|
"#{subject.id} #{mutation.digest(subject_start: subject.start_byte, original_source: subject.file_source)}"
|
|
98
161
|
end
|
|
@@ -0,0 +1,110 @@
|
|
|
1
|
+
# SPDX-License-Identifier: MIT
|
|
2
|
+
# frozen_string_literal: true
|
|
3
|
+
|
|
4
|
+
require "fileutils"
|
|
5
|
+
require "json"
|
|
6
|
+
require "securerandom"
|
|
7
|
+
|
|
8
|
+
module Saboteur
|
|
9
|
+
class Engine
|
|
10
|
+
# Identity file for a --resume directory (#471). The verdicts live in
|
|
11
|
+
# ResultCache; this only records what the run was computed from, so a
|
|
12
|
+
# later run with other inputs fails loudly instead of reusing the
|
|
13
|
+
# directory under a different meaning.
|
|
14
|
+
#
|
|
15
|
+
# The file is untrusted input: it is read only when it is a regular
|
|
16
|
+
# file (never a FIFO or a symlink), under a size cap, and must hold
|
|
17
|
+
# exactly the three expected keys with bounded String values.
|
|
18
|
+
class ResumeManifest
|
|
19
|
+
FORMAT = 1
|
|
20
|
+
FILE = "manifest.json"
|
|
21
|
+
KEYS = %w[format identity version].freeze
|
|
22
|
+
MAX_BYTES = 4096
|
|
23
|
+
MAX_FIELD = 512
|
|
24
|
+
|
|
25
|
+
# Raised when the directory belongs to another run or cannot be read.
|
|
26
|
+
class Mismatch < StandardError; end
|
|
27
|
+
|
|
28
|
+
# identity: a String that changes whenever the run's inputs change
|
|
29
|
+
# (the RunFingerprint digest joined with the settings that pick the
|
|
30
|
+
# mutants), at most MAX_FIELD characters.
|
|
31
|
+
def initialize(directory:, identity:)
|
|
32
|
+
@directory = directory
|
|
33
|
+
@identity = identity
|
|
34
|
+
end
|
|
35
|
+
|
|
36
|
+
def path
|
|
37
|
+
File.join(@directory, FILE)
|
|
38
|
+
end
|
|
39
|
+
|
|
40
|
+
# Creates the manifest in a fresh directory, accepts a matching one,
|
|
41
|
+
# and raises Mismatch for anything else.
|
|
42
|
+
def verify!
|
|
43
|
+
File.lstat(path)
|
|
44
|
+
compare(read)
|
|
45
|
+
rescue Errno::ENOENT
|
|
46
|
+
write
|
|
47
|
+
end
|
|
48
|
+
|
|
49
|
+
private
|
|
50
|
+
|
|
51
|
+
def compare(stored)
|
|
52
|
+
return if stored["identity"] == @identity && stored["version"] == VERSION
|
|
53
|
+
|
|
54
|
+
raise Mismatch, "#{path}: the run inputs changed since this directory was created; " \
|
|
55
|
+
"delete it or choose a new --resume path"
|
|
56
|
+
end
|
|
57
|
+
|
|
58
|
+
def read
|
|
59
|
+
raise Mismatch, "#{path}: unreadable manifest" unless File.lstat(path).file?
|
|
60
|
+
|
|
61
|
+
validate(parse(slurp))
|
|
62
|
+
rescue SystemCallError
|
|
63
|
+
raise Mismatch, "#{path}: unreadable manifest"
|
|
64
|
+
end
|
|
65
|
+
|
|
66
|
+
def slurp
|
|
67
|
+
File.open(path, File::RDONLY | File::NONBLOCK | File::NOFOLLOW) do |io|
|
|
68
|
+
text = io.read(MAX_BYTES) || +""
|
|
69
|
+
raise Mismatch, "#{path}: manifest too large" unless io.eof?
|
|
70
|
+
|
|
71
|
+
text
|
|
72
|
+
end
|
|
73
|
+
end
|
|
74
|
+
|
|
75
|
+
def parse(text)
|
|
76
|
+
raise Mismatch, "#{path}: unreadable manifest" unless text.force_encoding(Encoding::UTF_8).valid_encoding?
|
|
77
|
+
|
|
78
|
+
JSON.parse(text)
|
|
79
|
+
rescue JSON::ParserError
|
|
80
|
+
raise Mismatch, "#{path}: unreadable manifest"
|
|
81
|
+
end
|
|
82
|
+
|
|
83
|
+
def validate(data)
|
|
84
|
+
raise Mismatch, "#{path}: unsupported manifest format" unless
|
|
85
|
+
data.is_a?(Hash) && data.keys.sort == KEYS && data["format"] == FORMAT && bounded?(data)
|
|
86
|
+
|
|
87
|
+
data
|
|
88
|
+
end
|
|
89
|
+
|
|
90
|
+
def bounded?(data)
|
|
91
|
+
%w[identity version].all? { |key| data[key].is_a?(String) && data[key].length <= MAX_FIELD }
|
|
92
|
+
end
|
|
93
|
+
|
|
94
|
+
# Written to a private temp file and linked into place: link fails
|
|
95
|
+
# when another run created the manifest first, and then that run's
|
|
96
|
+
# manifest is the one compared.
|
|
97
|
+
def write
|
|
98
|
+
FileUtils.mkdir_p(@directory)
|
|
99
|
+
temp = "#{path}.#{Process.pid}.#{SecureRandom.hex(4)}.tmp"
|
|
100
|
+
File.write(temp, JSON.generate(format: FORMAT, version: VERSION, identity: @identity))
|
|
101
|
+
File.link(temp, path)
|
|
102
|
+
nil
|
|
103
|
+
rescue Errno::EEXIST
|
|
104
|
+
compare(read)
|
|
105
|
+
ensure
|
|
106
|
+
FileUtils.rm_f(temp) if temp
|
|
107
|
+
end
|
|
108
|
+
end
|
|
109
|
+
end
|
|
110
|
+
end
|
|
@@ -35,8 +35,11 @@ module Saboteur
|
|
|
35
35
|
|
|
36
36
|
# Absolute monotonic deadline shared with the schedulers (forked
|
|
37
37
|
# workers inherit it): a subject's remaining mutants stop once it
|
|
38
|
-
# passes
|
|
39
|
-
#
|
|
38
|
+
# passes. New mutation work never overshoots more than the fork in
|
|
39
|
+
# flight; a subject the deadline cut mid-list still confirms the
|
|
40
|
+
# kills it already measured (#588), and those checks and retries
|
|
41
|
+
# fork uncapped - the overshoot is bounded by the subject's
|
|
42
|
+
# distinct killing sequences times their full timeout.
|
|
40
43
|
def budget_deadline
|
|
41
44
|
config["max_duration"] && (@run_started + config["max_duration"])
|
|
42
45
|
end
|
|
@@ -61,11 +64,11 @@ module Saboteur
|
|
|
61
64
|
|
|
62
65
|
needed = per_pass - remaining
|
|
63
66
|
raise BudgetPhaseTimeout,
|
|
64
|
-
"
|
|
65
|
-
"
|
|
66
|
-
"
|
|
67
|
-
"#{
|
|
68
|
-
"
|
|
67
|
+
"re-run with --max-duration about #{needed.round}s higher, fewer --baseline-passes, or once " \
|
|
68
|
+
"without --max-duration so the coverage cache is built for later --incremental runs: " \
|
|
69
|
+
"the budget cannot cover the coverage map build - baseline passes took about " \
|
|
70
|
+
"#{per_pass.round}s each, only #{remaining.round(1)}s remain, and a cold build needs about " \
|
|
71
|
+
"one more full pass and keeps nothing when it is cut off"
|
|
69
72
|
end
|
|
70
73
|
|
|
71
74
|
# Boot and baselines can spend the budget by themselves: stop before
|
|
@@ -3,6 +3,7 @@
|
|
|
3
3
|
|
|
4
4
|
require "digest"
|
|
5
5
|
require "json"
|
|
6
|
+
require_relative "../config"
|
|
6
7
|
require_relative "../text"
|
|
7
8
|
require_relative "data_inputs"
|
|
8
9
|
|
|
@@ -13,7 +14,7 @@ module Saboteur
|
|
|
13
14
|
class RunFingerprint
|
|
14
15
|
FORMAT_VERSION = 1
|
|
15
16
|
MANIFESTS = %w[Gemfile gems.rb Gemfile.lock gems.locked .rspec .rspec-local].freeze
|
|
16
|
-
LOCATION_SETTINGS =
|
|
17
|
+
LOCATION_SETTINGS = Config::OUTPUT_KEYS
|
|
17
18
|
ENTRY_KINDS = %w[ruby manifest boot hook source_plugin extra data].freeze
|
|
18
19
|
EXCLUDED_DIRECTORIES = %w[.git vendor node_modules].freeze
|
|
19
20
|
Entry = Data.define(:path, :kind, :state, :sha256, :required)
|
|
@@ -39,8 +39,7 @@ module Saboteur
|
|
|
39
39
|
# same line twice (D's review, #178).
|
|
40
40
|
unless @sandbox_note_given
|
|
41
41
|
@sandbox_note_given = true
|
|
42
|
-
@diagnostics.warn(
|
|
43
|
-
"and canary only")
|
|
42
|
+
@diagnostics.warn(degraded_note(abi))
|
|
44
43
|
end
|
|
45
44
|
nil
|
|
46
45
|
end
|
|
@@ -55,11 +54,30 @@ module Saboteur
|
|
|
55
54
|
end
|
|
56
55
|
|
|
57
56
|
def self.sandbox_gap(abi)
|
|
58
|
-
|
|
57
|
+
if abi.zero?
|
|
58
|
+
# Landlock is Linux-only: on macOS there is no kernel to upgrade,
|
|
59
|
+
# so name the platform instead of a Linux version (#667).
|
|
60
|
+
return "macOS has no Landlock" if Execution::Seatbelt.darwin?
|
|
61
|
+
|
|
62
|
+
return "this kernel lacks Landlock (needs Linux 6.2+)"
|
|
63
|
+
end
|
|
59
64
|
|
|
60
65
|
"this kernel's Landlock ABI is #{abi} but ABI 3 (Linux 6.2+) is needed to restrict truncate()"
|
|
61
66
|
end
|
|
62
67
|
|
|
68
|
+
# The degraded-run note: on macOS the gap is permanent, so point at
|
|
69
|
+
# the Seatbelt opt-in; on Linux the kernel wording stands alone
|
|
70
|
+
# (#667). Severity stays a warning - degrading confinement is never
|
|
71
|
+
# routine.
|
|
72
|
+
def degraded_note(abi)
|
|
73
|
+
gap = self.class.sandbox_gap(abi)
|
|
74
|
+
tail = "children run with the hazard guard and canary only"
|
|
75
|
+
return "sandbox is on but #{gap}; #{tail}" unless Execution::Seatbelt.darwin?
|
|
76
|
+
|
|
77
|
+
"sandbox is on but #{gap}; #{tail} - opt into the macOS sandbox with c.sandbox = \"seatbelt\" " \
|
|
78
|
+
"(docs/guides/macos-seatbelt-sandbox.md), or set c.sandbox = false to silence this"
|
|
79
|
+
end
|
|
80
|
+
|
|
63
81
|
def restrict_hook
|
|
64
82
|
writable = sandbox_writable_dirs
|
|
65
83
|
->(keep_fds) { Execution::Sandbox.restrict!(writable: writable, keep_fds: keep_fds) }
|
|
@@ -126,7 +144,8 @@ module Saboteur
|
|
|
126
144
|
def check_sandbox_write_set(all_dirs, user_dirs)
|
|
127
145
|
holder = all_dirs.find { |dir| path_contains?(dir, @root) }
|
|
128
146
|
if holder
|
|
129
|
-
message = "sandbox grant #{holder}
|
|
147
|
+
message = "narrow the sandbox grant #{holder} so it no longer contains the project root\n " \
|
|
148
|
+
"sandbox grant #{holder} contains the project root; mutant children can write the whole tree"
|
|
130
149
|
raise Error, %(#{message} - refusing under c.sandbox = "strict") if @config["sandbox"] == "strict"
|
|
131
150
|
|
|
132
151
|
@diagnostics.warn(message)
|
|
@@ -135,7 +154,9 @@ module Saboteur
|
|
|
135
154
|
(all_dirs - [dir]).each do |other|
|
|
136
155
|
next unless path_contains?(dir, other)
|
|
137
156
|
|
|
138
|
-
|
|
157
|
+
safe_other = Text.display_safe(other)
|
|
158
|
+
@diagnostics.warn("drop the redundant sandbox grant #{safe_other}\n " \
|
|
159
|
+
"sandbox grant #{Text.display_safe(dir)} contains the granted dir #{safe_other}; " \
|
|
139
160
|
"the narrower grant is redundant")
|
|
140
161
|
end
|
|
141
162
|
end
|