saboteur 0.8.1 → 0.9.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +100 -0
- data/README.md +10 -0
- data/exe/saboteur +6 -2
- data/lib/saboteur/attribute_finder.rb +3 -0
- data/lib/saboteur/baseline/run_evidence.rb +2 -1
- data/lib/saboteur/cli/agent_provenance.rb +59 -0
- data/lib/saboteur/cli/dirty_tree_notice.rb +28 -0
- data/lib/saboteur/cli/doctor_command.rb +41 -0
- data/lib/saboteur/cli/error_cause.rb +40 -0
- data/lib/saboteur/cli/execution_options.rb +46 -0
- data/lib/saboteur/cli/exit_status.rb +12 -14
- data/lib/saboteur/cli/explicit_file_options.rb +21 -0
- data/lib/saboteur/cli/explicit_sources.rb +46 -0
- data/lib/saboteur/cli/failure_output.rb +35 -0
- data/lib/saboteur/cli/help_command.rb +23 -0
- data/lib/saboteur/cli/matrix_option.rb +17 -0
- data/lib/saboteur/cli/output_options.rb +27 -0
- data/lib/saboteur/cli/preview_command.rb +26 -0
- data/lib/saboteur/cli/run_budget_options.rb +3 -0
- data/lib/saboteur/cli/run_command.rb +33 -0
- data/lib/saboteur/cli/run_output.rb +38 -0
- data/lib/saboteur/cli/run_reporting.rb +6 -5
- data/lib/saboteur/cli/run_validation.rb +54 -0
- data/lib/saboteur/cli.rb +29 -307
- data/lib/saboteur/config.rb +72 -6
- data/lib/saboteur/constant_subject_finder.rb +3 -0
- data/lib/saboteur/custom_map.rb +20 -12
- data/lib/saboteur/deep_source.rb +49 -0
- data/lib/saboteur/dirty_tree_warning.rb +1 -1
- data/lib/saboteur/doctor.rb +6 -0
- data/lib/saboteur/engine/baseline_guard.rb +50 -13
- data/lib/saboteur/engine/baseline_parts.rb +51 -0
- data/lib/saboteur/engine/boot.rb +4 -0
- data/lib/saboteur/engine/candidate_selection.rb +15 -5
- data/lib/saboteur/engine/candidates.rb +4 -1
- data/lib/saboteur/engine/coverage_inputs_snapshot.rb +146 -0
- data/lib/saboteur/engine/coverage_miss_note.rb +86 -0
- data/lib/saboteur/engine/coverage_partial.rb +48 -0
- data/lib/saboteur/engine/coverage_parts.rb +50 -0
- data/lib/saboteur/engine/coverage_shadow.rb +92 -0
- data/lib/saboteur/engine/coverage_store.rb +275 -22
- data/lib/saboteur/engine/diagnostics.rb +23 -6
- data/lib/saboteur/engine/distribution.rb +2 -2
- data/lib/saboteur/engine/gated_run.rb +1 -0
- data/lib/saboteur/engine/load_coverage.rb +54 -8
- data/lib/saboteur/engine/partial_invalidation.rb +115 -0
- data/lib/saboteur/engine/partial_merge.rb +40 -0
- data/lib/saboteur/engine/pipelines.rb +7 -7
- data/lib/saboteur/engine/reference_specs.rb +71 -0
- data/lib/saboteur/engine/result_cache.rb +68 -5
- data/lib/saboteur/engine/resume_manifest.rb +110 -0
- data/lib/saboteur/engine/run_budget.rb +5 -5
- data/lib/saboteur/engine/run_fingerprint.rb +2 -1
- data/lib/saboteur/engine/sandbox_policy.rb +5 -2
- data/lib/saboteur/engine/scheduler.rb +1 -1
- data/lib/saboteur/engine/selection_announcement.rb +1 -0
- data/lib/saboteur/engine/shadow_comparison.rb +121 -0
- data/lib/saboteur/engine/shadow_run.rb +38 -0
- data/lib/saboteur/engine/subject_selection.rb +33 -37
- data/lib/saboteur/engine/suite_partition.rb +22 -0
- data/lib/saboteur/engine/test_guess.rb +82 -18
- data/lib/saboteur/engine/unchanged_syntax.rb +10 -2
- data/lib/saboteur/engine.rb +41 -9
- data/lib/saboteur/environment_hints.rb +55 -0
- data/lib/saboteur/execution/bundle_gemfile_pin.rb +72 -0
- data/lib/saboteur/execution/external_runner.rb +105 -1
- data/lib/saboteur/execution/gemfile_path_sources.rb +166 -0
- data/lib/saboteur/execution/sandbox.rb +30 -17
- data/lib/saboteur/execution/supervisor.rb +40 -7
- data/lib/saboteur/execution/supervisor_batch.rb +105 -0
- data/lib/saboteur/execution/worker_pool.rb +48 -4
- data/lib/saboteur/frameworks/rails.rb +2 -5
- data/lib/saboteur/gates.rb +121 -28
- data/lib/saboteur/install.rb +6 -1
- data/lib/saboteur/integration/base.rb +19 -2
- data/lib/saboteur/integration/rspec.rb +46 -7
- data/lib/saboteur/line_hits.rb +81 -0
- data/lib/saboteur/method_calls.rb +91 -0
- data/lib/saboteur/mutation_source/builtin.rb +32 -22
- data/lib/saboteur/mutation_source/condition_negation.rb +3 -12
- data/lib/saboteur/mutation_source/jump_value.rb +59 -0
- data/lib/saboteur/mutation_source/literal_replacement.rb +6 -23
- data/lib/saboteur/mutation_source/operator_examples.rb +5 -0
- data/lib/saboteur/mutation_source/operator_registry.rb +5 -0
- data/lib/saboteur/mutation_source/operator_visit.rb +7 -1
- data/lib/saboteur/mutation_source/rescue_removal.rb +69 -0
- data/lib/saboteur/mutation_source/unary_removal.rb +47 -0
- data/lib/saboteur/mutation_source/when_removal.rb +51 -0
- data/lib/saboteur/preview.rb +9 -1
- data/lib/saboteur/refinement_activations.rb +22 -10
- data/lib/saboteur/reporter/agent.rb +2 -2
- data/lib/saboteur/reporter/agent_incomplete.rb +11 -0
- data/lib/saboteur/reporter/agent_summary.rb +16 -10
- data/lib/saboteur/reporter/review_comments.rb +151 -0
- data/lib/saboteur/reporter/safe_out.rb +32 -0
- data/lib/saboteur/reporter/stryker.rb +8 -6
- data/lib/saboteur/reporter/survivor_classification.rb +41 -3
- data/lib/saboteur/reporter/survivor_hints.rb +5 -0
- data/lib/saboteur/reporter/terminal.rb +29 -4
- data/lib/saboteur/result.rb +14 -4
- data/lib/saboteur/selection/coverage_map.rb +136 -10
- data/lib/saboteur/selection/external_test_map.rb +4 -0
- data/lib/saboteur/selection/git_diff.rb +24 -2
- data/lib/saboteur/selection/layout_only_lines.rb +45 -1
- data/lib/saboteur/selection/recorder.rb +52 -22
- data/lib/saboteur/selection/trace_recorder.rb +83 -0
- data/lib/saboteur/selection/untracked_changes.rb +1 -2
- data/lib/saboteur/subject.rb +16 -0
- data/lib/saboteur/subject_finder.rb +91 -6
- data/lib/saboteur/text.rb +31 -0
- data/lib/saboteur/version.rb +1 -1
- data/lib/saboteur/working_tree.rb +7 -2
- data/lib/saboteur.rb +8 -0
- metadata +44 -2
|
@@ -0,0 +1,40 @@
|
|
|
1
|
+
# SPDX-License-Identifier: MIT
|
|
2
|
+
# frozen_string_literal: true
|
|
3
|
+
|
|
4
|
+
module Saboteur
|
|
5
|
+
class Engine
|
|
6
|
+
# The map a partial rebuild produces (#46, step 3): the base map with all
|
|
7
|
+
# evidence of the rerun spec files removed, plus the fresh map the child
|
|
8
|
+
# recorded over just those files. Whole spec files are replaced, never
|
|
9
|
+
# single examples, since a file's example ids can shift when it changes.
|
|
10
|
+
# Load-time evidence, loaded files and preloaded files carry over from
|
|
11
|
+
# both: every run boots the same app.
|
|
12
|
+
module PartialMerge
|
|
13
|
+
module_function
|
|
14
|
+
|
|
15
|
+
def call(base, fresh, spec_files)
|
|
16
|
+
specs = spec_files.to_set
|
|
17
|
+
dropped = base.tests.select { |id| specs.include?(PartialInvalidation.spec_of(id)) }.to_set
|
|
18
|
+
trimmed = Selection::CoverageMap.new(
|
|
19
|
+
lines: without(base.lines, dropped), methods: without(base.methods, dropped),
|
|
20
|
+
branches: without(base.branches, dropped), tests: base.tests.reject { |id| dropped.include?(id) },
|
|
21
|
+
durations: base.durations.except(*dropped),
|
|
22
|
+
load_lines: base.load_lines, load_methods: base.load_methods,
|
|
23
|
+
loaded_files: base.loaded_files, preloaded: base.preloaded.to_a, build_seconds: base.build_seconds
|
|
24
|
+
)
|
|
25
|
+
Selection::CoverageMap.combine([trimmed, fresh])
|
|
26
|
+
end
|
|
27
|
+
|
|
28
|
+
# {file => {line-or-span => ids}} without the dropped ids. An entry left
|
|
29
|
+
# with no test is removed, and so is a file left with no entry: a file
|
|
30
|
+
# with no test behind it must read as no evidence, not as covered.
|
|
31
|
+
def without(hits, dropped)
|
|
32
|
+
hits.each_with_object({}) do |(file, by_line), acc|
|
|
33
|
+
kept = by_line.transform_values { |ids| ids.reject { |id| dropped.include?(id) } }
|
|
34
|
+
kept.reject! { |_, ids| ids.empty? }
|
|
35
|
+
acc[file] = kept unless kept.empty?
|
|
36
|
+
end
|
|
37
|
+
end
|
|
38
|
+
end
|
|
39
|
+
end
|
|
40
|
+
end
|
|
@@ -24,7 +24,7 @@ module Saboteur
|
|
|
24
24
|
mutate(subjects, files, integration, scope, ignore_list,
|
|
25
25
|
load_lines:, load_methods:)
|
|
26
26
|
end
|
|
27
|
-
@diagnostics.
|
|
27
|
+
@diagnostics.add_warnings(ignore_list.apply(results, unscoped_digests: @unscoped_digests))
|
|
28
28
|
build_report(results, subjects, scope, started)
|
|
29
29
|
ensure
|
|
30
30
|
# Cleanup runs however the call ends - a partway boot failure, no
|
|
@@ -46,15 +46,15 @@ module Saboteur
|
|
|
46
46
|
@diagnostics.inform("external test command: #{config['test_command']} " \
|
|
47
47
|
"(serial, no coverage, unsandboxed, mutants applied in a disposable copy)")
|
|
48
48
|
if Execution::TestCommandShape.multi_file_ruby?(config["test_command"])
|
|
49
|
-
@diagnostics.warn("the
|
|
50
|
-
"
|
|
51
|
-
"
|
|
52
|
-
"
|
|
49
|
+
@diagnostics.warn("use the framework's multi-file form (bin/rails test f1 f2, rspec f1 f2) or a " \
|
|
50
|
+
"shell loop (for f in ...; do ruby \"$f\" || exit 1; done)\n " \
|
|
51
|
+
"the test command hands several test files to one ruby: ruby executes only " \
|
|
52
|
+
"the first, so the rest never run and their mutants silently survive (#611)")
|
|
53
53
|
end
|
|
54
54
|
files = config.included_files
|
|
55
55
|
subjects, scope = select_subjects(files)
|
|
56
56
|
results = subjects.empty? ? [] : mutate_external(subjects, scope, ignore_list)
|
|
57
|
-
@diagnostics.
|
|
57
|
+
@diagnostics.add_warnings(ignore_list.apply(results, unscoped_digests: @unscoped_digests))
|
|
58
58
|
build_report(results, subjects, scope, started)
|
|
59
59
|
end
|
|
60
60
|
|
|
@@ -86,7 +86,7 @@ module Saboteur
|
|
|
86
86
|
end
|
|
87
87
|
run_candidates(runnable_subjects, runnable_lists, scheduler, nil, ignore_list)
|
|
88
88
|
ensure
|
|
89
|
-
runner
|
|
89
|
+
runner&.close
|
|
90
90
|
end
|
|
91
91
|
|
|
92
92
|
# Warn loudly when the command cannot see mutants at all: an
|
|
@@ -0,0 +1,71 @@
|
|
|
1
|
+
# SPDX-License-Identifier: MIT
|
|
2
|
+
# frozen_string_literal: true
|
|
3
|
+
|
|
4
|
+
module Saboteur
|
|
5
|
+
class Engine
|
|
6
|
+
# The specs that read a constant a source file defines, or require it: the
|
|
7
|
+
# cold-start choice for a file with no spec by name. The constant match is
|
|
8
|
+
# the lexical matching #538 uses for changed specs; the require match
|
|
9
|
+
# covers a file that defines only top-level methods. A file no spec reads or requires, or one read by more than
|
|
10
|
+
# CAP specs (a hub: the choice is not selective), gets no entry.
|
|
11
|
+
class ReferenceSpecs
|
|
12
|
+
CAP = 10
|
|
13
|
+
REQUIRE = /^\s*(require_relative|require)[ (]\s*["']([^"']+)["']/
|
|
14
|
+
|
|
15
|
+
# constants: file => the constants it defines, as TestGuess shapes them
|
|
16
|
+
# for ConstantReads.
|
|
17
|
+
def initialize(root, constants)
|
|
18
|
+
@root = root
|
|
19
|
+
@constants = constants
|
|
20
|
+
end
|
|
21
|
+
|
|
22
|
+
# chosen: file => sorted root-relative spec paths; crowded: file => how
|
|
23
|
+
# many specs read it, for the files over the cap (no choice was made).
|
|
24
|
+
Found = Data.define(:chosen, :crowded)
|
|
25
|
+
|
|
26
|
+
# The announcement for the files that got specs by reference.
|
|
27
|
+
def self.note(referenced)
|
|
28
|
+
named = referenced.map { |file, specs| "#{file} (#{specs.join(', ')})" }
|
|
29
|
+
"inferred specs: selected by constant reference or require, not by name: #{named.join(', ')}; " \
|
|
30
|
+
"a spec that reaches the file another way is not found"
|
|
31
|
+
end
|
|
32
|
+
|
|
33
|
+
def self.crowd_note(count) = "#{count} specs reference or require it, over the cap of #{CAP}"
|
|
34
|
+
|
|
35
|
+
# files: root-relative source files; spec_paths: root-relative spec
|
|
36
|
+
# files of the suite.
|
|
37
|
+
def call(files, spec_paths)
|
|
38
|
+
absolute = spec_paths.map { |path| File.join(@root, path) }
|
|
39
|
+
absolute.select! { |path| File.file?(path) && !File.symlink?(path) }
|
|
40
|
+
found = files.to_h { |file| [file, readers_of(file, absolute)] }.reject { |_, readers| readers.empty? }
|
|
41
|
+
crowded, chosen = found.partition { |_, readers| readers.size > CAP }.map(&:to_h)
|
|
42
|
+
Found.new(chosen: chosen, crowded: crowded.transform_values(&:size))
|
|
43
|
+
end
|
|
44
|
+
|
|
45
|
+
private
|
|
46
|
+
|
|
47
|
+
def readers_of(file, absolute_specs)
|
|
48
|
+
read = @constants.call(file).flat_map { |constant| ConstantReads.across(absolute_specs, constant).keys }
|
|
49
|
+
(read + requirers_of(file, absolute_specs)).uniq.sort.map { |path| path.delete_prefix("#{@root}/") }
|
|
50
|
+
end
|
|
51
|
+
|
|
52
|
+
# Specs that require the file: by path from the spec (require_relative)
|
|
53
|
+
# or by load path from the file's source root (require "a/b" for
|
|
54
|
+
# lib/a/b.rb). A bare basename is not a load path, so it matches nothing.
|
|
55
|
+
def requirers_of(file, absolute_specs)
|
|
56
|
+
stem = File.join(@root, file).delete_suffix(".rb")
|
|
57
|
+
loadable = file.delete_suffix(".rb").sub(%r{\A[^/]+/}, "")
|
|
58
|
+
absolute_specs.select { |spec| requires?(spec, stem, loadable) }
|
|
59
|
+
end
|
|
60
|
+
|
|
61
|
+
def requires?(spec, stem, loadable)
|
|
62
|
+
File.read(spec, encoding: "UTF-8").scrub.scan(REQUIRE).any? do |kind, target|
|
|
63
|
+
target = target.delete_suffix(".rb")
|
|
64
|
+
kind == "require_relative" ? File.expand_path(target, File.dirname(spec)) == stem : target == loadable
|
|
65
|
+
end
|
|
66
|
+
rescue SystemCallError
|
|
67
|
+
false
|
|
68
|
+
end
|
|
69
|
+
end
|
|
70
|
+
end
|
|
71
|
+
end
|
|
@@ -4,6 +4,7 @@
|
|
|
4
4
|
require "digest"
|
|
5
5
|
require "fileutils"
|
|
6
6
|
require "json"
|
|
7
|
+
require "securerandom"
|
|
7
8
|
require_relative "../text"
|
|
8
9
|
|
|
9
10
|
module Saboteur
|
|
@@ -25,6 +26,7 @@ module Saboteur
|
|
|
25
26
|
# stale.
|
|
26
27
|
class ResultCache
|
|
27
28
|
FORMAT = 1
|
|
29
|
+
MAX_BYTES = 64 * 1024 * 1024
|
|
28
30
|
STORED = %i[killed survived timeout].freeze
|
|
29
31
|
LOCKFILES = %w[Gemfile.lock gems.locked].freeze
|
|
30
32
|
# Dependency declarations themselves, so the no-lockfile library
|
|
@@ -34,9 +36,15 @@ module Saboteur
|
|
|
34
36
|
|
|
35
37
|
SOURCE_EXCLUDES = %w[.git vendor node_modules].freeze
|
|
36
38
|
|
|
39
|
+
# Seconds between checkpoint saves while verdicts arrive, so a run
|
|
40
|
+
# killed mid-phase (OOM, CI timeout, SIGKILL) keeps what it finished
|
|
41
|
+
# (#471).
|
|
42
|
+
SAVE_INTERVAL = 30
|
|
43
|
+
|
|
37
44
|
# suite_digest: SuiteFiles#digest, so an edit to a spec helper or
|
|
38
45
|
# support file invalidates every verdict.
|
|
39
|
-
def initialize(path:, root:, inputs:, suite_digest: "", coverage: nil, fallback_files: []
|
|
46
|
+
def initialize(path:, root:, inputs:, suite_digest: "", coverage: nil, fallback_files: [],
|
|
47
|
+
save_interval: SAVE_INTERVAL, clock: -> { Process.clock_gettime(Process::CLOCK_MONOTONIC) })
|
|
40
48
|
@path = path
|
|
41
49
|
@root = root
|
|
42
50
|
@inputs = inputs
|
|
@@ -47,6 +55,9 @@ module Saboteur
|
|
|
47
55
|
@fallback_files = fallback_files.map { |file| File.expand_path(file, root) }.sort
|
|
48
56
|
@entries = load
|
|
49
57
|
@file_digests = {}
|
|
58
|
+
@save_interval = save_interval
|
|
59
|
+
@clock = clock
|
|
60
|
+
@last_saved = clock.call
|
|
50
61
|
end
|
|
51
62
|
|
|
52
63
|
# The stored verdict as a hash (status, reason, killed_by, kill_cause,
|
|
@@ -67,15 +78,39 @@ module Saboteur
|
|
|
67
78
|
"reason" => result.reason, "killed_by" => result.killed_by, "kill_cause" => result.kill_cause,
|
|
68
79
|
"runtime" => result.runtime
|
|
69
80
|
}
|
|
81
|
+
checkpoint if @clock.call - @last_saved >= @save_interval
|
|
70
82
|
end
|
|
71
83
|
|
|
84
|
+
# Written to a temp file and renamed, so a kill mid-write leaves the
|
|
85
|
+
# previous complete file, never a truncated one. A failed write or
|
|
86
|
+
# rename removes the temp file and raises.
|
|
72
87
|
def save
|
|
73
88
|
FileUtils.mkdir_p(File.dirname(@path))
|
|
74
|
-
|
|
89
|
+
temp = "#{@path}.#{SecureRandom.hex(8)}.tmp"
|
|
90
|
+
begin
|
|
91
|
+
# "wx": a planted file or symlink at the temp name is never followed.
|
|
92
|
+
File.open(temp, "wx", 0o600) { |out| out.write(JSON.generate(format: FORMAT, entries: @entries)) }
|
|
93
|
+
File.rename(temp, @path)
|
|
94
|
+
rescue StandardError
|
|
95
|
+
FileUtils.rm_f(temp)
|
|
96
|
+
raise
|
|
97
|
+
end
|
|
98
|
+
@last_saved = @clock.call
|
|
75
99
|
end
|
|
76
100
|
|
|
77
101
|
private
|
|
78
102
|
|
|
103
|
+
# A mid-run save is a safety net, so it is best effort: a full disk or
|
|
104
|
+
# an unwritable directory must not abort a run that is still producing
|
|
105
|
+
# verdicts. The wait restarts either way, so a broken disk is tried
|
|
106
|
+
# once per interval, not once per verdict. The final save is not
|
|
107
|
+
# swallowed (Engine calls #save).
|
|
108
|
+
def checkpoint
|
|
109
|
+
save
|
|
110
|
+
rescue SystemCallError
|
|
111
|
+
@last_saved = @clock.call
|
|
112
|
+
end
|
|
113
|
+
|
|
79
114
|
# The current project source files, relative and sorted. A file added
|
|
80
115
|
# after the map was built changes the set and invalidates every stored
|
|
81
116
|
# verdict, whether or not any watched file references it yet.
|
|
@@ -86,13 +121,41 @@ module Saboteur
|
|
|
86
121
|
.join("\0")
|
|
87
122
|
end
|
|
88
123
|
|
|
124
|
+
# The file is untrusted input (a restored cache, a --resume directory):
|
|
125
|
+
# read only as a regular file, never a symlink or FIFO, under a size
|
|
126
|
+
# cap, and kept only when entries is a Hash of Hashes.
|
|
89
127
|
def load
|
|
90
|
-
|
|
91
|
-
|
|
92
|
-
|
|
128
|
+
return {} unless File.lstat(@path).file?
|
|
129
|
+
|
|
130
|
+
text = slurp
|
|
131
|
+
return {} unless text
|
|
132
|
+
|
|
133
|
+
data = JSON.parse(Text.utf8_strict(text))
|
|
134
|
+
entries = data["entries"] if data.is_a?(Hash) && data["format"] == FORMAT
|
|
135
|
+
well_formed?(entries) ? entries : {}
|
|
136
|
+
rescue SystemCallError, JSON::ParserError, ArgumentError, EncodingError
|
|
93
137
|
{}
|
|
94
138
|
end
|
|
95
139
|
|
|
140
|
+
def slurp
|
|
141
|
+
File.open(@path, File::RDONLY | File::NONBLOCK | File::NOFOLLOW) do |io|
|
|
142
|
+
text = io.read(MAX_BYTES)
|
|
143
|
+
text if io.eof?
|
|
144
|
+
end
|
|
145
|
+
end
|
|
146
|
+
|
|
147
|
+
def well_formed?(entries)
|
|
148
|
+
entries.is_a?(Hash) && entries.values.all? { |entry| entry.is_a?(Hash) && typed?(entry) }
|
|
149
|
+
end
|
|
150
|
+
|
|
151
|
+
# The fields the reports read, typed so a crafted entry cannot crash
|
|
152
|
+
# them.
|
|
153
|
+
def typed?(entry)
|
|
154
|
+
STORED.map(&:to_s).include?(entry["status"]) && (entry["runtime"].nil? || entry["runtime"].is_a?(Numeric)) &&
|
|
155
|
+
%w[reason kill_cause].all? { |key| entry[key].nil? || entry[key].is_a?(String) } &&
|
|
156
|
+
entry["killed_by"].is_a?(Array) && entry["killed_by"].all?(String)
|
|
157
|
+
end
|
|
158
|
+
|
|
96
159
|
def key(subject, mutation)
|
|
97
160
|
"#{subject.id} #{mutation.digest(subject_start: subject.start_byte, original_source: subject.file_source)}"
|
|
98
161
|
end
|
|
@@ -0,0 +1,110 @@
|
|
|
1
|
+
# SPDX-License-Identifier: MIT
|
|
2
|
+
# frozen_string_literal: true
|
|
3
|
+
|
|
4
|
+
require "fileutils"
|
|
5
|
+
require "json"
|
|
6
|
+
require "securerandom"
|
|
7
|
+
|
|
8
|
+
module Saboteur
|
|
9
|
+
class Engine
|
|
10
|
+
# Identity file for a --resume directory (#471). The verdicts live in
|
|
11
|
+
# ResultCache; this only records what the run was computed from, so a
|
|
12
|
+
# later run with other inputs fails loudly instead of reusing the
|
|
13
|
+
# directory under a different meaning.
|
|
14
|
+
#
|
|
15
|
+
# The file is untrusted input: it is read only when it is a regular
|
|
16
|
+
# file (never a FIFO or a symlink), under a size cap, and must hold
|
|
17
|
+
# exactly the three expected keys with bounded String values.
|
|
18
|
+
class ResumeManifest
|
|
19
|
+
FORMAT = 1
|
|
20
|
+
FILE = "manifest.json"
|
|
21
|
+
KEYS = %w[format identity version].freeze
|
|
22
|
+
MAX_BYTES = 4096
|
|
23
|
+
MAX_FIELD = 512
|
|
24
|
+
|
|
25
|
+
# Raised when the directory belongs to another run or cannot be read.
|
|
26
|
+
class Mismatch < StandardError; end
|
|
27
|
+
|
|
28
|
+
# identity: a String that changes whenever the run's inputs change
|
|
29
|
+
# (the RunFingerprint digest joined with the settings that pick the
|
|
30
|
+
# mutants), at most MAX_FIELD characters.
|
|
31
|
+
def initialize(directory:, identity:)
|
|
32
|
+
@directory = directory
|
|
33
|
+
@identity = identity
|
|
34
|
+
end
|
|
35
|
+
|
|
36
|
+
def path
|
|
37
|
+
File.join(@directory, FILE)
|
|
38
|
+
end
|
|
39
|
+
|
|
40
|
+
# Creates the manifest in a fresh directory, accepts a matching one,
|
|
41
|
+
# and raises Mismatch for anything else.
|
|
42
|
+
def verify!
|
|
43
|
+
File.lstat(path)
|
|
44
|
+
compare(read)
|
|
45
|
+
rescue Errno::ENOENT
|
|
46
|
+
write
|
|
47
|
+
end
|
|
48
|
+
|
|
49
|
+
private
|
|
50
|
+
|
|
51
|
+
def compare(stored)
|
|
52
|
+
return if stored["identity"] == @identity && stored["version"] == VERSION
|
|
53
|
+
|
|
54
|
+
raise Mismatch, "#{path}: the run inputs changed since this directory was created; " \
|
|
55
|
+
"delete it or choose a new --resume path"
|
|
56
|
+
end
|
|
57
|
+
|
|
58
|
+
def read
|
|
59
|
+
raise Mismatch, "#{path}: unreadable manifest" unless File.lstat(path).file?
|
|
60
|
+
|
|
61
|
+
validate(parse(slurp))
|
|
62
|
+
rescue SystemCallError
|
|
63
|
+
raise Mismatch, "#{path}: unreadable manifest"
|
|
64
|
+
end
|
|
65
|
+
|
|
66
|
+
def slurp
|
|
67
|
+
File.open(path, File::RDONLY | File::NONBLOCK | File::NOFOLLOW) do |io|
|
|
68
|
+
text = io.read(MAX_BYTES) || +""
|
|
69
|
+
raise Mismatch, "#{path}: manifest too large" unless io.eof?
|
|
70
|
+
|
|
71
|
+
text
|
|
72
|
+
end
|
|
73
|
+
end
|
|
74
|
+
|
|
75
|
+
def parse(text)
|
|
76
|
+
raise Mismatch, "#{path}: unreadable manifest" unless text.force_encoding(Encoding::UTF_8).valid_encoding?
|
|
77
|
+
|
|
78
|
+
JSON.parse(text)
|
|
79
|
+
rescue JSON::ParserError
|
|
80
|
+
raise Mismatch, "#{path}: unreadable manifest"
|
|
81
|
+
end
|
|
82
|
+
|
|
83
|
+
def validate(data)
|
|
84
|
+
raise Mismatch, "#{path}: unsupported manifest format" unless
|
|
85
|
+
data.is_a?(Hash) && data.keys.sort == KEYS && data["format"] == FORMAT && bounded?(data)
|
|
86
|
+
|
|
87
|
+
data
|
|
88
|
+
end
|
|
89
|
+
|
|
90
|
+
def bounded?(data)
|
|
91
|
+
%w[identity version].all? { |key| data[key].is_a?(String) && data[key].length <= MAX_FIELD }
|
|
92
|
+
end
|
|
93
|
+
|
|
94
|
+
# Written to a private temp file and linked into place: link fails
|
|
95
|
+
# when another run created the manifest first, and then that run's
|
|
96
|
+
# manifest is the one compared.
|
|
97
|
+
def write
|
|
98
|
+
FileUtils.mkdir_p(@directory)
|
|
99
|
+
temp = "#{path}.#{Process.pid}.#{SecureRandom.hex(4)}.tmp"
|
|
100
|
+
File.write(temp, JSON.generate(format: FORMAT, version: VERSION, identity: @identity))
|
|
101
|
+
File.link(temp, path)
|
|
102
|
+
nil
|
|
103
|
+
rescue Errno::EEXIST
|
|
104
|
+
compare(read)
|
|
105
|
+
ensure
|
|
106
|
+
FileUtils.rm_f(temp) if temp
|
|
107
|
+
end
|
|
108
|
+
end
|
|
109
|
+
end
|
|
110
|
+
end
|
|
@@ -64,11 +64,11 @@ module Saboteur
|
|
|
64
64
|
|
|
65
65
|
needed = per_pass - remaining
|
|
66
66
|
raise BudgetPhaseTimeout,
|
|
67
|
-
"
|
|
68
|
-
"
|
|
69
|
-
"
|
|
70
|
-
"#{
|
|
71
|
-
"
|
|
67
|
+
"re-run with --max-duration about #{needed.round}s higher, fewer --baseline-passes, or once " \
|
|
68
|
+
"without --max-duration so the coverage cache is built for later --incremental runs: " \
|
|
69
|
+
"the budget cannot cover the coverage map build - baseline passes took about " \
|
|
70
|
+
"#{per_pass.round}s each, only #{remaining.round(1)}s remain, and a cold build needs about " \
|
|
71
|
+
"one more full pass and keeps nothing when it is cut off"
|
|
72
72
|
end
|
|
73
73
|
|
|
74
74
|
# Boot and baselines can spend the budget by themselves: stop before
|
|
@@ -3,6 +3,7 @@
|
|
|
3
3
|
|
|
4
4
|
require "digest"
|
|
5
5
|
require "json"
|
|
6
|
+
require_relative "../config"
|
|
6
7
|
require_relative "../text"
|
|
7
8
|
require_relative "data_inputs"
|
|
8
9
|
|
|
@@ -13,7 +14,7 @@ module Saboteur
|
|
|
13
14
|
class RunFingerprint
|
|
14
15
|
FORMAT_VERSION = 1
|
|
15
16
|
MANIFESTS = %w[Gemfile gems.rb Gemfile.lock gems.locked .rspec .rspec-local].freeze
|
|
16
|
-
LOCATION_SETTINGS =
|
|
17
|
+
LOCATION_SETTINGS = Config::OUTPUT_KEYS
|
|
17
18
|
ENTRY_KINDS = %w[ruby manifest boot hook source_plugin extra data].freeze
|
|
18
19
|
EXCLUDED_DIRECTORIES = %w[.git vendor node_modules].freeze
|
|
19
20
|
Entry = Data.define(:path, :kind, :state, :sha256, :required)
|
|
@@ -144,7 +144,8 @@ module Saboteur
|
|
|
144
144
|
def check_sandbox_write_set(all_dirs, user_dirs)
|
|
145
145
|
holder = all_dirs.find { |dir| path_contains?(dir, @root) }
|
|
146
146
|
if holder
|
|
147
|
-
message = "sandbox grant #{holder}
|
|
147
|
+
message = "narrow the sandbox grant #{holder} so it no longer contains the project root\n " \
|
|
148
|
+
"sandbox grant #{holder} contains the project root; mutant children can write the whole tree"
|
|
148
149
|
raise Error, %(#{message} - refusing under c.sandbox = "strict") if @config["sandbox"] == "strict"
|
|
149
150
|
|
|
150
151
|
@diagnostics.warn(message)
|
|
@@ -153,7 +154,9 @@ module Saboteur
|
|
|
153
154
|
(all_dirs - [dir]).each do |other|
|
|
154
155
|
next unless path_contains?(dir, other)
|
|
155
156
|
|
|
156
|
-
|
|
157
|
+
safe_other = Text.display_safe(other)
|
|
158
|
+
@diagnostics.warn("drop the redundant sandbox grant #{safe_other}\n " \
|
|
159
|
+
"sandbox grant #{Text.display_safe(dir)} contains the granted dir #{safe_other}; " \
|
|
157
160
|
"the narrower grant is redundant")
|
|
158
161
|
end
|
|
159
162
|
end
|
|
@@ -337,7 +337,7 @@ module Saboteur
|
|
|
337
337
|
# Each mutant is checked separately: changing a body can change which
|
|
338
338
|
# definition a test reaches, so one unreachable verdict cannot skip later work.
|
|
339
339
|
def note_unreachable(plan, outcome)
|
|
340
|
-
@diagnostics.warn("#{plan.subject.id}: #{outcome.message}")
|
|
340
|
+
@diagnostics.warn("#{plan.subject.id}: a mutant never ran and is not scored\n #{outcome.message}")
|
|
341
341
|
end
|
|
342
342
|
|
|
343
343
|
# The text a known def-time wrapper needs re-evaluated with the def.
|
|
@@ -0,0 +1,121 @@
|
|
|
1
|
+
# SPDX-License-Identifier: MIT
|
|
2
|
+
# frozen_string_literal: true
|
|
3
|
+
|
|
4
|
+
module Saboteur
|
|
5
|
+
class Engine
|
|
6
|
+
# Compares the map a partial rebuild produced with the full map (#46,
|
|
7
|
+
# step 3). The full map is the one a run uses; this only says whether the
|
|
8
|
+
# partial path would have been right.
|
|
9
|
+
#
|
|
10
|
+
# Two things are compared, and both must match:
|
|
11
|
+
# - the semantic evidence: which tests exist and, per source file, which
|
|
12
|
+
# tests hit which lines, methods and branch arms, and which lines ran
|
|
13
|
+
# while the app loaded. Durations, build time and the order of ids are
|
|
14
|
+
# left out, as are entries with no test.
|
|
15
|
+
# - what the result cache would watch: for every test the source files
|
|
16
|
+
# it touched, plus the boot files and the loaded files nothing
|
|
17
|
+
# observed. Evidence that matches line by line but yields another
|
|
18
|
+
# dependency set is worse than a line mismatch, since the set decides
|
|
19
|
+
# whether an old verdict is reused.
|
|
20
|
+
module ShadowComparison
|
|
21
|
+
MAX_DIFFERENCES = 50
|
|
22
|
+
KINDS = %i[lines methods branches].freeze
|
|
23
|
+
|
|
24
|
+
module_function
|
|
25
|
+
|
|
26
|
+
# { result: :match | :superset | :mismatch, match: bool, total: Integer, extra: Integer,
|
|
27
|
+
# gained: Integer,
|
|
28
|
+
# differences: [{kind:, name:}] } with at most MAX_DIFFERENCES entries,
|
|
29
|
+
# sorted by kind and name.
|
|
30
|
+
#
|
|
31
|
+
# :superset means the partial map credits tests the full map does not,
|
|
32
|
+
# on lines the full map also credits, and never lacks a credit the
|
|
33
|
+
# full map has. Anything else that differs is :mismatch, including an
|
|
34
|
+
# extra credit on a line the full map calls uncovered: it would turn
|
|
35
|
+
# "no test covers this" into "covered". Those are counted apart, in
|
|
36
|
+
# gained. extra counts the other extra credits (one per file, line and
|
|
37
|
+
# test) in lines, methods and branches.
|
|
38
|
+
def call(partial, full, fallback_files: [])
|
|
39
|
+
maps = [partial, full]
|
|
40
|
+
found = evidence_differences(maps) + dependency_differences(maps, fallback_files)
|
|
41
|
+
counted = found.select { |item| KINDS.map(&:to_s).include?(item[:kind]) }
|
|
42
|
+
extra = counted.sum { |item| item[:extra] }
|
|
43
|
+
gained = counted.sum { |item| item[:gained] }
|
|
44
|
+
result = if found.empty? then :match
|
|
45
|
+
elsif gained.positive? || found.any? { |item| item[:missing] } then :mismatch
|
|
46
|
+
else :superset
|
|
47
|
+
end
|
|
48
|
+
shown = found.sort_by { |item| [item[:kind], item[:name]] }.first(MAX_DIFFERENCES)
|
|
49
|
+
{ result: result, match: result == :match, total: found.size, extra: extra, gained: gained,
|
|
50
|
+
differences: shown.map { |item| item.slice(:kind, :name) } }
|
|
51
|
+
end
|
|
52
|
+
|
|
53
|
+
def evidence_differences(maps)
|
|
54
|
+
partial, full = maps
|
|
55
|
+
found = []
|
|
56
|
+
unless partial.tests.sort == full.tests.sort
|
|
57
|
+
found << { kind: "tests", name: "the set of test ids", missing: true }
|
|
58
|
+
end
|
|
59
|
+
KINDS.each { |kind| found.concat(hit_differences(kind.to_s, hits(partial, kind), hits(full, kind))) }
|
|
60
|
+
found.concat(exact_differences("load", load_hits(partial), load_hits(full)))
|
|
61
|
+
end
|
|
62
|
+
|
|
63
|
+
# Per name: the credits only the partial map has (extra) and the ones
|
|
64
|
+
# only the full map has (missing).
|
|
65
|
+
def hit_differences(kind, left, right)
|
|
66
|
+
(left.keys | right.keys).filter_map do |name|
|
|
67
|
+
extra, gained, missing = credits(left[name] || {}, right[name] || {})
|
|
68
|
+
next if extra.zero? && gained.zero? && !missing
|
|
69
|
+
|
|
70
|
+
{ kind: kind, name: name, extra: extra, gained: gained, missing: missing }
|
|
71
|
+
end
|
|
72
|
+
end
|
|
73
|
+
|
|
74
|
+
def credits(left, right)
|
|
75
|
+
extra = gained = 0
|
|
76
|
+
missing = false
|
|
77
|
+
(left.keys | right.keys).each do |line|
|
|
78
|
+
mine = left[line] || []
|
|
79
|
+
theirs = right[line] || []
|
|
80
|
+
(theirs.empty? ? gained += mine.size : extra += (mine - theirs).size)
|
|
81
|
+
missing ||= (theirs - mine).any?
|
|
82
|
+
end
|
|
83
|
+
[extra, gained, missing]
|
|
84
|
+
end
|
|
85
|
+
|
|
86
|
+
def exact_differences(kind, left, right)
|
|
87
|
+
(left.keys | right.keys).reject { |name| left[name] == right[name] }
|
|
88
|
+
.map { |name| { kind: kind, name: name, missing: true } }
|
|
89
|
+
end
|
|
90
|
+
|
|
91
|
+
# {file => {line-or-span => sorted ids}} without empty entries.
|
|
92
|
+
def hits(map, kind)
|
|
93
|
+
map.public_send(kind).each_with_object({}) do |(file, by_line), acc|
|
|
94
|
+
kept = by_line.transform_values(&:sort).reject { |_line, ids| ids.empty? }
|
|
95
|
+
acc[file] = kept unless kept.empty?
|
|
96
|
+
end
|
|
97
|
+
end
|
|
98
|
+
|
|
99
|
+
def load_hits(map)
|
|
100
|
+
[map.load_lines, map.load_methods].flat_map(&:to_a).each_with_object({}) do |(file, lines), acc|
|
|
101
|
+
acc[file] = (acc.fetch(file, []) | lines).sort unless lines.empty?
|
|
102
|
+
end
|
|
103
|
+
end
|
|
104
|
+
|
|
105
|
+
# The files every verdict depends on must match exactly. What each
|
|
106
|
+
# test depends on follows from its line and method credits, which are
|
|
107
|
+
# compared above, so it is not compared again.
|
|
108
|
+
def dependency_differences(maps, fallback_files)
|
|
109
|
+
partial, full = maps
|
|
110
|
+
left = dependencies(partial, fallback_files)
|
|
111
|
+
right = dependencies(full, fallback_files)
|
|
112
|
+
left.keys.reject { |name| left[name] == right[name] }.map { |name| { kind: "deps", name: name, missing: true } }
|
|
113
|
+
end
|
|
114
|
+
|
|
115
|
+
def dependencies(map, fallback_files)
|
|
116
|
+
unobserved = (fallback_files | map.loaded_files).select { |file| map.empty_for?([file]) }.sort
|
|
117
|
+
{ "(boot files)" => map.boot_files, "(unobserved files)" => unobserved }
|
|
118
|
+
end
|
|
119
|
+
end
|
|
120
|
+
end
|
|
121
|
+
end
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
# SPDX-License-Identifier: MIT
|
|
2
|
+
# frozen_string_literal: true
|
|
3
|
+
|
|
4
|
+
module Saboteur
|
|
5
|
+
class Engine
|
|
6
|
+
# One shadow comparison (#46, step 3): would a partial rebuild from the
|
|
7
|
+
# last map have matched the full map this run just built? The run uses
|
|
8
|
+
# the full map either way; this only produces the report.
|
|
9
|
+
#
|
|
10
|
+
# The report always carries the share threshold and, when the partial
|
|
11
|
+
# path was possible, the observed stale share, so the constant can be
|
|
12
|
+
# fitted from data later.
|
|
13
|
+
module ShadowRun
|
|
14
|
+
module_function
|
|
15
|
+
|
|
16
|
+
# base: the earlier inputs snapshot and its map. current: the inputs
|
|
17
|
+
# snapshot of this run. rerun.call(stale_spec_files, seed) builds a map
|
|
18
|
+
# over just those files. clean turns a name into a line-safe string.
|
|
19
|
+
def call(base:, base_map:, current:, full_map:, invalidation:, fallback_files:, rerun:, clean:)
|
|
20
|
+
report = { "threshold" => PartialInvalidation::THRESHOLD, "base_map" => base["map_key"] }
|
|
21
|
+
stale = invalidation.stale_specs(base, current)
|
|
22
|
+
return report.merge("result" => "skipped", "reason" => "a full rebuild is needed") if stale == :full
|
|
23
|
+
|
|
24
|
+
merged = stale.empty? ? base_map : PartialMerge.call(base_map, rerun.call(stale, base["seed"]), stale)
|
|
25
|
+
verdict = ShadowComparison.call(merged, full_map, fallback_files: fallback_files)
|
|
26
|
+
report.merge("result" => verdict[:result].to_s, "extra_credits" => verdict[:extra],
|
|
27
|
+
"gained_credits" => verdict[:gained],
|
|
28
|
+
"stale" => stale.map(&clean),
|
|
29
|
+
"stale_share" => invalidation.share(stale), "total" => verdict[:total],
|
|
30
|
+
"differences" => verdict[:differences].map do |item|
|
|
31
|
+
item.to_h do |key, value|
|
|
32
|
+
[key.to_s, clean.call(value)]
|
|
33
|
+
end
|
|
34
|
+
end)
|
|
35
|
+
end
|
|
36
|
+
end
|
|
37
|
+
end
|
|
38
|
+
end
|