saboteur 0.5.0 → 0.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +399 -0
- data/README.md +202 -42
- data/exe/saboteur +34 -7
- data/lib/saboteur/baseline/comparison.rb +1 -0
- data/lib/saboteur/baseline/evidence.rb +11 -1
- data/lib/saboteur/baseline/run_evidence.rb +1 -1
- data/lib/saboteur/baseline/workflow.rb +2 -1
- data/lib/saboteur/baseline_health.rb +40 -6
- data/lib/saboteur/cli/exit_status.rb +49 -4
- data/lib/saboteur/cli/help_options.rb +16 -0
- data/lib/saboteur/cli/presets.rb +36 -0
- data/lib/saboteur/cli/run_budget_options.rb +37 -0
- data/lib/saboteur/cli/run_reporting.rb +75 -6
- data/lib/saboteur/cli.rb +80 -48
- data/lib/saboteur/config.rb +88 -13
- data/lib/saboteur/crash_report.rb +154 -0
- data/lib/saboteur/custom_map.rb +3 -1
- data/lib/saboteur/doctor.rb +64 -2
- data/lib/saboteur/engine/baseline_guard.rb +79 -5
- data/lib/saboteur/engine/boot.rb +12 -1
- data/lib/saboteur/engine/canary.rb +41 -10
- data/lib/saboteur/engine/candidate_selection.rb +9 -1
- data/lib/saboteur/engine/candidates.rb +4 -1
- data/lib/saboteur/engine/coverage_guard.rb +29 -0
- data/lib/saboteur/engine/coverage_store.rb +48 -6
- data/lib/saboteur/engine/diagnostics.rb +23 -2
- data/lib/saboteur/engine/diff_scope.rb +24 -5
- data/lib/saboteur/engine/distribution.rb +124 -8
- data/lib/saboteur/engine/early_stop_recording.rb +46 -0
- data/lib/saboteur/engine/result_cache.rb +2 -1
- data/lib/saboteur/engine/run_budget.rb +63 -0
- data/lib/saboteur/engine/run_fingerprint.rb +5 -1
- data/lib/saboteur/engine/sandbox_policy.rb +18 -11
- data/lib/saboteur/engine/scheduler.rb +114 -18
- data/lib/saboteur/engine/subject_selection.rb +84 -5
- data/lib/saboteur/engine/suite_files.rb +3 -1
- data/lib/saboteur/engine/test_guess.rb +183 -23
- data/lib/saboteur/engine/thread_warnings.rb +53 -0
- data/lib/saboteur/engine/unscoped_suite_warning.rb +41 -0
- data/lib/saboteur/engine/verdict_check.rb +54 -18
- data/lib/saboteur/engine.rb +175 -96
- data/lib/saboteur/execution/protocol.rb +2 -1
- data/lib/saboteur/execution/runner.rb +179 -17
- data/lib/saboteur/execution/sandbox.rb +6 -1
- data/lib/saboteur/execution/supervisor.rb +18 -3
- data/lib/saboteur/execution/worker_pool.rb +87 -7
- data/lib/saboteur/frameworks/database_hosts.rb +30 -0
- data/lib/saboteur/frameworks/rails.rb +8 -10
- data/lib/saboteur/help.rb +55 -10
- data/lib/saboteur/ignore_list.rb +11 -1
- data/lib/saboteur/install.rb +4 -3
- data/lib/saboteur/integration/base.rb +7 -2
- data/lib/saboteur/integration/rspec.rb +40 -4
- data/lib/saboteur/mutation_source/builtin.rb +22 -13
- data/lib/saboteur/mutation_source/hazard_guard.rb +21 -3
- data/lib/saboteur/mutation_source/interpolation_removal.rb +41 -0
- data/lib/saboteur/mutation_source/method_swap.rb +45 -0
- data/lib/saboteur/mutation_source/mutation.rb +5 -3
- data/lib/saboteur/mutation_source/operand_replacement.rb +54 -0
- data/lib/saboteur/mutation_source/operator_examples.rb +38 -0
- data/lib/saboteur/mutation_source/operator_registry.rb +36 -8
- data/lib/saboteur/mutation_source/operator_visit.rb +39 -0
- data/lib/saboteur/preview.rb +12 -2
- data/lib/saboteur/reporter/agent_commands.rb +49 -16
- data/lib/saboteur/reporter/agent_incomplete.rb +27 -2
- data/lib/saboteur/reporter/agent_summary.rb +1 -1
- data/lib/saboteur/reporter/history_policy.rb +35 -0
- data/lib/saboteur/reporter/history_record.rb +4 -5
- data/lib/saboteur/reporter/next_steps.rb +14 -4
- data/lib/saboteur/reporter/score_history.rb +19 -3
- data/lib/saboteur/reporter/stryker.rb +19 -8
- data/lib/saboteur/reporter/survivor_hints.rb +3 -0
- data/lib/saboteur/reporter/terminal.rb +43 -10
- data/lib/saboteur/result.rb +25 -7
- data/lib/saboteur/selection/coverage_map.rb +2 -1
- data/lib/saboteur/selection/git_diff.rb +15 -5
- data/lib/saboteur/selection/untracked_changes.rb +126 -0
- data/lib/saboteur/subject.rb +11 -0
- data/lib/saboteur/text.rb +42 -0
- data/lib/saboteur/version.rb +1 -1
- data/lib/saboteur.rb +16 -1
- metadata +20 -2
|
@@ -2,6 +2,7 @@
|
|
|
2
2
|
# frozen_string_literal: true
|
|
3
3
|
|
|
4
4
|
require "open3"
|
|
5
|
+
require_relative "../text"
|
|
5
6
|
|
|
6
7
|
module Saboteur
|
|
7
8
|
class Engine
|
|
@@ -15,7 +16,9 @@ module Saboteur
|
|
|
15
16
|
GIT_DIR = ".git"
|
|
16
17
|
|
|
17
18
|
def initialize(root, cache_dir: ".saboteur")
|
|
18
|
-
|
|
19
|
+
# Retag losslessly: path bytes are identity here, and later
|
|
20
|
+
# File.join calls must not mix encodings.
|
|
21
|
+
@root = Text.utf8_identity(root.to_s)
|
|
19
22
|
@cache_dir = cache_dir.to_s
|
|
20
23
|
@snapshot = inventory
|
|
21
24
|
end
|
|
@@ -29,16 +32,40 @@ module Saboteur
|
|
|
29
32
|
|
|
30
33
|
of = " of #{subject.id}" if subject
|
|
31
34
|
culprit = digest ? " The most recent mutant, #{digest}#{of}, is the likely cause." : ""
|
|
32
|
-
raise Saboteur::CanaryTripped,
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
|
|
36
|
-
|
|
37
|
-
|
|
35
|
+
raise Saboteur::CanaryTripped, message(missing, culprit)
|
|
36
|
+
end
|
|
37
|
+
|
|
38
|
+
# The cheap per-mutant probe (#235): stats the snapshot paths only -
|
|
39
|
+
# no git, no glob - so it can run after every mutant.
|
|
40
|
+
def probe
|
|
41
|
+
@snapshot.reject { |rel| present?(rel) }
|
|
42
|
+
end
|
|
43
|
+
|
|
44
|
+
# Like check!, but for the per-mutant probe: the mutant whose run just
|
|
45
|
+
# ended is named, never the subject's most recent mutant.
|
|
46
|
+
def probe!(subject: nil, digest: nil)
|
|
47
|
+
missing = probe
|
|
48
|
+
return if missing.empty?
|
|
49
|
+
|
|
50
|
+
of = " of #{subject.id}" if subject
|
|
51
|
+
culprit = digest ? " The mutant that just ran, #{digest}#{of}, is the likely cause." : ""
|
|
52
|
+
raise Saboteur::CanaryTripped.new(message(missing, culprit), culprit_digest: digest)
|
|
38
53
|
end
|
|
39
54
|
|
|
40
55
|
private
|
|
41
56
|
|
|
57
|
+
def present?(rel)
|
|
58
|
+
File.exist?(File.join(@root, rel)) || File.symlink?(File.join(@root, rel))
|
|
59
|
+
end
|
|
60
|
+
|
|
61
|
+
def message(missing, culprit)
|
|
62
|
+
"canary: #{missing.size} watched file(s) vanished from the working tree while mutants ran " \
|
|
63
|
+
"(first: #{Text.utf8(missing.first)}).#{culprit} The canary watches tracked files and .git presence " \
|
|
64
|
+
"and only detects the loss after the fact; it cannot restore anything. Recover from your " \
|
|
65
|
+
"own backup, a fresh checkout, or a reclone - a deleted .git or uncommitted work is gone " \
|
|
66
|
+
"unless you have another copy."
|
|
67
|
+
end
|
|
68
|
+
|
|
42
69
|
def inventory
|
|
43
70
|
files = git? ? git_tracked_files : walk_files
|
|
44
71
|
files << GIT_DIR if File.exist?(File.join(@root, GIT_DIR))
|
|
@@ -54,15 +81,19 @@ module Saboteur
|
|
|
54
81
|
return walk_files unless status.success?
|
|
55
82
|
|
|
56
83
|
# ls-files reads the index, which still names a deleted file; keep
|
|
57
|
-
# only paths present in the worktree so a deletion shows up.
|
|
58
|
-
|
|
84
|
+
# only paths present in the worktree so a deletion shows up. Split
|
|
85
|
+
# the raw bytes first and retag each path losslessly: scrubbing a
|
|
86
|
+
# raw-byte name would make the File.exist? probe fail and drop the
|
|
87
|
+
# file from the watch, so its deletion would go unseen.
|
|
88
|
+
out.b.split("\0".b).map { |rel| Text.utf8_identity(rel) }
|
|
89
|
+
.reject { |rel| cache_path?(rel) }
|
|
59
90
|
.select { |rel| File.exist?(File.join(@root, rel)) || File.symlink?(File.join(@root, rel)) }
|
|
60
91
|
end
|
|
61
92
|
|
|
62
93
|
def walk_files
|
|
63
94
|
Dir.glob(File.join(@root, "**", "*"), File::FNM_DOTMATCH)
|
|
64
95
|
.select { |path| File.file?(path) }
|
|
65
|
-
.map { |path| path.delete_prefix("#{@root}/") }
|
|
96
|
+
.map { |path| Text.utf8_identity(path).delete_prefix("#{@root}/") }
|
|
66
97
|
.reject { |rel| rel.start_with?("#{GIT_DIR}/") || cache_path?(rel) }
|
|
67
98
|
end
|
|
68
99
|
|
|
@@ -6,7 +6,7 @@ module Saboteur
|
|
|
6
6
|
# Selects a bounded list for each covered subject before workers fork.
|
|
7
7
|
# Subject skips are not mutants and do not enter the mutation score.
|
|
8
8
|
class CandidateSelection
|
|
9
|
-
attr_reader :skipped_subjects, :selected
|
|
9
|
+
attr_reader :skipped_subjects, :selected, :unscoped_digests
|
|
10
10
|
|
|
11
11
|
def initialize(candidates:, map:, limit:, diagnostics:)
|
|
12
12
|
@candidates = candidates
|
|
@@ -15,6 +15,7 @@ module Saboteur
|
|
|
15
15
|
@diagnostics = diagnostics
|
|
16
16
|
@skipped_subjects = {}
|
|
17
17
|
@selected = {}
|
|
18
|
+
@unscoped_digests = {}
|
|
18
19
|
end
|
|
19
20
|
|
|
20
21
|
def for(subject, scope: nil)
|
|
@@ -25,6 +26,13 @@ module Saboteur
|
|
|
25
26
|
end
|
|
26
27
|
|
|
27
28
|
list = @candidates.for(subject)
|
|
29
|
+
# The full pre-scope digests let IgnoreList#apply tell "mutation gone"
|
|
30
|
+
# from "mutation scoped out of this run" (#284).
|
|
31
|
+
if scope
|
|
32
|
+
@unscoped_digests[subject.id] = list.map do |mutation, _|
|
|
33
|
+
mutation.digest(subject_start: subject.start_byte, original_source: subject.file_source)
|
|
34
|
+
end
|
|
35
|
+
end
|
|
28
36
|
list = scope.restrict(subject, list) if scope
|
|
29
37
|
omitted = list.size - @limit
|
|
30
38
|
if omitted.positive?
|
|
@@ -13,7 +13,10 @@ module Saboteur
|
|
|
13
13
|
def self.build_sources(config)
|
|
14
14
|
root = File.realpath(config.root)
|
|
15
15
|
config["sources"].map do |entry|
|
|
16
|
-
|
|
16
|
+
if entry == "builtin"
|
|
17
|
+
next MutationSource::Builtin.new(arid: config["arid"], operators: config["operators"],
|
|
18
|
+
exclude_operators: config["exclude_operators"])
|
|
19
|
+
end
|
|
17
20
|
|
|
18
21
|
file, const = entry.split(":", 2)
|
|
19
22
|
raise ConfigError, "mutation source #{entry.inspect} must be builtin or path.rb:ClassName" unless const
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
# SPDX-License-Identifier: MIT
|
|
2
|
+
# frozen_string_literal: true
|
|
3
|
+
|
|
4
|
+
module Saboteur
|
|
5
|
+
class Engine
|
|
6
|
+
# Owns the host process's Coverage state across a run: suspends after
|
|
7
|
+
# boot, and at the end resumes only its own suspension. A CLI process
|
|
8
|
+
# exits right after, so resume matters to hosts that embed the engine -
|
|
9
|
+
# a spec, or the dogfood's own coverage-map build (#254).
|
|
10
|
+
class CoverageGuard
|
|
11
|
+
def initialize(diagnostics)
|
|
12
|
+
@diagnostics = diagnostics
|
|
13
|
+
end
|
|
14
|
+
|
|
15
|
+
def suspend
|
|
16
|
+
@was_running = Coverage.state == :running
|
|
17
|
+
Coverage.suspend if @was_running
|
|
18
|
+
end
|
|
19
|
+
|
|
20
|
+
# Best-effort: a host that disturbed Coverage mid-run gets a warning,
|
|
21
|
+
# never a masked run outcome.
|
|
22
|
+
def resume
|
|
23
|
+
Coverage.resume if @was_running && Coverage.state == :suspended
|
|
24
|
+
rescue StandardError => e
|
|
25
|
+
@diagnostics.warn("coverage resume failed: #{e.class}: #{e.message}")
|
|
26
|
+
end
|
|
27
|
+
end
|
|
28
|
+
end
|
|
29
|
+
end
|
|
@@ -15,19 +15,29 @@ module Saboteur
|
|
|
15
15
|
LOCKFILES = %w[Gemfile.lock gems.locked].freeze
|
|
16
16
|
LOG = "coverage-build.log"
|
|
17
17
|
|
|
18
|
-
def initialize(config:, integration:, hooks:, supervisor:, diagnostics:, load_lines: {})
|
|
18
|
+
def initialize(config:, integration:, hooks:, supervisor:, diagnostics:, load_lines: {}, timeout_cap: nil)
|
|
19
19
|
@config = config
|
|
20
20
|
@integration = integration
|
|
21
21
|
@hooks = hooks
|
|
22
22
|
@supervisor = supervisor
|
|
23
23
|
@diagnostics = diagnostics
|
|
24
|
+
@timeout_cap = timeout_cap
|
|
24
25
|
@load_lines = load_lines
|
|
25
26
|
@root = File.realpath(config.root)
|
|
26
27
|
end
|
|
27
28
|
|
|
29
|
+
# True when fetch would reuse the cached map instead of running the
|
|
30
|
+
# suite pass that builds it: lets the run announce the real number of
|
|
31
|
+
# full passes up front (#260). Loads the map with the same validation
|
|
32
|
+
# fetch uses and retains it, so a corrupt or truncated cache announces
|
|
33
|
+
# a build pass, not a reuse it cannot keep.
|
|
34
|
+
def reusable?(files)
|
|
35
|
+
!cached_map(files).nil?
|
|
36
|
+
end
|
|
37
|
+
|
|
28
38
|
def fetch(files)
|
|
29
|
-
path =
|
|
30
|
-
cached =
|
|
39
|
+
path = cache_path(files)
|
|
40
|
+
cached = cached_map(files)
|
|
31
41
|
return refuse_empty_map(cached, files) if cached
|
|
32
42
|
|
|
33
43
|
map = refuse_empty_map(build(files), files)
|
|
@@ -39,6 +49,16 @@ module Saboteur
|
|
|
39
49
|
|
|
40
50
|
private
|
|
41
51
|
|
|
52
|
+
def cache_path(files)
|
|
53
|
+
File.join(@config.path("cache_dir"), "coverage-#{cache_key(files)}.json")
|
|
54
|
+
end
|
|
55
|
+
|
|
56
|
+
def cached_map(files)
|
|
57
|
+
return nil unless @config["cache"]
|
|
58
|
+
|
|
59
|
+
@cached_map ||= Selection::CoverageMap.load(cache_path(files))
|
|
60
|
+
end
|
|
61
|
+
|
|
42
62
|
def cache_key(files)
|
|
43
63
|
inputs = JSON.generate(CACHE_INPUTS.to_h { |key| [key, @config[key]] })
|
|
44
64
|
all = files + @integration.test_files + SuiteFiles.new(@config).files + lockfiles
|
|
@@ -49,6 +69,12 @@ module Saboteur
|
|
|
49
69
|
LOCKFILES.map { |file| File.join(@root, file) }.select { |file| File.file?(file) }
|
|
50
70
|
end
|
|
51
71
|
|
|
72
|
+
# An admitted build still gets only the time left in the run budget
|
|
73
|
+
# (see BaselineGuard#phase_timeout).
|
|
74
|
+
def build_timeout
|
|
75
|
+
@timeout_cap ? @timeout_cap.call(@config["coverage_timeout"]) : @config["coverage_timeout"]
|
|
76
|
+
end
|
|
77
|
+
|
|
52
78
|
def build(files)
|
|
53
79
|
unless Coverage.respond_to?(:state) && Coverage.state == :suspended
|
|
54
80
|
raise Error, "Coverage is #{Coverage.state}; saboteur must set it up before the app loads. " \
|
|
@@ -61,13 +87,16 @@ module Saboteur
|
|
|
61
87
|
# reduce later copying of shared pages.
|
|
62
88
|
Execution::ForkReadiness.prepare_heap
|
|
63
89
|
heartbeat = @diagnostics.progress_heartbeat("building coverage map").watch(log_path)
|
|
64
|
-
|
|
90
|
+
admitted = build_timeout
|
|
91
|
+
result = @supervisor.run(timeout: admitted, on_tick: heartbeat.method(:tick),
|
|
92
|
+
label: "coverage child died during the map build") do |writer|
|
|
65
93
|
record(writer, files)
|
|
66
94
|
end
|
|
67
|
-
map_from(result)
|
|
95
|
+
map_from(result, admitted)
|
|
68
96
|
end
|
|
69
97
|
|
|
70
|
-
def map_from(result)
|
|
98
|
+
def map_from(result, admitted = @config["coverage_timeout"])
|
|
99
|
+
raise_budget_stop(admitted) if budget_stopped?(result, admitted)
|
|
71
100
|
raise Error, "coverage map build stopped: #{result.detail}" if result.stopped_by
|
|
72
101
|
raise Error, "coverage map build failed: #{died(result.exit_status)}" if result.payload.empty?
|
|
73
102
|
|
|
@@ -77,6 +106,19 @@ module Saboteur
|
|
|
77
106
|
with_parent_load_lines(Selection::CoverageMap.from_json(result.payload))
|
|
78
107
|
end
|
|
79
108
|
|
|
109
|
+
# A build that hit its deadline only because --max-duration shortened
|
|
110
|
+
# the admission below the configured phase timeout: the budget ended
|
|
111
|
+
# the run, the suite is not broken (see BaselineGuard#budget_stopped?).
|
|
112
|
+
def budget_stopped?(result, admitted)
|
|
113
|
+
result.stopped_by == :timeout && admitted < @config["coverage_timeout"]
|
|
114
|
+
end
|
|
115
|
+
|
|
116
|
+
def raise_budget_stop(admitted)
|
|
117
|
+
raise BudgetPhaseTimeout, "the --max-duration budget stopped the coverage map build " \
|
|
118
|
+
"(admitted #{admitted.round(1)}s of the #{@config['coverage_timeout']}s " \
|
|
119
|
+
"phase timeout)"
|
|
120
|
+
end
|
|
121
|
+
|
|
80
122
|
# The parent's load lines merge into the freshly built map: the forked
|
|
81
123
|
# build child cannot inherit the parent's Coverage counters.
|
|
82
124
|
def with_parent_load_lines(map)
|
|
@@ -76,8 +76,9 @@ module Saboteur
|
|
|
76
76
|
attr_reader :warnings, :invalid_mutations, :generation_events, :omissions, :discovery_failures,
|
|
77
77
|
:hazard_suppressions, :inline_suppressions
|
|
78
78
|
|
|
79
|
-
def initialize(err)
|
|
79
|
+
def initialize(err, verbose: false)
|
|
80
80
|
@err = err
|
|
81
|
+
@verbose = verbose
|
|
81
82
|
@warnings = []
|
|
82
83
|
@invalid_mutations = 0
|
|
83
84
|
@generation_events = []
|
|
@@ -102,8 +103,11 @@ module Saboteur
|
|
|
102
103
|
@generation_events << GenerationEvent.new(*values)
|
|
103
104
|
end
|
|
104
105
|
|
|
106
|
+
# The selection and the test inference each walk the same changed
|
|
107
|
+
# files in one run; a failure either of them finds is recorded once.
|
|
105
108
|
def record_discovery_failure(file, reason:)
|
|
106
|
-
|
|
109
|
+
failure = DiscoveryFailure.new(file: file.to_s.dup.freeze, reason: reason.to_s.dup.freeze)
|
|
110
|
+
@discovery_failures << failure unless @discovery_failures.include?(failure)
|
|
107
111
|
end
|
|
108
112
|
|
|
109
113
|
# A mutant the hazard guard refused to generate (issue #102): counted
|
|
@@ -129,6 +133,23 @@ module Saboteur
|
|
|
129
133
|
@err.puts("saboteur: #{message}")
|
|
130
134
|
end
|
|
131
135
|
|
|
136
|
+
# --verbose: one line per finished subject with its result marks, so a
|
|
137
|
+
# survivor is attributable to a method as soon as the subject
|
|
138
|
+
# completes, before the whole run ends (#262).
|
|
139
|
+
def subject_progress(subject, results)
|
|
140
|
+
return unless @verbose
|
|
141
|
+
return if results.empty?
|
|
142
|
+
|
|
143
|
+
marks = results.map { |result| MARKS.fetch(result.status, "?") }.join
|
|
144
|
+
inform("#{subject.id} #{marks}")
|
|
145
|
+
end
|
|
146
|
+
|
|
147
|
+
# A message that is both a report warning and a stderr note.
|
|
148
|
+
def announce(message)
|
|
149
|
+
warn(message)
|
|
150
|
+
inform(message)
|
|
151
|
+
end
|
|
152
|
+
|
|
132
153
|
# A heartbeat for a long phase: the supervisor's wait loop calls tick
|
|
133
154
|
# in the parent after the fork, so no extra thread is ever alive at
|
|
134
155
|
# fork time. Prints "<label>: N s elapsed" about every interval
|
|
@@ -1,6 +1,8 @@
|
|
|
1
1
|
# SPDX-License-Identifier: MIT
|
|
2
2
|
# frozen_string_literal: true
|
|
3
3
|
|
|
4
|
+
require_relative "../text"
|
|
5
|
+
|
|
4
6
|
module Saboteur
|
|
5
7
|
class Engine
|
|
6
8
|
# Review-shaped diff mode (--since). A reviewer asks "do the tests check
|
|
@@ -14,7 +16,7 @@ module Saboteur
|
|
|
14
16
|
end
|
|
15
17
|
|
|
16
18
|
def restrict(subject, candidates)
|
|
17
|
-
changed =
|
|
19
|
+
changed = lines_for(subject)
|
|
18
20
|
candidates.select do |mutation, _|
|
|
19
21
|
mutation.coarse || mutation.edits.any? { |edit| edited_lines(subject, edit).any? { changed.include?(_1) } }
|
|
20
22
|
end
|
|
@@ -24,22 +26,39 @@ module Saboteur
|
|
|
24
26
|
# several lines is mutated as one piece and reported at its first line,
|
|
25
27
|
# which can sit above the line that changed; this names the changed line.
|
|
26
28
|
def changed_lines_for(subject, mutation)
|
|
27
|
-
changed =
|
|
29
|
+
changed = lines_for(subject)
|
|
28
30
|
mutation.edits.flat_map { |edit| edited_lines(subject, edit).to_a }.uniq.select { changed.include?(_1) }.sort
|
|
29
31
|
end
|
|
30
32
|
|
|
31
33
|
# Distance to the nearest changed line in this file. Unknown outside
|
|
32
|
-
# the diff is nil, not a made-up nearby location.
|
|
34
|
+
# the diff is nil, not a made-up nearby location. A range answers
|
|
35
|
+
# geometrically - 0 when the edit overlaps, else the gap to the nearer
|
|
36
|
+
# endpoint - so a whole-file range never materializes one entry per
|
|
37
|
+
# line.
|
|
33
38
|
def distance_for(subject, mutation)
|
|
34
|
-
changed =
|
|
39
|
+
changed = lines_for(subject)
|
|
35
40
|
mutation.edits.map do |edit|
|
|
36
41
|
first, last = edited_lines(subject, edit).minmax
|
|
37
|
-
changed.
|
|
42
|
+
if changed.is_a?(Range)
|
|
43
|
+
if last < changed.begin then changed.begin - last
|
|
44
|
+
elsif first > changed.end then first - changed.end
|
|
45
|
+
else 0
|
|
46
|
+
end
|
|
47
|
+
else
|
|
48
|
+
changed.map { |line| (line - line.clamp(first, last)).abs }.min
|
|
49
|
+
end
|
|
38
50
|
end.min
|
|
39
51
|
end
|
|
40
52
|
|
|
41
53
|
private
|
|
42
54
|
|
|
55
|
+
# subject.file carries the locale tag under a non-UTF-8 locale
|
|
56
|
+
# while the map keys are UTF-8; retag losslessly or a non-ASCII
|
|
57
|
+
# path never matches and every fine mutant is silently dropped.
|
|
58
|
+
def lines_for(subject)
|
|
59
|
+
@changed_lines.fetch(Text.utf8_identity(subject.file), Set.new)
|
|
60
|
+
end
|
|
61
|
+
|
|
43
62
|
def edited_lines(subject, edit)
|
|
44
63
|
first = subject.line_of(edit.start_byte)
|
|
45
64
|
last = subject.line_of([edit.end_byte - 1, edit.start_byte].max)
|
|
@@ -9,23 +9,31 @@ module Saboteur
|
|
|
9
9
|
# candidate list, so results look the same either way.
|
|
10
10
|
class Distribution
|
|
11
11
|
# After run: the subjects a stop_after callable left unrun (always empty
|
|
12
|
-
# without one)
|
|
13
|
-
|
|
12
|
+
# without one), and the subjects that started but stopped mid-list as
|
|
13
|
+
# [subject, omitted_candidate_count] pairs (a spent --max-duration
|
|
14
|
+
# breaking a subject's scheduler loop). unverified lists the subjects
|
|
15
|
+
# whose fresh kills the spent budget left unchecked (#296).
|
|
16
|
+
attr_reader :unreached, :truncated, :unverified
|
|
14
17
|
|
|
15
18
|
# stop_after: nil, or a callable that gets one subject's results and
|
|
16
19
|
# returns true to stop before the next subject (--fail-fast). It only
|
|
17
20
|
# works on one process: parallel workers have already started the
|
|
18
21
|
# other subjects.
|
|
19
22
|
def initialize(jobs:, scheduler:, diagnostics:, cache:, on_start: nil, after_subject: nil,
|
|
20
|
-
stop_after: nil
|
|
23
|
+
stop_after: nil, halt_check: nil,
|
|
24
|
+
drain_timeout: 30) # seconds; two minimum-strength mutant deadlines at default settings
|
|
21
25
|
@jobs = jobs
|
|
22
26
|
@stop_after = stop_after
|
|
27
|
+
@halt_check = halt_check
|
|
23
28
|
@unreached = []
|
|
29
|
+
@truncated = []
|
|
30
|
+
@unverified = []
|
|
24
31
|
@on_start = on_start
|
|
25
32
|
@scheduler = scheduler
|
|
26
33
|
@diagnostics = diagnostics
|
|
27
34
|
@cache = cache
|
|
28
35
|
@after_subject = after_subject
|
|
36
|
+
@drain_timeout = drain_timeout
|
|
29
37
|
end
|
|
30
38
|
|
|
31
39
|
# lists: each subject's [[mutation, text]] candidates, in subject order.
|
|
@@ -35,23 +43,116 @@ module Saboteur
|
|
|
35
43
|
# The watcher runs as each subject's row arrives, before the pool
|
|
36
44
|
# hands out more work, so a tripped canary stops the queue (jobs>1
|
|
37
45
|
# used to check only after the whole batch returned, issue #102).
|
|
38
|
-
|
|
39
|
-
|
|
46
|
+
# A tool error no longer raises on sight: one wiped log dir hits
|
|
47
|
+
# every worker, and a bystander whose current job ran no mutant
|
|
48
|
+
# reports "an unknown mutant", which could land before the report
|
|
49
|
+
# of the worker whose mutant did the damage (macOS CI on #285).
|
|
50
|
+
# The pool is halted instead - no new work, and the drain is
|
|
51
|
+
# bounded by drain_timeout so a wedged worker cannot stall the
|
|
52
|
+
# abort - and the abort raises the report with the best
|
|
53
|
+
# attribution.
|
|
54
|
+
# The abort reports and the pool are run state: a Distribution
|
|
55
|
+
# runs once. @pool is only read from arrivals, which fire inside
|
|
56
|
+
# @pool.map after the assignment.
|
|
57
|
+
# A budget already spent by boot, baselines or the coverage map
|
|
58
|
+
# launches nothing: every subject comes back unreached (#245).
|
|
59
|
+
if @halt_check&.call
|
|
60
|
+
@unreached = subjects
|
|
61
|
+
return []
|
|
40
62
|
end
|
|
41
|
-
|
|
42
|
-
|
|
63
|
+
|
|
64
|
+
@abort_reports = []
|
|
65
|
+
arrival = arrival_for(subjects, lists)
|
|
66
|
+
@pool = Execution::WorkerPool.new(jobs: @jobs, on_start: @on_start, on_result: arrival,
|
|
67
|
+
drain_timeout: @drain_timeout)
|
|
68
|
+
rows = @pool.map(subjects.each_index.to_a, weights: lists.map(&:size)) do |i|
|
|
43
69
|
work(subjects[i], lists[i])
|
|
44
70
|
end
|
|
45
|
-
|
|
71
|
+
raise_preferred_tool_error(@abort_reports) if @abort_reports.any?
|
|
72
|
+
record_unreached(rows, subjects, lists)
|
|
73
|
+
rows.each_with_index.flat_map do |row, i|
|
|
74
|
+
next [] if row.nil?
|
|
75
|
+
|
|
76
|
+
raise_if_tool_error(row)
|
|
77
|
+
rebuild(row, subjects[i], lists[i])
|
|
78
|
+
end
|
|
46
79
|
end
|
|
47
80
|
|
|
48
81
|
private
|
|
49
82
|
|
|
83
|
+
# @pool is only read here from arrivals, which fire inside @pool.map
|
|
84
|
+
# after the assignment. A spent --max-duration budget stops the queue
|
|
85
|
+
# without the safety-abort drain: in-flight subjects finish and
|
|
86
|
+
# report, and only never-started ones come back as nil holes, so
|
|
87
|
+
# unreached never mislabels an interrupted subject as unrun (#245).
|
|
88
|
+
def arrival_for(subjects, lists)
|
|
89
|
+
lambda do |i, row|
|
|
90
|
+
if row["tool_error"]
|
|
91
|
+
collect_abort_report(row)
|
|
92
|
+
else
|
|
93
|
+
check_subject(subjects, lists, i, row)
|
|
94
|
+
@pool.halt!(drain: false) if @halt_check&.call
|
|
95
|
+
end
|
|
96
|
+
end
|
|
97
|
+
end
|
|
98
|
+
|
|
99
|
+
# nil rows never started; a short results list is a subject whose
|
|
100
|
+
# scheduler stopped mid-list at the budget deadline. Worker-level error
|
|
101
|
+
# rows synthesize an error per candidate, so they never look truncated.
|
|
102
|
+
def unverified_result?(result)
|
|
103
|
+
result.is_a?(MutantResult) && result.status == :unverified
|
|
104
|
+
end
|
|
105
|
+
|
|
106
|
+
def record_unreached(rows, subjects, lists)
|
|
107
|
+
@unreached = []
|
|
108
|
+
@truncated = []
|
|
109
|
+
@unverified = []
|
|
110
|
+
rows.each_with_index do |row, i|
|
|
111
|
+
if row.nil?
|
|
112
|
+
@unreached << subjects[i]
|
|
113
|
+
next
|
|
114
|
+
end
|
|
115
|
+
next unless row["results"] && !row["error"]
|
|
116
|
+
|
|
117
|
+
@truncated << [subjects[i], lists[i].size - row["results"].size] if row["results"].size < lists[i].size
|
|
118
|
+
@unverified << subjects[i] if row["results"].any? { |result| result["status"] == "unverified" }
|
|
119
|
+
end
|
|
120
|
+
end
|
|
121
|
+
|
|
122
|
+
# @pool is always assigned when an arrival fires: arrivals only fire
|
|
123
|
+
# from inside @pool.map, which runs after the assignment.
|
|
124
|
+
def collect_abort_report(row)
|
|
125
|
+
@abort_reports << row
|
|
126
|
+
@pool.halt!
|
|
127
|
+
end
|
|
128
|
+
|
|
129
|
+
# A canary trip from the end-of-subject check folds into the same
|
|
130
|
+
# record + halt + preferred-report path as a worker's own report:
|
|
131
|
+
# escaping the pool here would override a collected report that
|
|
132
|
+
# names the real culprit (#285 review).
|
|
133
|
+
def check_subject(subjects, lists, index, row)
|
|
134
|
+
@after_subject&.call(subjects[index], decode(row, subjects[index], lists[index]))
|
|
135
|
+
rescue Error => e
|
|
136
|
+
collect_abort_report(Execution::WorkerPool.tool_error_row(e))
|
|
137
|
+
end
|
|
138
|
+
|
|
50
139
|
def run_serially(subjects, lists)
|
|
51
140
|
results = []
|
|
52
141
|
subjects.zip(lists).each_with_index do |(subject, list), index|
|
|
142
|
+
# Same pre-dispatch budget check as the parallel first wave: an
|
|
143
|
+
# already-spent baseline starts no further subjects (#245).
|
|
144
|
+
if @halt_check&.call
|
|
145
|
+
@unreached = subjects.drop(index)
|
|
146
|
+
break
|
|
147
|
+
end
|
|
148
|
+
|
|
53
149
|
subject_results = @scheduler.call(subject, list)
|
|
54
150
|
results.concat(subject_results)
|
|
151
|
+
# A scheduler that stopped mid-list at the budget deadline: the
|
|
152
|
+
# completed results stay, the omitted candidates are accounted.
|
|
153
|
+
omitted = list.size - subject_results.size
|
|
154
|
+
@truncated << [subject, omitted] if omitted.positive?
|
|
155
|
+
@unverified << subject if subject_results.any? { |result| unverified_result?(result) }
|
|
55
156
|
@after_subject&.call(subject, subject_results)
|
|
56
157
|
next unless @stop_after&.call(subject_results)
|
|
57
158
|
|
|
@@ -91,6 +192,21 @@ module Saboteur
|
|
|
91
192
|
|
|
92
193
|
# Turns a worker row back into results, without diagnostics or caching:
|
|
93
194
|
# used both at result arrival (the canary) and at rebuild time.
|
|
195
|
+
# A tool error from the worker (a vanished cache dir, ...) aborts
|
|
196
|
+
# the whole run with exit 3; it is not a per-mutant outcome (#283).
|
|
197
|
+
def raise_if_tool_error(row)
|
|
198
|
+
raise Error, row["tool_error"] if row["tool_error"]
|
|
199
|
+
end
|
|
200
|
+
|
|
201
|
+
# The abort names a culprit when any report can, ranked by
|
|
202
|
+
# attribution quality (#285 review): a report carrying its mutant's
|
|
203
|
+
# per-mutant digest first, then a canary trip (the hazard report
|
|
204
|
+
# still beats a bare report, #283), then the first bare report.
|
|
205
|
+
def raise_preferred_tool_error(rows)
|
|
206
|
+
row = rows.find { |r| r["culprit"] } || rows.find { |r| r["canary"] } || rows.first
|
|
207
|
+
raise Error, row["tool_error"]
|
|
208
|
+
end
|
|
209
|
+
|
|
94
210
|
def decode(row, subject, list)
|
|
95
211
|
return failed(row["error"], subject, list) if row["error"]
|
|
96
212
|
|
|
@@ -0,0 +1,46 @@
|
|
|
1
|
+
# SPDX-License-Identifier: MIT
|
|
2
|
+
# frozen_string_literal: true
|
|
3
|
+
|
|
4
|
+
module Saboteur
|
|
5
|
+
class Engine
|
|
6
|
+
# Records the subjects an early stop left behind - whole subjects that
|
|
7
|
+
# never started and candidates skipped mid-way through a subject - so
|
|
8
|
+
# the report says the verdict covers less than the selection.
|
|
9
|
+
module EarlyStopRecording
|
|
10
|
+
private
|
|
11
|
+
|
|
12
|
+
# The subjects a stop left unrun or mid-way: the verdict covers less than the selection.
|
|
13
|
+
def record_early_stop(distribution)
|
|
14
|
+
return if distribution.unreached.empty? && distribution.truncated.empty? && distribution.unverified.empty?
|
|
15
|
+
|
|
16
|
+
truncated = distribution.truncated.map { |subject, omitted| { subject: subject.id, omitted: omitted } }
|
|
17
|
+
@early_stop = EarlyStop.new(reason: stop_reason(distribution), unreached: distribution.unreached.map(&:id),
|
|
18
|
+
truncated: truncated, unverified: distribution.unverified.map(&:id))
|
|
19
|
+
end
|
|
20
|
+
|
|
21
|
+
# Mid-subject omissions and skipped checks are budget work: only a
|
|
22
|
+
# stop that left whole subjects unreached can be fail_fast's.
|
|
23
|
+
def stop_reason(distribution)
|
|
24
|
+
return "max_duration" if distribution.truncated.any? || distribution.unverified.any?
|
|
25
|
+
|
|
26
|
+
run_early_stop_reason
|
|
27
|
+
end
|
|
28
|
+
|
|
29
|
+
# max_duration was announced before fail_fast could stop the run: a
|
|
30
|
+
# budget stop overrides a failure stop announced later.
|
|
31
|
+
def run_early_stop_reason
|
|
32
|
+
@budget_announced ? "max_duration" : "fail_fast"
|
|
33
|
+
end
|
|
34
|
+
|
|
35
|
+
# A baseline or coverage phase stopped because --max-duration ate its
|
|
36
|
+
# allowance: budget exhaustion, not a broken suite. The run reports
|
|
37
|
+
# partial with every selected subject unreached (exit 4) (#296).
|
|
38
|
+
def budget_phase_stop(error, subjects)
|
|
39
|
+
@diagnostics.warn(error.message)
|
|
40
|
+
@early_stop = EarlyStop.new(reason: "max_duration", unreached: subjects.map(&:id), truncated: [],
|
|
41
|
+
unverified: [])
|
|
42
|
+
[]
|
|
43
|
+
end
|
|
44
|
+
end
|
|
45
|
+
end
|
|
46
|
+
end
|
|
@@ -4,6 +4,7 @@
|
|
|
4
4
|
require "digest"
|
|
5
5
|
require "fileutils"
|
|
6
6
|
require "json"
|
|
7
|
+
require_relative "../text"
|
|
7
8
|
|
|
8
9
|
module Saboteur
|
|
9
10
|
class Engine
|
|
@@ -65,7 +66,7 @@ module Saboteur
|
|
|
65
66
|
private
|
|
66
67
|
|
|
67
68
|
def load
|
|
68
|
-
data = JSON.parse(File.read(@path))
|
|
69
|
+
data = JSON.parse(Text.utf8_strict(File.read(@path)))
|
|
69
70
|
data["format"] == FORMAT ? data["entries"] : {}
|
|
70
71
|
rescue SystemCallError, JSON::ParserError
|
|
71
72
|
{}
|