saboteur 0.11.1 → 0.11.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +35 -0
- data/README.md +1 -1
- data/lib/saboteur/cli/exit_status.rb +6 -2
- data/lib/saboteur/cli/explain_command.rb +51 -0
- data/lib/saboteur/cli/flaky_exit.rb +23 -0
- data/lib/saboteur/cli/run_command.rb +13 -0
- data/lib/saboteur/cli/run_output.rb +10 -0
- data/lib/saboteur/cli/run_reporting.rb +13 -2
- data/lib/saboteur/cli/survivors_command.rb +1 -1
- data/lib/saboteur/cli.rb +3 -0
- data/lib/saboteur/config.rb +7 -2
- data/lib/saboteur/doctor.rb +8 -1
- data/lib/saboteur/engine/coverage_store.rb +54 -3
- data/lib/saboteur/engine/distribution.rb +6 -2
- data/lib/saboteur/engine/pipelines.rb +14 -2
- data/lib/saboteur/engine/scheduler.rb +1 -1
- data/lib/saboteur/engine/selection_announcement.rb +5 -4
- data/lib/saboteur/engine/subject_selection.rb +10 -0
- data/lib/saboteur/engine.rb +35 -5
- data/lib/saboteur/help.rb +11 -0
- data/lib/saboteur/ignore_list.rb +22 -2
- data/lib/saboteur/mutation_source/builtin.rb +1 -0
- data/lib/saboteur/mutation_source/operator_explain.rb +181 -0
- data/lib/saboteur/reporter/agent_commands.rb +23 -6
- data/lib/saboteur/reporter/agent_incomplete.rb +9 -2
- data/lib/saboteur/reporter/agent_summary.rb +2 -1
- data/lib/saboteur/reporter/gap_kinds.rb +80 -0
- data/lib/saboteur/reporter/html.rb +19 -4
- data/lib/saboteur/reporter/review_comments.rb +6 -5
- data/lib/saboteur/reporter/score_table.rb +8 -7
- data/lib/saboteur/reporter/survivor_classification.rb +26 -8
- data/lib/saboteur/reporter/survivor_hints.rb +3 -2
- data/lib/saboteur/reporter/survivor_list.rb +53 -3
- data/lib/saboteur/reporter/terminal.rb +27 -29
- data/lib/saboteur/reporter/wording.rb +37 -0
- data/lib/saboteur/result.rb +10 -0
- data/lib/saboteur/text.rb +6 -0
- data/lib/saboteur/version.rb +1 -1
- metadata +6 -1
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 45635354d2b8067e2de9a11025677af615e3eb54da14dfcd26f0afbdcdd00073
|
|
4
|
+
data.tar.gz: 40a395d6909fb4bcec99596c6b34e4b40edf1b72c76b349ab25cb3ae44b48726
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: 3f3dd155c87c3c6a45d1fd66a66cc5ec21d8c56dd5113dab2568cd07cc957ee2ba34bfc1b773cf4610aeee822e354bf36cc747418bcee24902365563f5f2b5c2
|
|
7
|
+
data.tar.gz: 23f785f27b3396a38195f44d6a418d819d6c7359fe34a8ec9f8422855206b53a5ebd634eb70d8c8d40cf622787a9795d99395950932a5870a2418d8651520aa9
|
data/CHANGELOG.md
CHANGED
|
@@ -5,6 +5,41 @@ All notable changes are documented here. The format follows
|
|
|
5
5
|
[Semantic Versioning](https://semver.org/).
|
|
6
6
|
|
|
7
7
|
## [Unreleased]
|
|
8
|
+
|
|
9
|
+
## [0.11.2] - 2026-10-22
|
|
10
|
+
|
|
11
|
+
### Added
|
|
12
|
+
|
|
13
|
+
- A full run warns when an ignore entry's subject never appears, so a
|
|
14
|
+
renamed or removed method can no longer kill the entry silently
|
|
15
|
+
(#1250, #1206).
|
|
16
|
+
- Survivor gaps are labeled by kind instead of the blanket
|
|
17
|
+
`missing_test` label (#1246, #1153, #1064), and the README names
|
|
18
|
+
`tool_limit` and the gap kinds in the survivor triage labels (#1247).
|
|
19
|
+
- The `explain` command prints a human-readable rationale for any
|
|
20
|
+
registry operator (#1251).
|
|
21
|
+
- The HTML report lists survivors, timeouts and errors before killed
|
|
22
|
+
rows inside each file, keeping source order within each group
|
|
23
|
+
(#1253, #1233).
|
|
24
|
+
- Survivor blocks consult the whole-suite coverage map, ending false
|
|
25
|
+
survivors from diff-scoped runs (#1252).
|
|
26
|
+
|
|
27
|
+
### Changed
|
|
28
|
+
|
|
29
|
+
- Output polish across the terminal, JSON and HTML reports: the flaky
|
|
30
|
+
check names why it exits 3, `doctor` prints its coverage line and a
|
|
31
|
+
run without a config says so (#1239); stubbed-test wording, the
|
|
32
|
+
timeout tally and the replay scope line are clearer (#1240);
|
|
33
|
+
selection messaging states nothing-to-test once and names the
|
|
34
|
+
`--since` scope on `--subject` misses (#1241); survivor blocks count
|
|
35
|
+
blocks, not mutants (#1242); the run opener and coverage-map counts
|
|
36
|
+
are pluralized (#1244); the clean screen labels its scope, hints at
|
|
37
|
+
the live reports and summarizes an empty diff (#1245).
|
|
38
|
+
- Docs: the ignore list is named as the one way to accept a survivor,
|
|
39
|
+
the pre-1.0 policy for test selection and operator defaults is
|
|
40
|
+
stated, and the CLI-docs rule points at `docs/` rather than the
|
|
41
|
+
short README (#1248).
|
|
42
|
+
|
|
8
43
|
## [0.11.1] - 2026-10-22
|
|
9
44
|
|
|
10
45
|
### Added
|
data/README.md
CHANGED
|
@@ -80,7 +80,7 @@ tree refuses with exit 2 and points at the generator. Engine gem with
|
|
|
80
80
|
assertion, 1 by a crash`), and an error-dominated run prints
|
|
81
81
|
`Score: n/a` instead of a number you would misread.
|
|
82
82
|
- 🔍 **Survivors you can act on.** Survivors group per statement with a
|
|
83
|
-
triage label (`likely_equivalent`, `arid`, `
|
|
83
|
+
triage label (`likely_equivalent`, `arid`, `tool_limit`, or a gap kind) and their
|
|
84
84
|
covering tests - split into the tests that ran against the mutant and
|
|
85
85
|
the ones that never did. `saboteur survivors` browses them read-only:
|
|
86
86
|
one digest shows the full diff and the exact replay command.
|
|
@@ -1,11 +1,15 @@
|
|
|
1
1
|
# SPDX-License-Identifier: MIT
|
|
2
2
|
# frozen_string_literal: true
|
|
3
3
|
|
|
4
|
+
require_relative "flaky_exit"
|
|
5
|
+
|
|
4
6
|
module Saboteur
|
|
5
7
|
class CLI
|
|
6
8
|
# How a finished run maps to an exit code and a recorded exit cause. Mixed
|
|
7
9
|
# into the CLI, which supplies `config` and the `@err` stream.
|
|
8
10
|
module ExitStatus
|
|
11
|
+
include FlakyExit
|
|
12
|
+
|
|
9
13
|
private
|
|
10
14
|
|
|
11
15
|
def exit_code(report)
|
|
@@ -15,7 +19,7 @@ module Saboteur
|
|
|
15
19
|
|
|
16
20
|
return EXIT_TOOL if untested_partial?(report)
|
|
17
21
|
|
|
18
|
-
return EXIT_TOOL if unverifiable_scope?(report)
|
|
22
|
+
return EXIT_TOOL if unverifiable_scope?(report) || flaky_disagreement?(report)
|
|
19
23
|
|
|
20
24
|
return EXIT_TOOL if mostly_errored?(report)
|
|
21
25
|
|
|
@@ -117,7 +121,7 @@ module Saboteur
|
|
|
117
121
|
else
|
|
118
122
|
if report.majority_errored?
|
|
119
123
|
"errorMajority"
|
|
120
|
-
elsif unverifiable_review?(report) || unverifiable_unreachable?(report) ||
|
|
124
|
+
elsif unverifiable_review?(report) || unverifiable_unreachable?(report) || report.results.any?(&:flaky?) ||
|
|
121
125
|
(report.early_stop && report.score_breakdown.tested.zero?)
|
|
122
126
|
"incomplete"
|
|
123
127
|
else
|
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
# SPDX-License-Identifier: MIT
|
|
2
|
+
# frozen_string_literal: true
|
|
3
|
+
|
|
4
|
+
require_relative "../help"
|
|
5
|
+
require_relative "../reporter/palette"
|
|
6
|
+
require_relative "../mutation_source/operator_explain"
|
|
7
|
+
|
|
8
|
+
module Saboteur
|
|
9
|
+
class CLI
|
|
10
|
+
# Shared rendering for `saboteur explain <operator>` (#1097): the
|
|
11
|
+
# curated definition, a before/after example and survive/kill guidance
|
|
12
|
+
# for one operator, so a survivor's operator can be understood and
|
|
13
|
+
# targeted without leaving the terminal.
|
|
14
|
+
module ExplainCommand
|
|
15
|
+
private
|
|
16
|
+
|
|
17
|
+
def explain_command
|
|
18
|
+
name = @argv.shift
|
|
19
|
+
raise UsageError, "usage: saboteur explain <operator>" if name.nil?
|
|
20
|
+
|
|
21
|
+
entry = MutationSource::OperatorExplain.for(name)
|
|
22
|
+
if entry.nil?
|
|
23
|
+
@err.puts("saboteur: unknown operator: #{name}")
|
|
24
|
+
suggestion = MutationSource::OperatorRegistry.suggestion(name)
|
|
25
|
+
@err.puts("saboteur: #{suggestion}") unless suggestion.empty?
|
|
26
|
+
@err.puts("saboteur: operators: #{MutationSource::OperatorRegistry::DESCRIPTIONS.keys.join(', ')}")
|
|
27
|
+
return EXIT_USAGE
|
|
28
|
+
end
|
|
29
|
+
|
|
30
|
+
@out.puts(render_explain(entry, palette: Reporter::Palette.for(@out, mode: :auto)))
|
|
31
|
+
EXIT_OK
|
|
32
|
+
rescue UsageError => e
|
|
33
|
+
@err.puts("saboteur: #{e.message}")
|
|
34
|
+
EXIT_USAGE
|
|
35
|
+
end
|
|
36
|
+
|
|
37
|
+
def render_explain(entry, palette: Reporter::Palette.new(enabled: false))
|
|
38
|
+
original, replacement = entry.example.split(" -> ")
|
|
39
|
+
[
|
|
40
|
+
"#{palette.paint(:head, 'Operator:')} #{entry.name}",
|
|
41
|
+
"#{palette.paint(:head, 'Definition:')} #{entry.definition}",
|
|
42
|
+
palette.paint(:head, "Example:"),
|
|
43
|
+
" #{palette.paint(:fail, "- #{original}")}",
|
|
44
|
+
" #{palette.paint(:pass, "+ #{replacement}")}",
|
|
45
|
+
"#{palette.paint(:warn, 'Survives when:')} #{entry.survives}",
|
|
46
|
+
"#{palette.paint(:info, 'Kill it with:')} #{entry.kill}"
|
|
47
|
+
].join("\n")
|
|
48
|
+
end
|
|
49
|
+
end
|
|
50
|
+
end
|
|
51
|
+
end
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
# SPDX-License-Identifier: MIT
|
|
2
|
+
# frozen_string_literal: true
|
|
3
|
+
|
|
4
|
+
module Saboteur
|
|
5
|
+
class CLI
|
|
6
|
+
# --flaky-check (#1167): a verdict that flipped on the reordered rerun is
|
|
7
|
+
# unverified, outside the score. The run was not stopped by a budget, so
|
|
8
|
+
# it is not exit 4, but it cannot go green either.
|
|
9
|
+
module FlakyExit
|
|
10
|
+
private
|
|
11
|
+
|
|
12
|
+
def flaky_disagreement?(report)
|
|
13
|
+
flaky = report.results.select(&:flaky?)
|
|
14
|
+
return false if flaky.empty?
|
|
15
|
+
|
|
16
|
+
@err.puts("saboteur: #{flaky.size} mutant#{'s' unless flaky.one?} changed verdict on the --flaky-check " \
|
|
17
|
+
"reordered rerun (#{flaky.first(3).map(&:digest).join(', ')}), so they are unverified: " \
|
|
18
|
+
"your tests depend on run order; fix that and run again")
|
|
19
|
+
true
|
|
20
|
+
end
|
|
21
|
+
end
|
|
22
|
+
end
|
|
23
|
+
end
|
|
@@ -17,6 +17,18 @@ module Saboteur
|
|
|
17
17
|
|
|
18
18
|
private
|
|
19
19
|
|
|
20
|
+
# A fresh checkout has no config file: say once that defaults are in
|
|
21
|
+
# use, and who writes one (#1172). Agent output stays machine-clean;
|
|
22
|
+
# there is no init command - the Rails generator writes the config.
|
|
23
|
+
def announce_missing_config_file(palette)
|
|
24
|
+
# @format, not agent_requested?: parse! has already eaten --format.
|
|
25
|
+
return if @format == "agent" || config.config_file_present
|
|
26
|
+
|
|
27
|
+
path = options[:config] || Config::FILE
|
|
28
|
+
notice = "saboteur: no #{path} - using defaults (Rails: bundle exec rails generate saboteur:install writes one)"
|
|
29
|
+
@err.puts(palette.paint(:dim, notice))
|
|
30
|
+
end
|
|
31
|
+
|
|
20
32
|
def baseline_red?(error)
|
|
21
33
|
error.message.to_s.match?(BASELINE_RED_MESSAGE)
|
|
22
34
|
end
|
|
@@ -34,6 +46,7 @@ module Saboteur
|
|
|
34
46
|
config.deprecation_warnings.each do |warning|
|
|
35
47
|
@err.puts(Reporter::Notice.paint(notices, "saboteur: #{warning}"))
|
|
36
48
|
end
|
|
49
|
+
announce_missing_config_file(notices)
|
|
37
50
|
|
|
38
51
|
return preview if options[:list_operators] || options[:dry_run]
|
|
39
52
|
|
|
@@ -21,6 +21,16 @@ module Saboteur
|
|
|
21
21
|
@out.puts "\nReports: #{names.join(', ')}"
|
|
22
22
|
end
|
|
23
23
|
|
|
24
|
+
# The clean screen's scope label (#1173, #1174): --since, positional
|
|
25
|
+
# sources and --subject each name what the run covered.
|
|
26
|
+
def scope_label
|
|
27
|
+
parts = []
|
|
28
|
+
parts << "--since #{config['since']}" if config["since"]
|
|
29
|
+
parts.concat(@sources) if @sources
|
|
30
|
+
parts << "--subject #{config['subjects'].join(', ')}" if config["subjects"].any?
|
|
31
|
+
parts.empty? ? "whole project" : parts.join(", ")
|
|
32
|
+
end
|
|
33
|
+
|
|
24
34
|
# Agent mode: stdout carries exactly one JSON payload. The target's own output (boot
|
|
25
35
|
# files, hooks, spec loading, child processes) goes to stderr while the engine runs.
|
|
26
36
|
# Descriptor 1 points at stderr for the block, so Kernel#puts, system and forked
|
|
@@ -22,7 +22,9 @@ module Saboteur
|
|
|
22
22
|
overrides: config["palette"], threshold: config["threshold"],
|
|
23
23
|
replay_command: survivor_replay_command(report, engine),
|
|
24
24
|
all_mutants: options[:all_mutants] || false,
|
|
25
|
-
exit_code: code, previous_score: previous_score,
|
|
25
|
+
exit_code: code, previous_score: previous_score, scope: scope_label,
|
|
26
|
+
whole_suite_map: whole_suite_map_for(engine),
|
|
27
|
+
run_map: engine&.run_coverage_map,
|
|
26
28
|
baseline_gate: (baseline_gate(report) if config["baseline"])).call(report)
|
|
27
29
|
Reporter::Replay.new(@out).call(report) if options[:id]
|
|
28
30
|
# The JSON carries the exit cause an outer harness trusts, so it is
|
|
@@ -38,6 +40,14 @@ module Saboteur
|
|
|
38
40
|
code
|
|
39
41
|
end
|
|
40
42
|
|
|
43
|
+
# No engine or no cache: no whole-suite map can exist, so the survivor
|
|
44
|
+
# block keeps its legacy covered-by line over an impossible build (review C1).
|
|
45
|
+
def whole_suite_map_for(engine)
|
|
46
|
+
return engine.whole_suite_coverage_map if engine&.whole_suite_map_lookup?
|
|
47
|
+
|
|
48
|
+
Reporter::SurvivorList::NO_WHOLE_SUITE_LOOKUP
|
|
49
|
+
end
|
|
50
|
+
|
|
41
51
|
def agent_run(arguments)
|
|
42
52
|
tree = WorkingTree.new(config.root, ignored: config.ignored_tree_paths).state
|
|
43
53
|
# The machine path is exempt from the noise diet: quiet_start: false
|
|
@@ -94,7 +104,8 @@ module Saboteur
|
|
|
94
104
|
def survivor_replay_command(report, engine)
|
|
95
105
|
lambda do |digest|
|
|
96
106
|
tree = WorkingTree.new(config.root, ignored: config.ignored_tree_paths).state
|
|
97
|
-
Reporter::Agent::Commands.replay(agent_provenance(report, tree, engine), digest, capped: false
|
|
107
|
+
Reporter::Agent::Commands.replay(agent_provenance(report, tree, engine), digest, capped: false,
|
|
108
|
+
report: report)
|
|
98
109
|
end
|
|
99
110
|
end
|
|
100
111
|
|
|
@@ -31,7 +31,7 @@ module Saboteur
|
|
|
31
31
|
opts.on("--ranked", "Review-priority order: the report's ranked findings first")
|
|
32
32
|
opts.on("--file SUBSTR", "Only survivors whose path contains SUBSTR")
|
|
33
33
|
opts.on("--operator NAME", "Only survivors of operator NAME")
|
|
34
|
-
opts.on("--triage LABEL", "Only survivors classified LABEL (for example
|
|
34
|
+
opts.on("--triage LABEL", "Only survivors classified LABEL (for example boundary)")
|
|
35
35
|
opts.on("--show DIGEST", "Diff, replay command and ignore template for one survivor")
|
|
36
36
|
end
|
|
37
37
|
options = {}
|
data/lib/saboteur/cli.rb
CHANGED
|
@@ -15,6 +15,7 @@ require_relative "cli/baseline_gate"
|
|
|
15
15
|
require_relative "preview"
|
|
16
16
|
require_relative "cli/run_reporting"
|
|
17
17
|
require_relative "cli/survivors_command"
|
|
18
|
+
require_relative "cli/explain_command"
|
|
18
19
|
require_relative "cli/source_set"
|
|
19
20
|
require_relative "cli/run_budget_options"
|
|
20
21
|
require_relative "cli/shard_options"
|
|
@@ -44,6 +45,7 @@ module Saboteur
|
|
|
44
45
|
include BaselineGate
|
|
45
46
|
include RunReporting
|
|
46
47
|
include SurvivorsCommand
|
|
48
|
+
include ExplainCommand
|
|
47
49
|
include SourceSet
|
|
48
50
|
include RunBudgetOptions
|
|
49
51
|
include ShardOptions
|
|
@@ -103,6 +105,7 @@ module Saboteur
|
|
|
103
105
|
when "run" then run_command
|
|
104
106
|
when "baseline" then baseline
|
|
105
107
|
when "survivors" then survivors_command
|
|
108
|
+
when "explain" then explain_command
|
|
106
109
|
when "doctor" then doctor_command
|
|
107
110
|
when "badge", "trend" then history_entry(command)
|
|
108
111
|
when "version", "--version" then @out.puts(VERSION) || EXIT_OK
|
data/lib/saboteur/config.rb
CHANGED
|
@@ -146,7 +146,7 @@ module Saboteur
|
|
|
146
146
|
end
|
|
147
147
|
|
|
148
148
|
from_file = File.file?(path) ? ConfigFile.load(path) : {}
|
|
149
|
-
new(from_file.merge(stringify(overrides)), root: root)
|
|
149
|
+
new(from_file.merge(stringify(overrides)), root: root, config_file_present: File.file?(path))
|
|
150
150
|
end
|
|
151
151
|
|
|
152
152
|
def self.stringify(hash)
|
|
@@ -156,11 +156,16 @@ module Saboteur
|
|
|
156
156
|
# Deprecations found while loading: printed by the CLI, never fatal.
|
|
157
157
|
attr_reader :deprecation_warnings
|
|
158
158
|
|
|
159
|
-
|
|
159
|
+
# Whether the config file existed at load (#1172): the CLI says once
|
|
160
|
+
# when a run falls back to defaults because no file was there.
|
|
161
|
+
attr_reader :config_file_present
|
|
162
|
+
|
|
163
|
+
def initialize(values = {}, root: Dir.pwd, explicit: nil, config_file_present: true)
|
|
160
164
|
# Retag the root losslessly: under a non-UTF-8 locale it arrives
|
|
161
165
|
# locale-tagged, and every glob, join and expand below would mix
|
|
162
166
|
# encodings with the UTF-8 patterns from the config file (#231).
|
|
163
167
|
@root = Text.utf8_identity(File.expand_path(root))
|
|
168
|
+
@config_file_present = config_file_present
|
|
164
169
|
unknown = values.keys - DEFAULTS.keys
|
|
165
170
|
raise ConfigError, "unknown config key(s): #{unknown.join(', ')}" if unknown.any?
|
|
166
171
|
|
data/lib/saboteur/doctor.rb
CHANGED
|
@@ -76,7 +76,14 @@ module Saboteur
|
|
|
76
76
|
# post-boot check re-reads the state without printing a second line.
|
|
77
77
|
def check_coverage
|
|
78
78
|
state = Coverage.state
|
|
79
|
-
|
|
79
|
+
# "suspended" alone reads as a finding; it is the expected boot state
|
|
80
|
+
# - coverage is stopped while the app boots and starts for the map
|
|
81
|
+
# build (#1176). Any other state stays terse: verify_coverage raises.
|
|
82
|
+
if state == :suspended
|
|
83
|
+
@out.puts "Coverage: suspended during boot (saboteur starts it for the coverage map)"
|
|
84
|
+
else
|
|
85
|
+
@out.puts "Coverage: #{state}"
|
|
86
|
+
end
|
|
80
87
|
verify_coverage(state)
|
|
81
88
|
end
|
|
82
89
|
|
|
@@ -16,6 +16,9 @@ module Saboteur
|
|
|
16
16
|
include CoverageParts
|
|
17
17
|
|
|
18
18
|
CACHE_INPUTS = %w[integration require rspec_args rails tests coverage_recorder].freeze
|
|
19
|
+
# Names the freshest whole-suite map build; read as an untrusted hint:
|
|
20
|
+
# only a bare 64-hex key joins the cache dir (#427).
|
|
21
|
+
WHOLE_SUITE_POINTER = "whole-suite-map"
|
|
19
22
|
LOCKFILES = %w[Gemfile.lock gems.locked].freeze
|
|
20
23
|
LOG = "coverage-build.log"
|
|
21
24
|
# RSpec rewrites this on every run (example_status_persistence_file_path),
|
|
@@ -31,7 +34,7 @@ module Saboteur
|
|
|
31
34
|
TAIL_LINE_WIDTH = 300
|
|
32
35
|
|
|
33
36
|
def initialize(config:, integration:, hooks:, supervisor:, diagnostics:, load_lines: {}, load_methods: {},
|
|
34
|
-
timeout_cap: nil, parts: 1)
|
|
37
|
+
timeout_cap: nil, parts: 1, whole_suite_run: false)
|
|
35
38
|
@config = config
|
|
36
39
|
@parts = parts
|
|
37
40
|
@integration = integration
|
|
@@ -41,6 +44,7 @@ module Saboteur
|
|
|
41
44
|
@timeout_cap = timeout_cap
|
|
42
45
|
@load_lines = load_lines
|
|
43
46
|
@load_methods = load_methods
|
|
47
|
+
@whole_suite_run = whole_suite_run
|
|
44
48
|
@root = File.realpath(config.root)
|
|
45
49
|
@root_prefixes = Selection::CoverageMap.root_prefixes(config.root)
|
|
46
50
|
end
|
|
@@ -58,14 +62,14 @@ module Saboteur
|
|
|
58
62
|
path = cache_path(files)
|
|
59
63
|
preloaded = LoadCoverage.preloaded_uninstrumented(files)
|
|
60
64
|
cached = cached_map(files)
|
|
61
|
-
return
|
|
65
|
+
return serve_cached(path, cached, files, preloaded) if cached
|
|
62
66
|
|
|
63
67
|
note = miss_note(files)
|
|
64
68
|
@diagnostics.inform(note.message(scoped: !@config["tests"].empty?)) if @config["cache"]
|
|
65
69
|
started = Process.clock_gettime(Process::CLOCK_MONOTONIC)
|
|
66
70
|
map = fresh_map(files, note, preloaded)
|
|
67
71
|
build_seconds = Process.clock_gettime(Process::CLOCK_MONOTONIC) - started
|
|
68
|
-
|
|
72
|
+
persist_map(path, map)
|
|
69
73
|
shadow(files, map, note, build_seconds) if shadow?
|
|
70
74
|
note.save
|
|
71
75
|
File.write(File.join(@config.path("cache_dir"), "preview.json"),
|
|
@@ -73,6 +77,31 @@ module Saboteur
|
|
|
73
77
|
map
|
|
74
78
|
end
|
|
75
79
|
|
|
80
|
+
# The map a whole-suite coverage build wrote on this tree, or nil.
|
|
81
|
+
# Survivor presentation consults it (presentation only, never
|
|
82
|
+
# selection): in a scoped run the run's own map holds only the specs
|
|
83
|
+
# the run loaded, so a "covered by" list from it can read as complete
|
|
84
|
+
# when another spec pins the behavior (false survivors by default,
|
|
85
|
+
# review C1). A full run keys its map with tests: [], so the lookup
|
|
86
|
+
# overrides a scoped run's --test value.
|
|
87
|
+
# The map this store's fetch produced (or served from cache): the
|
|
88
|
+
# run's own selection map, read by survivor presentation (review C1).
|
|
89
|
+
attr_reader :fetched_map
|
|
90
|
+
|
|
91
|
+
def whole_suite_map
|
|
92
|
+
return nil unless @config["cache"]
|
|
93
|
+
|
|
94
|
+
pointer = File.join(@config.path("cache_dir"), WHOLE_SUITE_POINTER)
|
|
95
|
+
return nil unless File.file?(pointer)
|
|
96
|
+
|
|
97
|
+
key = File.read(pointer, encoding: Encoding::UTF_8).strip
|
|
98
|
+
return nil unless key.match?(/\A[0-9a-f]{64}\z/)
|
|
99
|
+
|
|
100
|
+
Selection::CoverageMap.load(File.join(@config.path("cache_dir"), "coverage-#{key}.json"))
|
|
101
|
+
rescue SystemCallError, EncodingError
|
|
102
|
+
nil
|
|
103
|
+
end
|
|
104
|
+
|
|
76
105
|
private
|
|
77
106
|
|
|
78
107
|
# The partial map when it is on and safe, else the full build.
|
|
@@ -106,6 +135,28 @@ module Saboteur
|
|
|
106
135
|
@cached_map ||= Selection::CoverageMap.load(cache_path(files))
|
|
107
136
|
end
|
|
108
137
|
|
|
138
|
+
def serve_cached(path, cached, files, preloaded)
|
|
139
|
+
record_whole_suite_map(path)
|
|
140
|
+
refuse_empty_map(@fetched_map = cached.with_preloaded(preloaded), files, preloaded)
|
|
141
|
+
end
|
|
142
|
+
|
|
143
|
+
def persist_map(path, map)
|
|
144
|
+
map.save(path)
|
|
145
|
+
record_whole_suite_map(path)
|
|
146
|
+
@fetched_map = map
|
|
147
|
+
end
|
|
148
|
+
|
|
149
|
+
# The engine tells us when this run is a whole-suite build (no
|
|
150
|
+
# --since, no subjects, no explicit --test): remember the map key for
|
|
151
|
+
# later scoped runs, whose tree has usually drifted past the exact
|
|
152
|
+
# cache key already (review C1).
|
|
153
|
+
def record_whole_suite_map(path)
|
|
154
|
+
return unless @whole_suite_run
|
|
155
|
+
|
|
156
|
+
File.write(File.join(@config.path("cache_dir"), WHOLE_SUITE_POINTER),
|
|
157
|
+
File.basename(path, ".json").delete_prefix("coverage-"), perm: 0o600)
|
|
158
|
+
end
|
|
159
|
+
|
|
109
160
|
def cache_key(files)
|
|
110
161
|
Selection::CoverageMap.cache_key(files: key_files(files), extra: key_inputs)
|
|
111
162
|
end
|
|
@@ -110,7 +110,11 @@ module Saboteur
|
|
|
110
110
|
# scheduler stopped mid-list at the budget deadline. Worker-level error
|
|
111
111
|
# rows synthesize an error per candidate, so they never look truncated.
|
|
112
112
|
def unverified_result?(result)
|
|
113
|
-
result.is_a?(MutantResult) && result.status == :unverified
|
|
113
|
+
result.is_a?(MutantResult) && result.status == :unverified && !result.flaky?
|
|
114
|
+
end
|
|
115
|
+
|
|
116
|
+
def budget_unverified_row?(result)
|
|
117
|
+
result["status"] == "unverified" && !MutantResult.flaky_reason?(result["reason"])
|
|
114
118
|
end
|
|
115
119
|
|
|
116
120
|
def record_unreached(rows, subjects, lists)
|
|
@@ -125,7 +129,7 @@ module Saboteur
|
|
|
125
129
|
next unless row["results"] && !row["error"]
|
|
126
130
|
|
|
127
131
|
@truncated << [subjects[i], lists[i].size - row["results"].size] if row["results"].size < lists[i].size
|
|
128
|
-
@unverified << subjects[i] if row["results"].any? { |result| result
|
|
132
|
+
@unverified << subjects[i] if row["results"].any? { |result| budget_unverified_row?(result) }
|
|
129
133
|
end
|
|
130
134
|
end
|
|
131
135
|
|
|
@@ -24,7 +24,9 @@ module Saboteur
|
|
|
24
24
|
mutate(subjects, files, integration, scope, ignore_list,
|
|
25
25
|
load_lines:, load_methods:)
|
|
26
26
|
end
|
|
27
|
-
|
|
27
|
+
warnings = ignore_list.apply(results, subjects: subjects,
|
|
28
|
+
unscoped_digests: @unscoped_digests, full_run: unscoped_full_run?)
|
|
29
|
+
@diagnostics.add_warnings(warnings)
|
|
28
30
|
build_report(results, subjects, scope, started)
|
|
29
31
|
ensure
|
|
30
32
|
# Cleanup runs however the call ends - a partway boot failure, no
|
|
@@ -54,7 +56,9 @@ module Saboteur
|
|
|
54
56
|
files = config.included_files
|
|
55
57
|
subjects, scope = select_subjects(files)
|
|
56
58
|
results = subjects.empty? ? [] : mutate_external(subjects, scope, ignore_list)
|
|
57
|
-
|
|
59
|
+
warnings = ignore_list.apply(results, subjects: subjects,
|
|
60
|
+
unscoped_digests: @unscoped_digests, full_run: unscoped_full_run?)
|
|
61
|
+
@diagnostics.add_warnings(warnings)
|
|
58
62
|
build_report(results, subjects, scope, started)
|
|
59
63
|
end
|
|
60
64
|
|
|
@@ -97,6 +101,14 @@ module Saboteur
|
|
|
97
101
|
@diagnostics.warn(warning) if warning
|
|
98
102
|
end
|
|
99
103
|
|
|
104
|
+
# "Full run" for ignore staleness (#1206): no --since diff, no --subject
|
|
105
|
+
# filter, and no narrowed include (explicit file arguments replace the
|
|
106
|
+
# configured include) - only then was every in-scope subject examined, so
|
|
107
|
+
# an entry whose subject never appears really points at gone code.
|
|
108
|
+
def unscoped_full_run?
|
|
109
|
+
!config["since"] && config["subjects"].empty? && config["include"] == Config::DEFAULTS["include"]
|
|
110
|
+
end
|
|
111
|
+
|
|
100
112
|
def external_command? = !config["test_command"].to_s.empty?
|
|
101
113
|
|
|
102
114
|
# The preflight's absolute monotonic deadline: its own
|
|
@@ -352,7 +352,7 @@ module Saboteur
|
|
|
352
352
|
def flaky_message(outcome, rerun)
|
|
353
353
|
decisive = (outcome.failed_ids | rerun.failed_ids).first(3)
|
|
354
354
|
named = decisive.empty? ? "" : " (decisive tests: #{decisive.join(', ')})"
|
|
355
|
-
"
|
|
355
|
+
"#{MutantResult::FLAKY_PREFIX}: first #{outcome.status}, then #{rerun.status} on a reordered confirmation " \
|
|
356
356
|
"run#{named} - the mutant is unverified, not #{outcome.status}"
|
|
357
357
|
end
|
|
358
358
|
|
|
@@ -11,12 +11,13 @@ module Saboteur
|
|
|
11
11
|
private
|
|
12
12
|
|
|
13
13
|
# A --since diff with no changed source file (a spec-only or docs-only
|
|
14
|
-
# change) yields zero subjects, so the suite never runs: say
|
|
15
|
-
#
|
|
16
|
-
# happen (#607).
|
|
14
|
+
# change) yields zero subjects, so the suite never runs: say why no
|
|
15
|
+
# spec was inferred rather than promise a whole-suite run that does not
|
|
16
|
+
# happen (#607). The conclusion itself is stated once, by the
|
|
17
|
+
# "--since <ref>: nothing to test - <reason>" outcome line (#1232).
|
|
17
18
|
def selection_message(guess)
|
|
18
19
|
if guess.changed.zero? && config["since"]
|
|
19
|
-
"inferred specs: no changed source file to infer a spec for
|
|
20
|
+
"inferred specs: no changed source file to infer a spec for"
|
|
20
21
|
elsif guess.changed.zero?
|
|
21
22
|
"inferred specs: no source file to infer a spec for, so the whole suite runs"
|
|
22
23
|
elsif guess.specs.empty?
|
|
@@ -169,6 +169,11 @@ module Saboteur
|
|
|
169
169
|
return missed.map { |pattern| missed_pattern_message(pattern) }.join("; ")
|
|
170
170
|
end
|
|
171
171
|
|
|
172
|
+
if @config["since"]
|
|
173
|
+
return "--subject #{missed.join(', ')} matches no subject in the --since #{@config['since']} " \
|
|
174
|
+
"diff scope (the scope limits the search to changed files; drop --since or pass --all)"
|
|
175
|
+
end
|
|
176
|
+
|
|
172
177
|
"--subject #{missed.join(', ')} matches no subject (patterns are names like " \
|
|
173
178
|
"Klass, Klass#meth or Klass.meth, not globs)"
|
|
174
179
|
end
|
|
@@ -180,6 +185,11 @@ module Saboteur
|
|
|
180
185
|
macro = attribute_macro_for(pattern)
|
|
181
186
|
return "--subject #{pattern} is defined by #{macro} and has no body to mutate" if macro
|
|
182
187
|
|
|
188
|
+
if @config["since"]
|
|
189
|
+
return "--subject #{pattern} matches no subject in the --since #{@config['since']} diff scope " \
|
|
190
|
+
"(the scope limits the search to changed files)"
|
|
191
|
+
end
|
|
192
|
+
|
|
183
193
|
"fix the --subject pattern \"#{pattern}\": it matches no subject (patterns are names like " \
|
|
184
194
|
"Klass, Klass#meth or Klass.meth, not globs)"
|
|
185
195
|
end
|
data/lib/saboteur/engine.rb
CHANGED
|
@@ -4,6 +4,7 @@
|
|
|
4
4
|
require "fileutils"
|
|
5
5
|
require "json"
|
|
6
6
|
require_relative "replay"
|
|
7
|
+
require_relative "text"
|
|
7
8
|
require_relative "mutation_source/hazard_guard"
|
|
8
9
|
require_relative "baseline/run_evidence"
|
|
9
10
|
require_relative "engine/early_stop_recording"
|
|
@@ -116,6 +117,24 @@ module Saboteur
|
|
|
116
117
|
warnings + map.duplicate_keys(path).map { |key| "duplicate custom map key #{key}: Ruby keeps the last" }
|
|
117
118
|
end
|
|
118
119
|
|
|
120
|
+
# The map this run selected tests from: survivor presentation compares
|
|
121
|
+
# it against the cached whole-suite map (review C1).
|
|
122
|
+
def run_coverage_map
|
|
123
|
+
@coverage_store&.fetched_map
|
|
124
|
+
end
|
|
125
|
+
|
|
126
|
+
# The cached whole-suite map on this tree, for survivor presentation
|
|
127
|
+
# in scoped runs; nil when caching is off or no full build ran (review C1).
|
|
128
|
+
def whole_suite_coverage_map
|
|
129
|
+
@coverage_store&.whole_suite_map
|
|
130
|
+
end
|
|
131
|
+
|
|
132
|
+
# False when caching is off, where no coverage build can ever leave a
|
|
133
|
+
# whole-suite map to consult, so advising one would be wrong (review C1).
|
|
134
|
+
def whole_suite_map_lookup?
|
|
135
|
+
config["cache"] ? true : false
|
|
136
|
+
end
|
|
137
|
+
|
|
119
138
|
private
|
|
120
139
|
|
|
121
140
|
# Inferred test selection, the default since 0.7.0: scope the run to the
|
|
@@ -312,16 +331,27 @@ module Saboteur
|
|
|
312
331
|
tests: map.tests.size, cached: @map_cached)
|
|
313
332
|
@diagnostics.quiet_inference(@quiet_spec_count)
|
|
314
333
|
else
|
|
315
|
-
@diagnostics.inform("#{subjects.size
|
|
334
|
+
@diagnostics.inform("#{Text.counted(subjects.size, 'subject')}, " \
|
|
335
|
+
"#{Text.counted(map.tests.size, 'test')} in coverage map")
|
|
316
336
|
end
|
|
317
337
|
ThreadWarnings.warn(@diagnostics, rails: config["rails"])
|
|
318
338
|
end
|
|
319
339
|
|
|
340
|
+
# This run's map is the whole-suite map only when nothing scoped it:
|
|
341
|
+
# no --since, no subjects, and tests either unset or inferred over every
|
|
342
|
+
# included file (never an explicit --test list) (review C1).
|
|
343
|
+
def whole_suite_run?
|
|
344
|
+
config["since"].nil? && config["subjects"].empty? &&
|
|
345
|
+
(config["tests"].empty? || !@guessed_tests.nil?)
|
|
346
|
+
end
|
|
347
|
+
|
|
320
348
|
def coverage_store(integration, load_lines, load_methods = {})
|
|
321
|
-
|
|
322
|
-
|
|
323
|
-
|
|
324
|
-
|
|
349
|
+
@coverage_store ||=
|
|
350
|
+
CoverageStore.new(config: config, integration: integration, hooks: hooks, load_lines:, load_methods:,
|
|
351
|
+
whole_suite_run: whole_suite_run?,
|
|
352
|
+
supervisor: supervisor(memory_factor: 4), diagnostics: @diagnostics,
|
|
353
|
+
timeout_cap: method(:budget_capped_timeout),
|
|
354
|
+
parts: config["coverage_parts"])
|
|
325
355
|
end
|
|
326
356
|
|
|
327
357
|
def check_baseline(integration, full_passes)
|
data/lib/saboteur/help.rb
CHANGED
|
@@ -152,6 +152,17 @@ module Saboteur
|
|
|
152
152
|
["Diff, replay command and ignore template for one mutant",
|
|
153
153
|
"saboteur survivors .saboteur/report.json --show 1a2b3c4d5e6f7a8b"]]
|
|
154
154
|
),
|
|
155
|
+
Command.new(
|
|
156
|
+
name: "explain",
|
|
157
|
+
summary: "Explain one mutation operator: definition, example and kill guidance",
|
|
158
|
+
description: "Prints what one operator does, a before/after example of the mutation, the test " \
|
|
159
|
+
"shapes that let it survive, and the assertion that kills it. Covers every operator " \
|
|
160
|
+
"the registry can emit.",
|
|
161
|
+
takes_options: false,
|
|
162
|
+
exit_codes: { 0 => "Explanation printed.",
|
|
163
|
+
2 => "Unknown or missing operator name; the valid names are listed." },
|
|
164
|
+
examples: [["Why did this mutant survive?", "saboteur explain operand_replacement"]]
|
|
165
|
+
),
|
|
155
166
|
Command.new(
|
|
156
167
|
name: "badge",
|
|
157
168
|
summary: "Print a shields.io endpoint badge over the opt-in score history",
|