saboteur 0.11.1 → 0.12.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +64 -0
- data/README.md +1 -1
- data/lib/saboteur/cli/exit_status.rb +6 -2
- data/lib/saboteur/cli/explain_command.rb +51 -0
- data/lib/saboteur/cli/flaky_exit.rb +23 -0
- data/lib/saboteur/cli/output_options.rb +4 -0
- data/lib/saboteur/cli/run_command.rb +19 -0
- data/lib/saboteur/cli/run_output.rb +10 -0
- data/lib/saboteur/cli/run_reporting.rb +14 -3
- data/lib/saboteur/cli/survivors_command.rb +2 -1
- data/lib/saboteur/cli.rb +4 -1
- data/lib/saboteur/config.rb +25 -3
- data/lib/saboteur/doctor.rb +8 -1
- data/lib/saboteur/engine/coverage_store.rb +54 -3
- data/lib/saboteur/engine/diagnostics.rb +4 -3
- data/lib/saboteur/engine/distribution.rb +6 -2
- data/lib/saboteur/engine/pipelines.rb +14 -2
- data/lib/saboteur/engine/scheduler.rb +1 -1
- data/lib/saboteur/engine/selection_announcement.rb +5 -4
- data/lib/saboteur/engine/subject_selection.rb +10 -0
- data/lib/saboteur/engine.rb +70 -12
- data/lib/saboteur/help.rb +11 -0
- data/lib/saboteur/ignore_list.rb +22 -2
- data/lib/saboteur/mutation_source/builtin.rb +1 -0
- data/lib/saboteur/mutation_source/operator_explain.rb +181 -0
- data/lib/saboteur/reporter/agent_commands.rb +23 -6
- data/lib/saboteur/reporter/agent_incomplete.rb +9 -2
- data/lib/saboteur/reporter/agent_summary.rb +2 -1
- data/lib/saboteur/reporter/default_findings.rb +83 -0
- data/lib/saboteur/reporter/gap_kinds.rb +80 -0
- data/lib/saboteur/reporter/html.rb +19 -4
- data/lib/saboteur/reporter/next_steps.rb +1 -0
- data/lib/saboteur/reporter/review_comments.rb +8 -5
- data/lib/saboteur/reporter/score_table.rb +8 -7
- data/lib/saboteur/reporter/speed_footer.rb +74 -0
- data/lib/saboteur/reporter/survivor_classification.rb +31 -8
- data/lib/saboteur/reporter/survivor_hints.rb +5 -3
- data/lib/saboteur/reporter/survivor_list.rb +78 -7
- data/lib/saboteur/reporter/terminal.rb +52 -36
- data/lib/saboteur/reporter/wording.rb +37 -0
- data/lib/saboteur/result.rb +28 -2
- data/lib/saboteur/survivor_guidance.rb +2 -2
- data/lib/saboteur/text.rb +6 -0
- data/lib/saboteur/version.rb +1 -1
- metadata +8 -1
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 7e61c3755190680c12faf62b3a0bcad5f74750da7ff3e3716736e8b9aa733c46
|
|
4
|
+
data.tar.gz: 85dc6bbddcc174034da44607b55c5c438c883bc5e0c5954a68d8115abb4b5984
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: eceece31e29ceb03758b67aaa4ecfddcf1f6ca1f864358706fabcb8c9fed1f725da5548cd4f810684ac85250651f234b182452dd0525309711b616b997bf8026
|
|
7
|
+
data.tar.gz: f62d86ac983b24a550ba5696372fee7f9883be0103e823997f5fdd64c7150b42a8d88aae38881cd65ecd3b2141b39c22c9b7513552506a6499dddceb30128a69
|
data/CHANGELOG.md
CHANGED
|
@@ -5,6 +5,70 @@ All notable changes are documented here. The format follows
|
|
|
5
5
|
[Semantic Versioning](https://semver.org/).
|
|
6
6
|
|
|
7
7
|
## [Unreleased]
|
|
8
|
+
|
|
9
|
+
## [0.12.0] - 2026-10-22
|
|
10
|
+
|
|
11
|
+
### Added
|
|
12
|
+
|
|
13
|
+
- The default report is short: a findings list, the top three expanded
|
|
14
|
+
survivor blocks and `--details` for the rest, with a per-phase
|
|
15
|
+
wall-time speed footer and the slowest mutant. Fixed template lines
|
|
16
|
+
fit 80 columns and the piped-progress margin is wider so the summary
|
|
17
|
+
fits 80-column logs (#1256, #1152).
|
|
18
|
+
- A run on a Rails app with no `config/saboteur.rb` now defaults
|
|
19
|
+
`include` to `app/**/*.rb` and says so in the missing-config notice,
|
|
20
|
+
instead of exiting 2. A written config, even an empty one, keeps the
|
|
21
|
+
plain defaults, and a CLI `--include` still wins (#1257, #1171).
|
|
22
|
+
- The terminal summary prints one dim line under Score when mutants sit
|
|
23
|
+
on lines no test runs, saying they are not in the score and that the
|
|
24
|
+
JSON report lists them as `NoCoverage` (#1259, #1190).
|
|
25
|
+
|
|
26
|
+
### Changed
|
|
27
|
+
|
|
28
|
+
- Uncovered survivors are labeled `input_gap`, separate from assertion
|
|
29
|
+
gaps (#1254, #1157).
|
|
30
|
+
|
|
31
|
+
### Tests
|
|
32
|
+
|
|
33
|
+
- End-to-end pins for `--shard K/N` and the score history, badge and
|
|
34
|
+
trend commands, and for a stub-only method reported as not executed
|
|
35
|
+
(#1260, #1261, #1158, #1159). The survivor-ranking regression
|
|
36
|
+
fixture is the real money gem `allocation.rb` (#1262, #1160).
|
|
37
|
+
|
|
38
|
+
## [0.11.2] - 2026-10-22
|
|
39
|
+
|
|
40
|
+
### Added
|
|
41
|
+
|
|
42
|
+
- A full run warns when an ignore entry's subject never appears, so a
|
|
43
|
+
renamed or removed method can no longer kill the entry silently
|
|
44
|
+
(#1250, #1206).
|
|
45
|
+
- Survivor gaps are labeled by kind instead of the blanket
|
|
46
|
+
`missing_test` label (#1246, #1153, #1064), and the README names
|
|
47
|
+
`tool_limit` and the gap kinds in the survivor triage labels (#1247).
|
|
48
|
+
- The `explain` command prints a human-readable rationale for any
|
|
49
|
+
registry operator (#1251).
|
|
50
|
+
- The HTML report lists survivors, timeouts and errors before killed
|
|
51
|
+
rows inside each file, keeping source order within each group
|
|
52
|
+
(#1253, #1233).
|
|
53
|
+
- Survivor blocks consult the whole-suite coverage map, ending false
|
|
54
|
+
survivors from diff-scoped runs (#1252).
|
|
55
|
+
|
|
56
|
+
### Changed
|
|
57
|
+
|
|
58
|
+
- Output polish across the terminal, JSON and HTML reports: the flaky
|
|
59
|
+
check names why it exits 3, `doctor` prints its coverage line and a
|
|
60
|
+
run without a config says so (#1239); stubbed-test wording, the
|
|
61
|
+
timeout tally and the replay scope line are clearer (#1240);
|
|
62
|
+
selection messaging states nothing-to-test once and names the
|
|
63
|
+
`--since` scope on `--subject` misses (#1241); survivor blocks count
|
|
64
|
+
blocks, not mutants (#1242); the run opener and coverage-map counts
|
|
65
|
+
are pluralized (#1244); the clean screen labels its scope, hints at
|
|
66
|
+
the live reports and summarizes an empty diff (#1245).
|
|
67
|
+
- Docs: the ignore list is named as the one way to accept a survivor,
|
|
68
|
+
the pre-1.0 policy for test selection and operator defaults is
|
|
69
|
+
stated, and the CLI-docs rule points at `docs/` rather than the
|
|
70
|
+
short README (#1248).
|
|
71
|
+
|
|
8
72
|
## [0.11.1] - 2026-10-22
|
|
9
73
|
|
|
10
74
|
### Added
|
data/README.md
CHANGED
|
@@ -80,7 +80,7 @@ tree refuses with exit 2 and points at the generator. Engine gem with
|
|
|
80
80
|
assertion, 1 by a crash`), and an error-dominated run prints
|
|
81
81
|
`Score: n/a` instead of a number you would misread.
|
|
82
82
|
- 🔍 **Survivors you can act on.** Survivors group per statement with a
|
|
83
|
-
triage label (`likely_equivalent`, `arid`, `
|
|
83
|
+
triage label (`likely_equivalent`, `arid`, `tool_limit`, or a gap kind) and their
|
|
84
84
|
covering tests - split into the tests that ran against the mutant and
|
|
85
85
|
the ones that never did. `saboteur survivors` browses them read-only:
|
|
86
86
|
one digest shows the full diff and the exact replay command.
|
|
@@ -1,11 +1,15 @@
|
|
|
1
1
|
# SPDX-License-Identifier: MIT
|
|
2
2
|
# frozen_string_literal: true
|
|
3
3
|
|
|
4
|
+
require_relative "flaky_exit"
|
|
5
|
+
|
|
4
6
|
module Saboteur
|
|
5
7
|
class CLI
|
|
6
8
|
# How a finished run maps to an exit code and a recorded exit cause. Mixed
|
|
7
9
|
# into the CLI, which supplies `config` and the `@err` stream.
|
|
8
10
|
module ExitStatus
|
|
11
|
+
include FlakyExit
|
|
12
|
+
|
|
9
13
|
private
|
|
10
14
|
|
|
11
15
|
def exit_code(report)
|
|
@@ -15,7 +19,7 @@ module Saboteur
|
|
|
15
19
|
|
|
16
20
|
return EXIT_TOOL if untested_partial?(report)
|
|
17
21
|
|
|
18
|
-
return EXIT_TOOL if unverifiable_scope?(report)
|
|
22
|
+
return EXIT_TOOL if unverifiable_scope?(report) || flaky_disagreement?(report)
|
|
19
23
|
|
|
20
24
|
return EXIT_TOOL if mostly_errored?(report)
|
|
21
25
|
|
|
@@ -117,7 +121,7 @@ module Saboteur
|
|
|
117
121
|
else
|
|
118
122
|
if report.majority_errored?
|
|
119
123
|
"errorMajority"
|
|
120
|
-
elsif unverifiable_review?(report) || unverifiable_unreachable?(report) ||
|
|
124
|
+
elsif unverifiable_review?(report) || unverifiable_unreachable?(report) || report.results.any?(&:flaky?) ||
|
|
121
125
|
(report.early_stop && report.score_breakdown.tested.zero?)
|
|
122
126
|
"incomplete"
|
|
123
127
|
else
|
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
# SPDX-License-Identifier: MIT
|
|
2
|
+
# frozen_string_literal: true
|
|
3
|
+
|
|
4
|
+
require_relative "../help"
|
|
5
|
+
require_relative "../reporter/palette"
|
|
6
|
+
require_relative "../mutation_source/operator_explain"
|
|
7
|
+
|
|
8
|
+
module Saboteur
|
|
9
|
+
class CLI
|
|
10
|
+
# Shared rendering for `saboteur explain <operator>` (#1097): the
|
|
11
|
+
# curated definition, a before/after example and survive/kill guidance
|
|
12
|
+
# for one operator, so a survivor's operator can be understood and
|
|
13
|
+
# targeted without leaving the terminal.
|
|
14
|
+
module ExplainCommand
|
|
15
|
+
private
|
|
16
|
+
|
|
17
|
+
def explain_command
|
|
18
|
+
name = @argv.shift
|
|
19
|
+
raise UsageError, "usage: saboteur explain <operator>" if name.nil?
|
|
20
|
+
|
|
21
|
+
entry = MutationSource::OperatorExplain.for(name)
|
|
22
|
+
if entry.nil?
|
|
23
|
+
@err.puts("saboteur: unknown operator: #{name}")
|
|
24
|
+
suggestion = MutationSource::OperatorRegistry.suggestion(name)
|
|
25
|
+
@err.puts("saboteur: #{suggestion}") unless suggestion.empty?
|
|
26
|
+
@err.puts("saboteur: operators: #{MutationSource::OperatorRegistry::DESCRIPTIONS.keys.join(', ')}")
|
|
27
|
+
return EXIT_USAGE
|
|
28
|
+
end
|
|
29
|
+
|
|
30
|
+
@out.puts(render_explain(entry, palette: Reporter::Palette.for(@out, mode: :auto)))
|
|
31
|
+
EXIT_OK
|
|
32
|
+
rescue UsageError => e
|
|
33
|
+
@err.puts("saboteur: #{e.message}")
|
|
34
|
+
EXIT_USAGE
|
|
35
|
+
end
|
|
36
|
+
|
|
37
|
+
def render_explain(entry, palette: Reporter::Palette.new(enabled: false))
|
|
38
|
+
original, replacement = entry.example.split(" -> ")
|
|
39
|
+
[
|
|
40
|
+
"#{palette.paint(:head, 'Operator:')} #{entry.name}",
|
|
41
|
+
"#{palette.paint(:head, 'Definition:')} #{entry.definition}",
|
|
42
|
+
palette.paint(:head, "Example:"),
|
|
43
|
+
" #{palette.paint(:fail, "- #{original}")}",
|
|
44
|
+
" #{palette.paint(:pass, "+ #{replacement}")}",
|
|
45
|
+
"#{palette.paint(:warn, 'Survives when:')} #{entry.survives}",
|
|
46
|
+
"#{palette.paint(:info, 'Kill it with:')} #{entry.kill}"
|
|
47
|
+
].join("\n")
|
|
48
|
+
end
|
|
49
|
+
end
|
|
50
|
+
end
|
|
51
|
+
end
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
# SPDX-License-Identifier: MIT
|
|
2
|
+
# frozen_string_literal: true
|
|
3
|
+
|
|
4
|
+
module Saboteur
|
|
5
|
+
class CLI
|
|
6
|
+
# --flaky-check (#1167): a verdict that flipped on the reordered rerun is
|
|
7
|
+
# unverified, outside the score. The run was not stopped by a budget, so
|
|
8
|
+
# it is not exit 4, but it cannot go green either.
|
|
9
|
+
module FlakyExit
|
|
10
|
+
private
|
|
11
|
+
|
|
12
|
+
def flaky_disagreement?(report)
|
|
13
|
+
flaky = report.results.select(&:flaky?)
|
|
14
|
+
return false if flaky.empty?
|
|
15
|
+
|
|
16
|
+
@err.puts("saboteur: #{flaky.size} mutant#{'s' unless flaky.one?} changed verdict on the --flaky-check " \
|
|
17
|
+
"reordered rerun (#{flaky.first(3).map(&:digest).join(', ')}), so they are unverified: " \
|
|
18
|
+
"your tests depend on run order; fix that and run again")
|
|
19
|
+
true
|
|
20
|
+
end
|
|
21
|
+
end
|
|
22
|
+
end
|
|
23
|
+
end
|
|
@@ -42,6 +42,10 @@ module Saboteur
|
|
|
42
42
|
end
|
|
43
43
|
opts.on("--all-mutants", "Text report: list every survived mutant;",
|
|
44
44
|
"default: one finding per statement, with the label mix") { set(:all_mutants, true) }
|
|
45
|
+
# D1: the default whole-project report lists findings and expands the
|
|
46
|
+
# top three; --details restores a block for every finding.
|
|
47
|
+
opts.on("--details", "Text report: full blocks for every finding;",
|
|
48
|
+
"default: one-line list plus the top 3 blocks") { set(:details, true) }
|
|
45
49
|
opts.on("--no-color", "Deprecated alias for --color=never") { @color = :never }
|
|
46
50
|
end
|
|
47
51
|
end
|
|
@@ -17,6 +17,24 @@ module Saboteur
|
|
|
17
17
|
|
|
18
18
|
private
|
|
19
19
|
|
|
20
|
+
# A fresh checkout has no config file: say once that defaults are in
|
|
21
|
+
# use, and who writes one (#1172). Agent output stays machine-clean;
|
|
22
|
+
# there is no init command - the Rails generator writes the config.
|
|
23
|
+
def announce_missing_config_file(palette)
|
|
24
|
+
# @format, not agent_requested?: parse! has already eaten --format.
|
|
25
|
+
return if @format == "agent" || config.config_file_present
|
|
26
|
+
|
|
27
|
+
path = options[:config] || Config::FILE
|
|
28
|
+
notice =
|
|
29
|
+
if config["include"] == Config::RAILS_INCLUDE && !config.explicit?("include")
|
|
30
|
+
"saboteur: no #{path} - Rails app detected, mutating app/**/*.rb " \
|
|
31
|
+
"(bundle exec rails generate saboteur:install writes a config)"
|
|
32
|
+
else
|
|
33
|
+
"saboteur: no #{path} - using defaults (Rails: bundle exec rails generate saboteur:install writes one)"
|
|
34
|
+
end
|
|
35
|
+
@err.puts(palette.paint(:dim, notice))
|
|
36
|
+
end
|
|
37
|
+
|
|
20
38
|
def baseline_red?(error)
|
|
21
39
|
error.message.to_s.match?(BASELINE_RED_MESSAGE)
|
|
22
40
|
end
|
|
@@ -34,6 +52,7 @@ module Saboteur
|
|
|
34
52
|
config.deprecation_warnings.each do |warning|
|
|
35
53
|
@err.puts(Reporter::Notice.paint(notices, "saboteur: #{warning}"))
|
|
36
54
|
end
|
|
55
|
+
announce_missing_config_file(notices)
|
|
37
56
|
|
|
38
57
|
return preview if options[:list_operators] || options[:dry_run]
|
|
39
58
|
|
|
@@ -21,6 +21,16 @@ module Saboteur
|
|
|
21
21
|
@out.puts "\nReports: #{names.join(', ')}"
|
|
22
22
|
end
|
|
23
23
|
|
|
24
|
+
# The clean screen's scope label (#1173, #1174): --since, positional
|
|
25
|
+
# sources and --subject each name what the run covered.
|
|
26
|
+
def scope_label
|
|
27
|
+
parts = []
|
|
28
|
+
parts << "--since #{config['since']}" if config["since"]
|
|
29
|
+
parts.concat(@sources) if @sources
|
|
30
|
+
parts << "--subject #{config['subjects'].join(', ')}" if config["subjects"].any?
|
|
31
|
+
parts.empty? ? "whole project" : parts.join(", ")
|
|
32
|
+
end
|
|
33
|
+
|
|
24
34
|
# Agent mode: stdout carries exactly one JSON payload. The target's own output (boot
|
|
25
35
|
# files, hooks, spec loading, child processes) goes to stderr while the engine runs.
|
|
26
36
|
# Descriptor 1 points at stderr for the block, so Kernel#puts, system and forked
|
|
@@ -21,8 +21,10 @@ module Saboteur
|
|
|
21
21
|
Reporter::Terminal.new(@out, verbose: @verbose || false, quiet: @quiet || false, color: @color || :auto,
|
|
22
22
|
overrides: config["palette"], threshold: config["threshold"],
|
|
23
23
|
replay_command: survivor_replay_command(report, engine),
|
|
24
|
-
all_mutants: options[:all_mutants] || false,
|
|
25
|
-
exit_code: code, previous_score: previous_score,
|
|
24
|
+
all_mutants: options[:all_mutants] || false, details: options[:details] || false,
|
|
25
|
+
exit_code: code, previous_score: previous_score, scope: scope_label,
|
|
26
|
+
whole_suite_map: whole_suite_map_for(engine),
|
|
27
|
+
run_map: engine&.run_coverage_map,
|
|
26
28
|
baseline_gate: (baseline_gate(report) if config["baseline"])).call(report)
|
|
27
29
|
Reporter::Replay.new(@out).call(report) if options[:id]
|
|
28
30
|
# The JSON carries the exit cause an outer harness trusts, so it is
|
|
@@ -38,6 +40,14 @@ module Saboteur
|
|
|
38
40
|
code
|
|
39
41
|
end
|
|
40
42
|
|
|
43
|
+
# No engine or no cache: no whole-suite map can exist, so the survivor
|
|
44
|
+
# block keeps its legacy covered-by line over an impossible build (review C1).
|
|
45
|
+
def whole_suite_map_for(engine)
|
|
46
|
+
return engine.whole_suite_coverage_map if engine&.whole_suite_map_lookup?
|
|
47
|
+
|
|
48
|
+
Reporter::SurvivorList::NO_WHOLE_SUITE_LOOKUP
|
|
49
|
+
end
|
|
50
|
+
|
|
41
51
|
def agent_run(arguments)
|
|
42
52
|
tree = WorkingTree.new(config.root, ignored: config.ignored_tree_paths).state
|
|
43
53
|
# The machine path is exempt from the noise diet: quiet_start: false
|
|
@@ -94,7 +104,8 @@ module Saboteur
|
|
|
94
104
|
def survivor_replay_command(report, engine)
|
|
95
105
|
lambda do |digest|
|
|
96
106
|
tree = WorkingTree.new(config.root, ignored: config.ignored_tree_paths).state
|
|
97
|
-
Reporter::Agent::Commands.replay(agent_provenance(report, tree, engine), digest, capped: false
|
|
107
|
+
Reporter::Agent::Commands.replay(agent_provenance(report, tree, engine), digest, capped: false,
|
|
108
|
+
report: report)
|
|
98
109
|
end
|
|
99
110
|
end
|
|
100
111
|
|
|
@@ -31,7 +31,7 @@ module Saboteur
|
|
|
31
31
|
opts.on("--ranked", "Review-priority order: the report's ranked findings first")
|
|
32
32
|
opts.on("--file SUBSTR", "Only survivors whose path contains SUBSTR")
|
|
33
33
|
opts.on("--operator NAME", "Only survivors of operator NAME")
|
|
34
|
-
opts.on("--triage LABEL", "Only survivors classified LABEL (for example
|
|
34
|
+
opts.on("--triage LABEL", "Only survivors classified LABEL (for example boundary)")
|
|
35
35
|
opts.on("--show DIGEST", "Diff, replay command and ignore template for one survivor")
|
|
36
36
|
end
|
|
37
37
|
options = {}
|
|
@@ -89,6 +89,7 @@ module Saboteur
|
|
|
89
89
|
" #{SurvivorGuidance::REPLAY_CAVEAT}",
|
|
90
90
|
"",
|
|
91
91
|
SurvivorGuidance::IGNORE_INTRO,
|
|
92
|
+
" #{SurvivorGuidance::IGNORE_CONFIG}",
|
|
92
93
|
" #{ignore_entry(survivor, subject)}"
|
|
93
94
|
].join("\n")
|
|
94
95
|
end
|
data/lib/saboteur/cli.rb
CHANGED
|
@@ -15,6 +15,7 @@ require_relative "cli/baseline_gate"
|
|
|
15
15
|
require_relative "preview"
|
|
16
16
|
require_relative "cli/run_reporting"
|
|
17
17
|
require_relative "cli/survivors_command"
|
|
18
|
+
require_relative "cli/explain_command"
|
|
18
19
|
require_relative "cli/source_set"
|
|
19
20
|
require_relative "cli/run_budget_options"
|
|
20
21
|
require_relative "cli/shard_options"
|
|
@@ -44,6 +45,7 @@ module Saboteur
|
|
|
44
45
|
include BaselineGate
|
|
45
46
|
include RunReporting
|
|
46
47
|
include SurvivorsCommand
|
|
48
|
+
include ExplainCommand
|
|
47
49
|
include SourceSet
|
|
48
50
|
include RunBudgetOptions
|
|
49
51
|
include ShardOptions
|
|
@@ -103,6 +105,7 @@ module Saboteur
|
|
|
103
105
|
when "run" then run_command
|
|
104
106
|
when "baseline" then baseline
|
|
105
107
|
when "survivors" then survivors_command
|
|
108
|
+
when "explain" then explain_command
|
|
106
109
|
when "doctor" then doctor_command
|
|
107
110
|
when "badge", "trend" then history_entry(command)
|
|
108
111
|
when "version", "--version" then @out.puts(VERSION) || EXIT_OK
|
|
@@ -136,7 +139,7 @@ module Saboteur
|
|
|
136
139
|
|
|
137
140
|
def agent_requested? = self.class.agent_requested?(@argv)
|
|
138
141
|
|
|
139
|
-
def overrides = options.except(:config, :id, :suite, :list_operators, :dry_run, :all_mutants, :quiet)
|
|
142
|
+
def overrides = options.except(:config, :id, :suite, :list_operators, :dry_run, :all_mutants, :quiet, :details)
|
|
140
143
|
|
|
141
144
|
def add_subject(expr) = (options[:subjects] ||= []) << expr
|
|
142
145
|
|
data/lib/saboteur/config.rb
CHANGED
|
@@ -18,6 +18,11 @@ module Saboteur
|
|
|
18
18
|
LEGACY_IGNORE_FILE = ".saboteur-ignore.yml"
|
|
19
19
|
IGNORE_FILE = "config/saboteur_ignore.rb"
|
|
20
20
|
|
|
21
|
+
# A Rails app (config/application.rb + app/) without a config file gets
|
|
22
|
+
# the generator's include instead of the gem-shaped default, so a first
|
|
23
|
+
# run mutates app/ rather than refusing over an empty source set (#1171).
|
|
24
|
+
RAILS_INCLUDE = ["app/**/*.rb"].freeze
|
|
25
|
+
|
|
21
26
|
# Settings that say where saboteur writes, never what it mutates: left
|
|
22
27
|
# out of the run fingerprint, baseline evidence and score-history scope.
|
|
23
28
|
OUTPUT_KEYS = %w[cache_dir json_report html_report score_history resume].freeze
|
|
@@ -145,9 +150,21 @@ module Saboteur
|
|
|
145
150
|
"saboteur no longer reads #{LEGACY_IGNORE_FILE}; move its entries to #{IGNORE_FILE} (see README)."
|
|
146
151
|
end
|
|
147
152
|
|
|
148
|
-
|
|
149
|
-
|
|
153
|
+
file_present = File.file?(path)
|
|
154
|
+
from_file = file_present ? ConfigFile.load(path) : {}
|
|
155
|
+
# Only a missing file triggers detection: a written config, even an
|
|
156
|
+
# empty one, means the defaults were chosen. The detected include is
|
|
157
|
+
# not an explicit key - the user set nothing.
|
|
158
|
+
detected = !file_present && rails_app?(root) ? { "include" => RAILS_INCLUDE } : {}
|
|
159
|
+
cli = stringify(overrides)
|
|
160
|
+
merged = detected.merge(from_file).merge(cli)
|
|
161
|
+
new(merged, root: root, explicit: from_file.keys | cli.keys, config_file_present: file_present)
|
|
162
|
+
end
|
|
163
|
+
|
|
164
|
+
def self.rails_app?(root)
|
|
165
|
+
File.file?(File.join(root, "config", "application.rb")) && File.directory?(File.join(root, "app"))
|
|
150
166
|
end
|
|
167
|
+
private_class_method :rails_app?
|
|
151
168
|
|
|
152
169
|
def self.stringify(hash)
|
|
153
170
|
hash.to_h { |key, value| [key.to_s, value] }.compact
|
|
@@ -156,11 +173,16 @@ module Saboteur
|
|
|
156
173
|
# Deprecations found while loading: printed by the CLI, never fatal.
|
|
157
174
|
attr_reader :deprecation_warnings
|
|
158
175
|
|
|
159
|
-
|
|
176
|
+
# Whether the config file existed at load (#1172): the CLI says once
|
|
177
|
+
# when a run falls back to defaults because no file was there.
|
|
178
|
+
attr_reader :config_file_present
|
|
179
|
+
|
|
180
|
+
def initialize(values = {}, root: Dir.pwd, explicit: nil, config_file_present: true)
|
|
160
181
|
# Retag the root losslessly: under a non-UTF-8 locale it arrives
|
|
161
182
|
# locale-tagged, and every glob, join and expand below would mix
|
|
162
183
|
# encodings with the UTF-8 patterns from the config file (#231).
|
|
163
184
|
@root = Text.utf8_identity(File.expand_path(root))
|
|
185
|
+
@config_file_present = config_file_present
|
|
164
186
|
unknown = values.keys - DEFAULTS.keys
|
|
165
187
|
raise ConfigError, "unknown config key(s): #{unknown.join(', ')}" if unknown.any?
|
|
166
188
|
|
data/lib/saboteur/doctor.rb
CHANGED
|
@@ -76,7 +76,14 @@ module Saboteur
|
|
|
76
76
|
# post-boot check re-reads the state without printing a second line.
|
|
77
77
|
def check_coverage
|
|
78
78
|
state = Coverage.state
|
|
79
|
-
|
|
79
|
+
# "suspended" alone reads as a finding; it is the expected boot state
|
|
80
|
+
# - coverage is stopped while the app boots and starts for the map
|
|
81
|
+
# build (#1176). Any other state stays terse: verify_coverage raises.
|
|
82
|
+
if state == :suspended
|
|
83
|
+
@out.puts "Coverage: suspended during boot (saboteur starts it for the coverage map)"
|
|
84
|
+
else
|
|
85
|
+
@out.puts "Coverage: #{state}"
|
|
86
|
+
end
|
|
80
87
|
verify_coverage(state)
|
|
81
88
|
end
|
|
82
89
|
|
|
@@ -16,6 +16,9 @@ module Saboteur
|
|
|
16
16
|
include CoverageParts
|
|
17
17
|
|
|
18
18
|
CACHE_INPUTS = %w[integration require rspec_args rails tests coverage_recorder].freeze
|
|
19
|
+
# Names the freshest whole-suite map build; read as an untrusted hint:
|
|
20
|
+
# only a bare 64-hex key joins the cache dir (#427).
|
|
21
|
+
WHOLE_SUITE_POINTER = "whole-suite-map"
|
|
19
22
|
LOCKFILES = %w[Gemfile.lock gems.locked].freeze
|
|
20
23
|
LOG = "coverage-build.log"
|
|
21
24
|
# RSpec rewrites this on every run (example_status_persistence_file_path),
|
|
@@ -31,7 +34,7 @@ module Saboteur
|
|
|
31
34
|
TAIL_LINE_WIDTH = 300
|
|
32
35
|
|
|
33
36
|
def initialize(config:, integration:, hooks:, supervisor:, diagnostics:, load_lines: {}, load_methods: {},
|
|
34
|
-
timeout_cap: nil, parts: 1)
|
|
37
|
+
timeout_cap: nil, parts: 1, whole_suite_run: false)
|
|
35
38
|
@config = config
|
|
36
39
|
@parts = parts
|
|
37
40
|
@integration = integration
|
|
@@ -41,6 +44,7 @@ module Saboteur
|
|
|
41
44
|
@timeout_cap = timeout_cap
|
|
42
45
|
@load_lines = load_lines
|
|
43
46
|
@load_methods = load_methods
|
|
47
|
+
@whole_suite_run = whole_suite_run
|
|
44
48
|
@root = File.realpath(config.root)
|
|
45
49
|
@root_prefixes = Selection::CoverageMap.root_prefixes(config.root)
|
|
46
50
|
end
|
|
@@ -58,14 +62,14 @@ module Saboteur
|
|
|
58
62
|
path = cache_path(files)
|
|
59
63
|
preloaded = LoadCoverage.preloaded_uninstrumented(files)
|
|
60
64
|
cached = cached_map(files)
|
|
61
|
-
return
|
|
65
|
+
return serve_cached(path, cached, files, preloaded) if cached
|
|
62
66
|
|
|
63
67
|
note = miss_note(files)
|
|
64
68
|
@diagnostics.inform(note.message(scoped: !@config["tests"].empty?)) if @config["cache"]
|
|
65
69
|
started = Process.clock_gettime(Process::CLOCK_MONOTONIC)
|
|
66
70
|
map = fresh_map(files, note, preloaded)
|
|
67
71
|
build_seconds = Process.clock_gettime(Process::CLOCK_MONOTONIC) - started
|
|
68
|
-
|
|
72
|
+
persist_map(path, map)
|
|
69
73
|
shadow(files, map, note, build_seconds) if shadow?
|
|
70
74
|
note.save
|
|
71
75
|
File.write(File.join(@config.path("cache_dir"), "preview.json"),
|
|
@@ -73,6 +77,31 @@ module Saboteur
|
|
|
73
77
|
map
|
|
74
78
|
end
|
|
75
79
|
|
|
80
|
+
# The map a whole-suite coverage build wrote on this tree, or nil.
|
|
81
|
+
# Survivor presentation consults it (presentation only, never
|
|
82
|
+
# selection): in a scoped run the run's own map holds only the specs
|
|
83
|
+
# the run loaded, so a "covered by" list from it can read as complete
|
|
84
|
+
# when another spec pins the behavior (false survivors by default,
|
|
85
|
+
# review C1). A full run keys its map with tests: [], so the lookup
|
|
86
|
+
# overrides a scoped run's --test value.
|
|
87
|
+
# The map this store's fetch produced (or served from cache): the
|
|
88
|
+
# run's own selection map, read by survivor presentation (review C1).
|
|
89
|
+
attr_reader :fetched_map
|
|
90
|
+
|
|
91
|
+
def whole_suite_map
|
|
92
|
+
return nil unless @config["cache"]
|
|
93
|
+
|
|
94
|
+
pointer = File.join(@config.path("cache_dir"), WHOLE_SUITE_POINTER)
|
|
95
|
+
return nil unless File.file?(pointer)
|
|
96
|
+
|
|
97
|
+
key = File.read(pointer, encoding: Encoding::UTF_8).strip
|
|
98
|
+
return nil unless key.match?(/\A[0-9a-f]{64}\z/)
|
|
99
|
+
|
|
100
|
+
Selection::CoverageMap.load(File.join(@config.path("cache_dir"), "coverage-#{key}.json"))
|
|
101
|
+
rescue SystemCallError, EncodingError
|
|
102
|
+
nil
|
|
103
|
+
end
|
|
104
|
+
|
|
76
105
|
private
|
|
77
106
|
|
|
78
107
|
# The partial map when it is on and safe, else the full build.
|
|
@@ -106,6 +135,28 @@ module Saboteur
|
|
|
106
135
|
@cached_map ||= Selection::CoverageMap.load(cache_path(files))
|
|
107
136
|
end
|
|
108
137
|
|
|
138
|
+
def serve_cached(path, cached, files, preloaded)
|
|
139
|
+
record_whole_suite_map(path)
|
|
140
|
+
refuse_empty_map(@fetched_map = cached.with_preloaded(preloaded), files, preloaded)
|
|
141
|
+
end
|
|
142
|
+
|
|
143
|
+
def persist_map(path, map)
|
|
144
|
+
map.save(path)
|
|
145
|
+
record_whole_suite_map(path)
|
|
146
|
+
@fetched_map = map
|
|
147
|
+
end
|
|
148
|
+
|
|
149
|
+
# The engine tells us when this run is a whole-suite build (no
|
|
150
|
+
# --since, no subjects, no explicit --test): remember the map key for
|
|
151
|
+
# later scoped runs, whose tree has usually drifted past the exact
|
|
152
|
+
# cache key already (review C1).
|
|
153
|
+
def record_whole_suite_map(path)
|
|
154
|
+
return unless @whole_suite_run
|
|
155
|
+
|
|
156
|
+
File.write(File.join(@config.path("cache_dir"), WHOLE_SUITE_POINTER),
|
|
157
|
+
File.basename(path, ".json").delete_prefix("coverage-"), perm: 0o600)
|
|
158
|
+
end
|
|
159
|
+
|
|
109
160
|
def cache_key(files)
|
|
110
161
|
Selection::CoverageMap.cache_key(files: key_files(files), extra: key_inputs)
|
|
111
162
|
end
|
|
@@ -77,9 +77,10 @@ module Saboteur
|
|
|
77
77
|
# progress (see start_progress); the letter is legend-only.
|
|
78
78
|
LEGEND = ".:killed S:survived T:timeout E:error s:skipped L:resource limit"
|
|
79
79
|
|
|
80
|
-
# Chunk sizing: leave room for the "
|
|
81
|
-
# below the floor
|
|
82
|
-
|
|
80
|
+
# Chunk sizing: leave room for the full " n/total (100%) ETA 123m45s"
|
|
81
|
+
# summary suffix (32 columns), never below the floor, so a piped line
|
|
82
|
+
# fits an 80-column CI log instead of wrapping mid-summary.
|
|
83
|
+
CHUNK_MARGIN = 32
|
|
83
84
|
CHUNK_FLOOR = 40
|
|
84
85
|
DEFAULT_WIDTH = 80
|
|
85
86
|
|
|
@@ -110,7 +110,11 @@ module Saboteur
|
|
|
110
110
|
# scheduler stopped mid-list at the budget deadline. Worker-level error
|
|
111
111
|
# rows synthesize an error per candidate, so they never look truncated.
|
|
112
112
|
def unverified_result?(result)
|
|
113
|
-
result.is_a?(MutantResult) && result.status == :unverified
|
|
113
|
+
result.is_a?(MutantResult) && result.status == :unverified && !result.flaky?
|
|
114
|
+
end
|
|
115
|
+
|
|
116
|
+
def budget_unverified_row?(result)
|
|
117
|
+
result["status"] == "unverified" && !MutantResult.flaky_reason?(result["reason"])
|
|
114
118
|
end
|
|
115
119
|
|
|
116
120
|
def record_unreached(rows, subjects, lists)
|
|
@@ -125,7 +129,7 @@ module Saboteur
|
|
|
125
129
|
next unless row["results"] && !row["error"]
|
|
126
130
|
|
|
127
131
|
@truncated << [subjects[i], lists[i].size - row["results"].size] if row["results"].size < lists[i].size
|
|
128
|
-
@unverified << subjects[i] if row["results"].any? { |result| result
|
|
132
|
+
@unverified << subjects[i] if row["results"].any? { |result| budget_unverified_row?(result) }
|
|
129
133
|
end
|
|
130
134
|
end
|
|
131
135
|
|
|
@@ -24,7 +24,9 @@ module Saboteur
|
|
|
24
24
|
mutate(subjects, files, integration, scope, ignore_list,
|
|
25
25
|
load_lines:, load_methods:)
|
|
26
26
|
end
|
|
27
|
-
|
|
27
|
+
warnings = ignore_list.apply(results, subjects: subjects,
|
|
28
|
+
unscoped_digests: @unscoped_digests, full_run: unscoped_full_run?)
|
|
29
|
+
@diagnostics.add_warnings(warnings)
|
|
28
30
|
build_report(results, subjects, scope, started)
|
|
29
31
|
ensure
|
|
30
32
|
# Cleanup runs however the call ends - a partway boot failure, no
|
|
@@ -54,7 +56,9 @@ module Saboteur
|
|
|
54
56
|
files = config.included_files
|
|
55
57
|
subjects, scope = select_subjects(files)
|
|
56
58
|
results = subjects.empty? ? [] : mutate_external(subjects, scope, ignore_list)
|
|
57
|
-
|
|
59
|
+
warnings = ignore_list.apply(results, subjects: subjects,
|
|
60
|
+
unscoped_digests: @unscoped_digests, full_run: unscoped_full_run?)
|
|
61
|
+
@diagnostics.add_warnings(warnings)
|
|
58
62
|
build_report(results, subjects, scope, started)
|
|
59
63
|
end
|
|
60
64
|
|
|
@@ -97,6 +101,14 @@ module Saboteur
|
|
|
97
101
|
@diagnostics.warn(warning) if warning
|
|
98
102
|
end
|
|
99
103
|
|
|
104
|
+
# "Full run" for ignore staleness (#1206): no --since diff, no --subject
|
|
105
|
+
# filter, and no narrowed include (explicit file arguments replace the
|
|
106
|
+
# configured include) - only then was every in-scope subject examined, so
|
|
107
|
+
# an entry whose subject never appears really points at gone code.
|
|
108
|
+
def unscoped_full_run?
|
|
109
|
+
!config["since"] && config["subjects"].empty? && config["include"] == Config::DEFAULTS["include"]
|
|
110
|
+
end
|
|
111
|
+
|
|
100
112
|
def external_command? = !config["test_command"].to_s.empty?
|
|
101
113
|
|
|
102
114
|
# The preflight's absolute monotonic deadline: its own
|
|
@@ -352,7 +352,7 @@ module Saboteur
|
|
|
352
352
|
def flaky_message(outcome, rerun)
|
|
353
353
|
decisive = (outcome.failed_ids | rerun.failed_ids).first(3)
|
|
354
354
|
named = decisive.empty? ? "" : " (decisive tests: #{decisive.join(', ')})"
|
|
355
|
-
"
|
|
355
|
+
"#{MutantResult::FLAKY_PREFIX}: first #{outcome.status}, then #{rerun.status} on a reordered confirmation " \
|
|
356
356
|
"run#{named} - the mutant is unverified, not #{outcome.status}"
|
|
357
357
|
end
|
|
358
358
|
|
|
@@ -11,12 +11,13 @@ module Saboteur
|
|
|
11
11
|
private
|
|
12
12
|
|
|
13
13
|
# A --since diff with no changed source file (a spec-only or docs-only
|
|
14
|
-
# change) yields zero subjects, so the suite never runs: say
|
|
15
|
-
#
|
|
16
|
-
# happen (#607).
|
|
14
|
+
# change) yields zero subjects, so the suite never runs: say why no
|
|
15
|
+
# spec was inferred rather than promise a whole-suite run that does not
|
|
16
|
+
# happen (#607). The conclusion itself is stated once, by the
|
|
17
|
+
# "--since <ref>: nothing to test - <reason>" outcome line (#1232).
|
|
17
18
|
def selection_message(guess)
|
|
18
19
|
if guess.changed.zero? && config["since"]
|
|
19
|
-
"inferred specs: no changed source file to infer a spec for
|
|
20
|
+
"inferred specs: no changed source file to infer a spec for"
|
|
20
21
|
elsif guess.changed.zero?
|
|
21
22
|
"inferred specs: no source file to infer a spec for, so the whole suite runs"
|
|
22
23
|
elsif guess.specs.empty?
|