saboteur 0.11.0 → 0.11.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +51 -0
- data/README.md +1 -1
- data/lib/saboteur/baseline/command.rb +46 -2
- data/lib/saboteur/baseline/comparison.rb +6 -1
- data/lib/saboteur/baseline/completeness.rb +21 -0
- data/lib/saboteur/baseline/evidence.rb +11 -3
- data/lib/saboteur/baseline/loader.rb +18 -5
- data/lib/saboteur/baseline/manifest.rb +13 -4
- data/lib/saboteur/baseline/run_evidence.rb +3 -1
- data/lib/saboteur/baseline/workflow.rb +57 -9
- data/lib/saboteur/baseline_health.rb +4 -1
- data/lib/saboteur/cli/dirty_tree_notice.rb +7 -1
- data/lib/saboteur/cli/doctor_command.rb +10 -1
- data/lib/saboteur/cli/exit_status.rb +4 -3
- data/lib/saboteur/cli/explicit_sources.rb +1 -1
- data/lib/saboteur/cli/failure_output.rb +2 -2
- data/lib/saboteur/cli/presets.rb +11 -1
- data/lib/saboteur/cli/run_command.rb +4 -1
- data/lib/saboteur/cli/survivors_command.rb +81 -17
- data/lib/saboteur/crash_report.rb +3 -2
- data/lib/saboteur/doctor.rb +3 -1
- data/lib/saboteur/engine/boot.rb +4 -2
- data/lib/saboteur/engine/result_cache.rb +4 -2
- data/lib/saboteur/engine/scheduler.rb +22 -20
- data/lib/saboteur/engine/subject_selection.rb +1 -1
- data/lib/saboteur/engine/survivor_confirmation.rb +1 -1
- data/lib/saboteur/engine/test_guess.rb +1 -1
- data/lib/saboteur/engine/thread_warnings.rb +9 -4
- data/lib/saboteur/engine.rb +22 -4
- data/lib/saboteur/execution/fiddle_loader.rb +29 -2
- data/lib/saboteur/help.rb +2 -1
- data/lib/saboteur/ignore_list.rb +15 -2
- data/lib/saboteur/install.rb +18 -3
- data/lib/saboteur/preview.rb +6 -1
- data/lib/saboteur/reporter/agent_commands.rb +6 -1
- data/lib/saboteur/reporter/baseline_tail.rb +9 -7
- data/lib/saboteur/reporter/html.rb +141 -6
- data/lib/saboteur/reporter/next_steps.rb +9 -7
- data/lib/saboteur/reporter/notice.rb +24 -0
- data/lib/saboteur/reporter/reuse_note.rb +37 -0
- data/lib/saboteur/reporter/review_comments.rb +11 -7
- data/lib/saboteur/reporter/stryker.rb +11 -1
- data/lib/saboteur/reporter/survivor_list.rb +6 -2
- data/lib/saboteur/reporter/survivor_ranking.rb +13 -5
- data/lib/saboteur/reporter/terminal.rb +5 -4
- data/lib/saboteur/result.rb +29 -8
- data/lib/saboteur/selection/git_diff.rb +19 -1
- data/lib/saboteur/survivor_guidance.rb +33 -0
- data/lib/saboteur/version.rb +1 -1
- metadata +5 -2
checksums.yaml
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
---
|
|
2
2
|
SHA256:
|
|
3
|
-
metadata.gz:
|
|
4
|
-
data.tar.gz:
|
|
3
|
+
metadata.gz: 74914fa96e0e644c7615013b6ad8a014c3c74b01e9b154931f2087fa0685dd31
|
|
4
|
+
data.tar.gz: 3265c82c7383a4ec2ce6355c009fc5816822e96334224e05a3cf6dac05404504
|
|
5
5
|
SHA512:
|
|
6
|
-
metadata.gz:
|
|
7
|
-
data.tar.gz:
|
|
6
|
+
metadata.gz: de0a4aa5db3d0403a46d9db2390fca263b9d7b433a9bbdb5d28049402c07d38ac75f2f68221bc4576ca3bc1a27c90c42aa7c62ba9f19d67eb6183507cfeef43b
|
|
7
|
+
data.tar.gz: 9d0286667bae0a37141efacd0547cb62d7185567ca8728f39be84902905f149f7d08f8eff2df07b2601083a8118a539d9d34fed06cfd47ec3840490c8ceb65a6
|
data/CHANGELOG.md
CHANGED
|
@@ -5,6 +5,57 @@ All notable changes are documented here. The format follows
|
|
|
5
5
|
[Semantic Versioning](https://semver.org/).
|
|
6
6
|
|
|
7
7
|
## [Unreleased]
|
|
8
|
+
## [0.11.1] - 2026-10-22
|
|
9
|
+
|
|
10
|
+
### Added
|
|
11
|
+
|
|
12
|
+
- Diff runs echo the real `--since` ref in hints and name the diff
|
|
13
|
+
scope in the reports (#1235, #1218, #1219), and `survivors` reads the
|
|
14
|
+
last report by default and refuses ambiguous digest prefixes (#1235,
|
|
15
|
+
#1189, #1188, #1192).
|
|
16
|
+
- `baseline propose` tolerates untested subjects: proposability is the
|
|
17
|
+
gate at propose time, while strict evidence completeness stays where
|
|
18
|
+
it is required (#1168, #1141). The baseline command names what it is
|
|
19
|
+
doing, the file it writes and the follow-up command at every step,
|
|
20
|
+
and blockers resolve to subject, operator and reason plus a
|
|
21
|
+
remediation line (#1217, #1195, #1196).
|
|
22
|
+
- The GitHub Action gains an opt-in `max-duration` input passed to
|
|
23
|
+
`--max-duration`, summary lines for exits 2/3/4/5/70, an annotation
|
|
24
|
+
on exit 4, and failure messages that each name the command or
|
|
25
|
+
setting that fixes them (#1220).
|
|
26
|
+
- Incremental runs report reuse feedback (#1221), and the HTML report
|
|
27
|
+
gains a copy button on survivor blocks, print rules and aria fixes
|
|
28
|
+
(#1231).
|
|
29
|
+
|
|
30
|
+
### Changed
|
|
31
|
+
|
|
32
|
+
- Warning notices, the baseline tail and the CLI rescue paths render
|
|
33
|
+
through the colour palette (monochrome output is byte-identical), and
|
|
34
|
+
first runs are quieter: the Ruby 3.4 fiddle stdlib notice is dropped
|
|
35
|
+
during the guarded require, Rails mode no longer warns about the
|
|
36
|
+
harmless AR Pool Reaper thread, and the no-spec stop ends with a
|
|
37
|
+
clearer custom-map pointer (#1236, #1226, #1229, #1230).
|
|
38
|
+
- Survivors rank by per-statement density first (the statement with
|
|
39
|
+
the most surviving mutants leads), then by thin coverage; ranking is
|
|
40
|
+
review priority only, with no score or exit-code effect (#1163).
|
|
41
|
+
- `survivors --show` prints the complete detail - diff, replay command
|
|
42
|
+
with the run's seed, and a paste-correct ignore template (#1194,
|
|
43
|
+
#1217) - and unknown-command and doctor include errors are clearer
|
|
44
|
+
(#1194).
|
|
45
|
+
- First-run DX: the footer ignore template pastes into `.saboteur.rb`
|
|
46
|
+
unchanged, install prints the next step, and lifecycle warnings are
|
|
47
|
+
quieter (#1217, #1221). The survivor-confirmation skip notice names
|
|
48
|
+
the real reason (#1231, #1184, #1177).
|
|
49
|
+
- The HTML report is printable and collapsible, with clearer baseline
|
|
50
|
+
errors (#1179).
|
|
51
|
+
|
|
52
|
+
### Fixed
|
|
53
|
+
|
|
54
|
+
- Survivor confirmation no longer crashes the run (exit 70): a
|
|
55
|
+
confirmation failure degrades to a warning (#1164, #1140).
|
|
56
|
+
- README hero output shows the right subject count (#1162), and the
|
|
57
|
+
exit-code docs match the gates (#1231, #1222).
|
|
58
|
+
|
|
8
59
|
|
|
9
60
|
## [0.11.0] - 2026-10-21
|
|
10
61
|
|
data/README.md
CHANGED
|
@@ -17,7 +17,7 @@
|
|
|
17
17
|
|
|
18
18
|
```text
|
|
19
19
|
$ bundle exec saboteur pr
|
|
20
|
-
saboteur 0.11.0:
|
|
20
|
+
saboteur 0.11.0: 2 subjects, 10 mutants, wall 0.70 s, seed 1 (repeat with --seed 1)
|
|
21
21
|
Score: 80.0% (8/10 killed, tested subjects)
|
|
22
22
|
saboteur: 8 killed, 2 survived in 1s - score 80.0%
|
|
23
23
|
```
|
|
@@ -20,6 +20,7 @@ module Saboteur
|
|
|
20
20
|
opts.on("--config PATH", "Config file (default config/saboteur.rb)") { |value| set(:config, value) }
|
|
21
21
|
opts.on("--subject EXPR", "Whole subject filter; repeatable") { |value| add_subject(value) }
|
|
22
22
|
opts.on("--seed N", Integer, "Repeatable run seed") { |value| set(:seed, value) }
|
|
23
|
+
opts.on("--format FORMAT", "propose: terminal (default) or json") { |value| @format = value }
|
|
23
24
|
end
|
|
24
25
|
end
|
|
25
26
|
|
|
@@ -34,7 +35,7 @@ module Saboteur
|
|
|
34
35
|
workflow = Baseline::Workflow.new(config: config, err: @err)
|
|
35
36
|
baseline_action(workflow, action)
|
|
36
37
|
end
|
|
37
|
-
@out.puts(JSON.generate(payload))
|
|
38
|
+
action == "propose" && @format != "json" ? print_proposal(payload) : @out.puts(JSON.generate(payload))
|
|
38
39
|
payload.fetch("exit_code", CLI::EXIT_OK)
|
|
39
40
|
rescue ConfigError, SystemCallError => e
|
|
40
41
|
@err.puts("saboteur: #{e.message}")
|
|
@@ -44,13 +45,17 @@ module Saboteur
|
|
|
44
45
|
CLI::EXIT_TOOL
|
|
45
46
|
end
|
|
46
47
|
|
|
48
|
+
def baseline_path_required(action)
|
|
49
|
+
"--baseline PATH is required (for example: saboteur baseline #{action} --baseline baseline.json)"
|
|
50
|
+
end
|
|
51
|
+
|
|
47
52
|
def parse_baseline(action)
|
|
48
53
|
raise CLI::UsageError, "baseline expects check, propose or accept" unless %w[check propose
|
|
49
54
|
accept].include?(action)
|
|
50
55
|
|
|
51
56
|
baseline_parser.parse!(@argv)
|
|
52
57
|
raise CLI::UsageError, "unexpected arguments: #{@argv.join(' ')}" if @argv.any?
|
|
53
|
-
raise CLI::UsageError,
|
|
58
|
+
raise CLI::UsageError, baseline_path_required(action) unless @baseline_path
|
|
54
59
|
raise CLI::UsageError, "--proposal PATH is required" if action != "check" && !@proposal_path
|
|
55
60
|
if action == "check" && (@proposal_path || @previous)
|
|
56
61
|
raise CLI::UsageError,
|
|
@@ -58,6 +63,15 @@ module Saboteur
|
|
|
58
63
|
end
|
|
59
64
|
validate_epsilon(action)
|
|
60
65
|
raise CLI::UsageError, "--previous needs a SHA-256" if @previous && !@previous.match?(/\A[0-9a-f]{64}\z/)
|
|
66
|
+
|
|
67
|
+
validate_format(action)
|
|
68
|
+
end
|
|
69
|
+
|
|
70
|
+
def validate_format(action)
|
|
71
|
+
unless @format.nil? || %w[terminal json].include?(@format)
|
|
72
|
+
raise CLI::UsageError, "--format must be terminal or json"
|
|
73
|
+
end
|
|
74
|
+
raise CLI::UsageError, "--format applies to propose only" if @format && action != "propose"
|
|
61
75
|
end
|
|
62
76
|
|
|
63
77
|
def validate_epsilon(action)
|
|
@@ -67,6 +81,36 @@ module Saboteur
|
|
|
67
81
|
raise CLI::UsageError, "--epsilon must be between 0 and 100" unless (0.0..100.0).cover?(@epsilon)
|
|
68
82
|
end
|
|
69
83
|
|
|
84
|
+
def print_proposal(payload)
|
|
85
|
+
@out.puts("baseline proposal written: #{payload.fetch('proposal')}")
|
|
86
|
+
survivors = payload.fetch("survivors")
|
|
87
|
+
@out.puts(" matched mutants: #{payload.fetch('candidates')} " \
|
|
88
|
+
"(#{plural(survivors, 'survivor')} to reason about)")
|
|
89
|
+
gaps = payload.fetch("gaps", [])
|
|
90
|
+
unless gaps.empty?
|
|
91
|
+
names = gaps.map { |gap| gap.sub(/\A(subject_skipped|omitted):/, "") }
|
|
92
|
+
@out.puts(" proposed with #{plural(names.size, 'untested subject')}: #{names.join(', ')} - " \
|
|
93
|
+
"the baseline cannot speak for them")
|
|
94
|
+
end
|
|
95
|
+
@out.puts(" next step: give each accepted survivor a reason in the proposal, then run: " \
|
|
96
|
+
"saboteur baseline accept#{scope_flags} " \
|
|
97
|
+
"--baseline #{payload.fetch('destination')} --proposal #{payload.fetch('proposal')}")
|
|
98
|
+
end
|
|
99
|
+
|
|
100
|
+
def plural(count, noun)
|
|
101
|
+
count == 1 ? "#{count} #{noun}" : "#{count} #{noun}s"
|
|
102
|
+
end
|
|
103
|
+
|
|
104
|
+
# accept re-verifies the recorded run settings, so the printed command
|
|
105
|
+
# repeats the scope flags this propose ran with.
|
|
106
|
+
def scope_flags
|
|
107
|
+
flags = +""
|
|
108
|
+
Array(options[:subjects]).each { |expr| flags << " --subject #{expr}" }
|
|
109
|
+
flags << " --seed #{options[:seed]}" if options[:seed]
|
|
110
|
+
flags << " --config #{options[:config]}" if options[:config]
|
|
111
|
+
flags
|
|
112
|
+
end
|
|
113
|
+
|
|
70
114
|
def baseline_action(workflow, action)
|
|
71
115
|
case action
|
|
72
116
|
when "check" then workflow.check(path: @baseline_path, epsilon: @epsilon)
|
|
@@ -62,7 +62,12 @@ module Saboteur
|
|
|
62
62
|
end
|
|
63
63
|
|
|
64
64
|
def outcome(current)
|
|
65
|
-
|
|
65
|
+
# The proposability tier is the comparability gate: a run over soft
|
|
66
|
+
# gaps still speaks for the subjects it executed. Legacy documents
|
|
67
|
+
# without the split fall back to the strict flag and full issue list.
|
|
68
|
+
proposable = current.fetch("proposable") { current.fetch("complete") }
|
|
69
|
+
blockers = current.fetch("blockers") { current.fetch("issues") }
|
|
70
|
+
return "incomplete" unless proposable && blockers.empty? && !current.fetch("candidates").empty?
|
|
66
71
|
return "incomplete" unless categories.fetch("errors").empty?
|
|
67
72
|
return "review_required" if %w[new changed unmatched].any? { |name| !categories.fetch(name).empty? }
|
|
68
73
|
return "review_required" if score&.fetch("exceeded")
|
|
@@ -6,8 +6,15 @@ module Saboteur
|
|
|
6
6
|
# Exact planned/result reconciliation, independent of survivor acceptance.
|
|
7
7
|
class Completeness
|
|
8
8
|
EXECUTED = %w[killed survived timeout].freeze
|
|
9
|
+
# Soft gaps limit what the baseline can speak for; they never refuse a
|
|
10
|
+
# proposal. Everything else is a hard blocker.
|
|
11
|
+
SOFT_PREFIXES = %w[subject_skipped: omitted:].freeze
|
|
9
12
|
attr_reader :issues
|
|
10
13
|
|
|
14
|
+
def self.soft_gap?(issue)
|
|
15
|
+
SOFT_PREFIXES.any? { |prefix| issue.start_with?(prefix) }
|
|
16
|
+
end
|
|
17
|
+
|
|
11
18
|
def initialize(planned:, results:, subjects:, stable_suite:, stable_inputs:, skipped_subjects:, omitted:,
|
|
12
19
|
generation_issues:)
|
|
13
20
|
@issues = []
|
|
@@ -25,6 +32,20 @@ module Saboteur
|
|
|
25
32
|
issues.empty?
|
|
26
33
|
end
|
|
27
34
|
|
|
35
|
+
def blockers
|
|
36
|
+
issues.reject { |issue| self.class.soft_gap?(issue) }
|
|
37
|
+
end
|
|
38
|
+
|
|
39
|
+
def gaps
|
|
40
|
+
issues.select { |issue| self.class.soft_gap?(issue) }
|
|
41
|
+
end
|
|
42
|
+
|
|
43
|
+
# A proposal may be written over soft gaps (they are listed in it);
|
|
44
|
+
# only hard blockers refuse.
|
|
45
|
+
def proposable?
|
|
46
|
+
blockers.empty?
|
|
47
|
+
end
|
|
48
|
+
|
|
28
49
|
private
|
|
29
50
|
|
|
30
51
|
def validate_inventory(planned, subjects)
|
|
@@ -10,8 +10,8 @@ module Saboteur
|
|
|
10
10
|
# Validates the producer document before it can enter the acceptance path.
|
|
11
11
|
# A claimed complete flag alone is never evidence of complete execution.
|
|
12
12
|
class Evidence
|
|
13
|
-
FIELDS = %w[compatibility snapshot subjects candidates planned complete issues
|
|
14
|
-
omissions discovery_failures inline_suppressions].freeze
|
|
13
|
+
FIELDS = %w[compatibility snapshot subjects candidates planned complete proposable issues blockers gaps
|
|
14
|
+
generation_events omissions discovery_failures inline_suppressions].freeze
|
|
15
15
|
COMPATIBILITY = %w[generator runtime config scope].freeze
|
|
16
16
|
PLAN_FIELDS = %w[id identity context].freeze
|
|
17
17
|
|
|
@@ -27,6 +27,13 @@ module Saboteur
|
|
|
27
27
|
validate_compatibility
|
|
28
28
|
validate_generation
|
|
29
29
|
validate_plan
|
|
30
|
+
blockers = @document.fetch("blockers")
|
|
31
|
+
gaps = @document.fetch("gaps")
|
|
32
|
+
unless @document.fetch("proposable") == blockers.empty? &&
|
|
33
|
+
(blockers + gaps).sort == @document.fetch("issues")
|
|
34
|
+
raise ConfigError, "baseline evidence: completeness tiers disagree"
|
|
35
|
+
end
|
|
36
|
+
|
|
30
37
|
# Manifest owns exact candidate, identity, subject and acceptance shapes.
|
|
31
38
|
Manifest.new(manifest(reasons: "validation"))
|
|
32
39
|
rescue ArgumentError, KeyError, TypeError, NoMethodError => e
|
|
@@ -34,7 +41,8 @@ module Saboteur
|
|
|
34
41
|
end
|
|
35
42
|
|
|
36
43
|
def manifest(reasons: "")
|
|
37
|
-
metadata = document.fetch("compatibility").merge(document.slice("snapshot", "subjects", "complete",
|
|
44
|
+
metadata = document.fetch("compatibility").merge(document.slice("snapshot", "subjects", "complete",
|
|
45
|
+
"proposable", "issues"))
|
|
38
46
|
accepted = document.fetch("candidates").select { |entry| entry.fetch("status") == "survived" }.map do |entry|
|
|
39
47
|
entry.slice("id", "context").merge("reason" => reasons)
|
|
40
48
|
end
|
|
@@ -24,17 +24,30 @@ module Saboteur
|
|
|
24
24
|
bytes = File.read(path)
|
|
25
25
|
digest = Digest::SHA256.hexdigest(bytes)
|
|
26
26
|
manifest = Manifest.new(JSON.parse(Text.utf8_strict(bytes)))
|
|
27
|
+
Evidence.new(restored_evidence(manifest))
|
|
28
|
+
[manifest, digest]
|
|
29
|
+
rescue Errno::ENOENT
|
|
30
|
+
raise ConfigError, "baseline file #{path} does not exist; " \
|
|
31
|
+
"create it with saboteur baseline propose --baseline #{path}"
|
|
32
|
+
rescue SystemCallError, JSON::ParserError, ArgumentError => e
|
|
33
|
+
raise ConfigError, "baseline file: #{e.message}"
|
|
34
|
+
end
|
|
35
|
+
|
|
36
|
+
# The evidence snapshot a manifest stands for: its metadata, candidates
|
|
37
|
+
# and plan, with no generation events (nothing was generated).
|
|
38
|
+
def restored_evidence(manifest)
|
|
27
39
|
metadata = manifest.to_h.fetch("metadata")
|
|
28
40
|
candidates = manifest.candidates.values
|
|
29
|
-
|
|
41
|
+
issues = metadata.fetch("issues")
|
|
42
|
+
metadata.slice("snapshot", "subjects", "complete", "issues").merge(
|
|
43
|
+
"proposable" => metadata.fetch("proposable", metadata["complete"]),
|
|
44
|
+
"blockers" => issues.reject { |issue| Completeness.soft_gap?(issue) },
|
|
45
|
+
"gaps" => issues.select { |issue| Completeness.soft_gap?(issue) }
|
|
46
|
+
).merge(
|
|
30
47
|
"compatibility" => manifest.compatibility, "candidates" => candidates,
|
|
31
48
|
"planned" => candidates.map { |entry| entry.slice(*Evidence::PLAN_FIELDS) },
|
|
32
49
|
"generation_events" => [], "omissions" => [], "discovery_failures" => [], "inline_suppressions" => []
|
|
33
50
|
)
|
|
34
|
-
Evidence.new(restored)
|
|
35
|
-
[manifest, digest]
|
|
36
|
-
rescue SystemCallError, JSON::ParserError, ArgumentError => e
|
|
37
|
-
raise ConfigError, "baseline file: #{e.message}"
|
|
38
51
|
end
|
|
39
52
|
end
|
|
40
53
|
end
|
|
@@ -4,12 +4,17 @@
|
|
|
4
4
|
require "digest"
|
|
5
5
|
require "json"
|
|
6
6
|
|
|
7
|
+
require_relative "completeness"
|
|
8
|
+
|
|
7
9
|
module Saboteur
|
|
8
10
|
module Baseline
|
|
9
11
|
# Validated, immutable human acceptance and the complete run it describes.
|
|
10
12
|
class Manifest
|
|
11
13
|
FORMAT = 1
|
|
12
|
-
METADATA = %w[generator runtime config scope snapshot subjects complete issues].freeze
|
|
14
|
+
METADATA = %w[generator runtime config scope snapshot subjects complete proposable issues].freeze
|
|
15
|
+
# Baselines written before the two-tier completeness split carry no
|
|
16
|
+
# proposable key; their strict complete flag is the proposability gate.
|
|
17
|
+
LEGACY_METADATA = (METADATA - %w[proposable]).freeze
|
|
13
18
|
STATUSES = %w[killed survived timeout].freeze
|
|
14
19
|
|
|
15
20
|
attr_reader :candidates, :accepted
|
|
@@ -48,8 +53,12 @@ module Saboteur
|
|
|
48
53
|
|
|
49
54
|
def validate_metadata
|
|
50
55
|
metadata = @document["metadata"]
|
|
51
|
-
|
|
52
|
-
|
|
56
|
+
fail!("metadata fields") unless metadata.is_a?(Hash)
|
|
57
|
+
keys = metadata.keys.sort
|
|
58
|
+
fail!("metadata fields") unless [METADATA.sort, LEGACY_METADATA.sort].include?(keys)
|
|
59
|
+
proposable = metadata.fetch("proposable", metadata["complete"])
|
|
60
|
+
hard = metadata.fetch("issues").reject { |issue| Completeness.soft_gap?(issue) }
|
|
61
|
+
fail!("incomplete evidence") unless proposable == true && hard.empty?
|
|
53
62
|
%w[generator runtime config scope snapshot].each do |name|
|
|
54
63
|
fail!("metadata #{name}") unless metadata[name].is_a?(Hash) && !metadata[name].empty?
|
|
55
64
|
end
|
|
@@ -121,7 +130,7 @@ module Saboteur
|
|
|
121
130
|
current = candidates[entry["id"]]
|
|
122
131
|
fail!("acceptance candidate") unless current && current["status"] == "survived"
|
|
123
132
|
fail!("acceptance context") unless current["context"] == entry["context"]
|
|
124
|
-
text!(entry["reason"], "reason")
|
|
133
|
+
text!(entry["reason"], "reason on acceptance #{entry['id'][0, 16]} (write a non-blank reason)")
|
|
125
134
|
end
|
|
126
135
|
|
|
127
136
|
def fields!(value, fields, name)
|
|
@@ -46,7 +46,9 @@ module Saboteur
|
|
|
46
46
|
completeness = assess(observed, stable_suite, skipped_subjects, omissions, errors)
|
|
47
47
|
document = { "compatibility" => compatibility, "snapshot" => @snapshot.to_h, "subjects" => @subjects,
|
|
48
48
|
"candidates" => observed, "planned" => @planned,
|
|
49
|
-
"complete" => completeness.complete?, "
|
|
49
|
+
"complete" => completeness.complete?, "proposable" => completeness.proposable?,
|
|
50
|
+
"issues" => completeness.issues, "blockers" => completeness.blockers,
|
|
51
|
+
"gaps" => completeness.gaps,
|
|
50
52
|
"generation_events" => generation, "omissions" => omissions.map(&:to_h),
|
|
51
53
|
"discovery_failures" => discovery, "inline_suppressions" => suppressions }
|
|
52
54
|
Engine::RunFingerprint.canonical_value(document)
|
|
@@ -3,6 +3,7 @@
|
|
|
3
3
|
|
|
4
4
|
require_relative "acceptance"
|
|
5
5
|
require_relative "comparison"
|
|
6
|
+
require_relative "identity"
|
|
6
7
|
require_relative "loader"
|
|
7
8
|
require_relative "../engine"
|
|
8
9
|
require_relative "../text"
|
|
@@ -41,21 +42,48 @@ module Saboteur
|
|
|
41
42
|
payload
|
|
42
43
|
end
|
|
43
44
|
|
|
45
|
+
# Blockers carry only a digest and a status. Name the subject, operator
|
|
46
|
+
# and the engine's recorded reason, and say how to unblock the proposal.
|
|
47
|
+
def refuse_proposal(evidence, report)
|
|
48
|
+
blockers = evidence.fetch("blockers")
|
|
49
|
+
by_id = report.results.to_h do |result|
|
|
50
|
+
[Identity.new(subject: result.subject, mutation: result.mutation, root: @root).entry.fetch("id"), result]
|
|
51
|
+
end
|
|
52
|
+
blockers.each { |issue| @err.puts("saboteur: baseline proposal blocker: #{describe_blocker(issue, by_id)}") }
|
|
53
|
+
if blockers.any? { |issue| issue.start_with?("unexecuted:") }
|
|
54
|
+
@err.puts("saboteur: cover the skipped code with specs or narrow include, then propose again")
|
|
55
|
+
end
|
|
56
|
+
raise Error, "baseline proposal needs a proposable run"
|
|
57
|
+
end
|
|
58
|
+
|
|
59
|
+
def describe_blocker(issue, by_id)
|
|
60
|
+
match = /\Aunexecuted:([0-9a-f]{64}):(\w+)\z/.match(issue)
|
|
61
|
+
result = match && by_id[match[1]]
|
|
62
|
+
return issue unless result
|
|
63
|
+
|
|
64
|
+
detail = "#{result.subject.id} #{result.mutation.operator}"
|
|
65
|
+
detail += " - #{result.reason}" if result.reason
|
|
66
|
+
"#{issue} (#{detail})"
|
|
67
|
+
end
|
|
68
|
+
|
|
44
69
|
def propose(proposal:, destination:, previous: nil)
|
|
45
70
|
guard_artifacts(proposal, destination)
|
|
71
|
+
preflight_outputs(proposal, destination, previous)
|
|
46
72
|
report = Engine.new(@config, err: @err, capture_evidence: true).call
|
|
47
|
-
|
|
48
|
-
|
|
49
|
-
"baseline proposal requires complete execution evidence"
|
|
50
|
-
end
|
|
73
|
+
evidence = report.run_evidence
|
|
74
|
+
refuse_proposal(evidence, report) unless evidence["proposable"] == true
|
|
51
75
|
|
|
52
|
-
captured = Evidence.new(
|
|
76
|
+
captured = Evidence.new(evidence)
|
|
53
77
|
store(captured.document.fetch("snapshot")).propose(document: captured.manifest,
|
|
54
78
|
evidence: captured.document, proposal: proposal,
|
|
55
79
|
destination: destination, previous: previous)
|
|
56
|
-
{ "state" => "proposal", "proposal" => File.expand_path(proposal),
|
|
57
|
-
|
|
58
|
-
|
|
80
|
+
payload = { "state" => "proposal", "proposal" => File.expand_path(proposal),
|
|
81
|
+
"destination" => File.expand_path(destination),
|
|
82
|
+
"survivors" => captured.manifest.fetch("accepted").size,
|
|
83
|
+
"candidates" => captured.manifest.fetch("candidates").size }
|
|
84
|
+
gaps = evidence.fetch("gaps")
|
|
85
|
+
payload["gaps"] = gaps unless gaps.empty?
|
|
86
|
+
payload
|
|
59
87
|
end
|
|
60
88
|
|
|
61
89
|
def accept(proposal:, destination:, previous: nil)
|
|
@@ -80,12 +108,20 @@ module Saboteur
|
|
|
80
108
|
extras = saved.inventory.select { |entry| entry.kind == "extra" }.map do |entry|
|
|
81
109
|
{ path: input_path(entry.path), required: entry.required }
|
|
82
110
|
end
|
|
83
|
-
Engine::RunFingerprint.new(config: @config, extra_files: extras).verify(saved)
|
|
111
|
+
raise_unless_unchanged(Engine::RunFingerprint.new(config: @config, extra_files: extras).verify(saved))
|
|
84
112
|
end
|
|
85
113
|
Acceptance.new(directory: File.join(Identity.canonical_path(@config.path("cache_dir")), "baseline-evidence"),
|
|
86
114
|
protected_paths: protected, verify: verifier, require_evidence: true)
|
|
87
115
|
end
|
|
88
116
|
|
|
117
|
+
def raise_unless_unchanged(result)
|
|
118
|
+
return true if result.unchanged
|
|
119
|
+
|
|
120
|
+
detail = result.changes.map { |change| "#{change.path} #{change.reason}" }.join(", ")
|
|
121
|
+
detail = "verification incomplete" if detail.empty?
|
|
122
|
+
raise ConfigError, "baseline inputs changed or unverified: #{detail}"
|
|
123
|
+
end
|
|
124
|
+
|
|
89
125
|
def input_paths(snapshot)
|
|
90
126
|
snapshot.inventory.map { |entry| input_path(entry.path) }
|
|
91
127
|
end
|
|
@@ -94,6 +130,18 @@ module Saboteur
|
|
|
94
130
|
path.start_with?("external:") ? path.delete_prefix("external:") : File.join(@root, path)
|
|
95
131
|
end
|
|
96
132
|
|
|
133
|
+
# Cheap pre-checks before the full run; Acceptance re-verifies exclusively after it.
|
|
134
|
+
def preflight_outputs(proposal, destination, previous)
|
|
135
|
+
raise ConfigError, "baseline output exists: #{Identity.canonical_path(proposal)}" if File.exist?(proposal)
|
|
136
|
+
|
|
137
|
+
# Mirrors Acceptance#verify_previous so a doomed proposal fails before the run.
|
|
138
|
+
if previous.nil?
|
|
139
|
+
raise ConfigError, "baseline destination exists" if File.exist?(destination)
|
|
140
|
+
elsif !File.file?(destination) || Digest::SHA256.file(destination).hexdigest != previous
|
|
141
|
+
raise ConfigError, "baseline previous destination changed"
|
|
142
|
+
end
|
|
143
|
+
end
|
|
144
|
+
|
|
97
145
|
def guard_artifacts(*paths)
|
|
98
146
|
paths.each do |path|
|
|
99
147
|
expanded = Identity.canonical_path(path)
|
|
@@ -35,7 +35,10 @@ module Saboteur
|
|
|
35
35
|
|
|
36
36
|
def call
|
|
37
37
|
expected = @integration.all_tests.map(&:id)
|
|
38
|
-
|
|
38
|
+
if expected.empty?
|
|
39
|
+
raise Error, "baseline has no tests; check rspec_args and the suite filters, " \
|
|
40
|
+
"then run saboteur doctor --suite again"
|
|
41
|
+
end
|
|
39
42
|
|
|
40
43
|
first = check_run(1, expected)
|
|
41
44
|
second = check_run(2, expected)
|
|
@@ -1,6 +1,9 @@
|
|
|
1
1
|
# SPDX-License-Identifier: MIT
|
|
2
2
|
# frozen_string_literal: true
|
|
3
3
|
|
|
4
|
+
require_relative "../reporter/palette"
|
|
5
|
+
require_relative "../reporter/notice"
|
|
6
|
+
|
|
4
7
|
module Saboteur
|
|
5
8
|
class CLI
|
|
6
9
|
# The uncommitted-work warning printed before a run (#334). Extracted
|
|
@@ -21,7 +24,10 @@ module Saboteur
|
|
|
21
24
|
else
|
|
22
25
|
config
|
|
23
26
|
end
|
|
24
|
-
DirtyTreeWarning.for(warning, config_path: options[:config])&.then
|
|
27
|
+
DirtyTreeWarning.for(warning, config_path: options[:config])&.then do |text|
|
|
28
|
+
palette = Reporter::Palette.for(@err, mode: @color || :auto, overrides: warning["palette"])
|
|
29
|
+
@err.puts(Reporter::Notice.paint(palette, text))
|
|
30
|
+
end
|
|
25
31
|
end
|
|
26
32
|
end
|
|
27
33
|
end
|
|
@@ -22,13 +22,22 @@ module Saboteur
|
|
|
22
22
|
|
|
23
23
|
def doctor_command = help_requested? ? show_help("doctor") : doctor
|
|
24
24
|
|
|
25
|
+
# The environment is healthy only if a run would have something to
|
|
26
|
+
# mutate: the same empty source set `run` refuses (exit 2).
|
|
27
|
+
def doctor_verdict
|
|
28
|
+
return EXIT_OK unless config.included_files.empty?
|
|
29
|
+
|
|
30
|
+
@err.puts("saboteur: #{empty_source_set_message}")
|
|
31
|
+
EXIT_USAGE
|
|
32
|
+
end
|
|
33
|
+
|
|
25
34
|
def doctor
|
|
26
35
|
doctor_parser.parse!(@argv)
|
|
27
36
|
raise UsageError, "unexpected arguments: #{@argv.join(' ')}" if @argv.any?
|
|
28
37
|
|
|
29
38
|
options[:tests]&.each { |test| require_existing_test(test) }
|
|
30
39
|
Doctor.new(config, out: @out, suite: options[:suite] || options.key?(:tests)).call
|
|
31
|
-
|
|
40
|
+
doctor_verdict
|
|
32
41
|
rescue ConfigError => e
|
|
33
42
|
@err.puts("saboteur: #{Text.display_safe_lines(e.message)}")
|
|
34
43
|
EXIT_USAGE
|
|
@@ -60,9 +60,10 @@ module Saboteur
|
|
|
60
60
|
# must not exit 0 with score n/a and green-light an untested change.
|
|
61
61
|
def unverifiable_scope?(report)
|
|
62
62
|
if unverifiable_review?(report)
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
63
|
+
# A text run already printed each skip up front; json and agent
|
|
64
|
+
# runs read this sentence as the exit explanation (#1227).
|
|
65
|
+
skips = %w[json agent].include?(@format) ? report.skipped_subjects : report.unannounced_skips
|
|
66
|
+
skips.each { |subject, reason| @err.puts(subject.skipped_gate_sentence(reason, report.root)) }
|
|
66
67
|
return true
|
|
67
68
|
end
|
|
68
69
|
return false unless unverifiable_unreachable?(report)
|
|
@@ -39,7 +39,7 @@ module Saboteur
|
|
|
39
39
|
# A test may carry an example selector: spec/a_spec.rb:12 or spec/a_spec.rb[1:2].
|
|
40
40
|
def require_existing_test(test)
|
|
41
41
|
path = test.sub(/(:\d+|\[[\d:,]+\])+\z/, "")
|
|
42
|
-
raise UsageError, "test file not found: #{test}" unless File.exist?(path)
|
|
42
|
+
raise UsageError, "test file not found: #{test}; check the path or drop --test" unless File.exist?(path)
|
|
43
43
|
end
|
|
44
44
|
end
|
|
45
45
|
end
|
|
@@ -25,7 +25,7 @@ module Saboteur
|
|
|
25
25
|
end
|
|
26
26
|
|
|
27
27
|
def usage_error(message = nil)
|
|
28
|
-
@err.puts(message) if message
|
|
28
|
+
@err.puts(Reporter::Palette.for(@err, mode: @color || :auto).paint(:fail, message)) if message
|
|
29
29
|
@err.puts(USAGE)
|
|
30
30
|
agent_failure(message || USAGE, EXIT_USAGE)
|
|
31
31
|
end
|
|
@@ -43,7 +43,7 @@ module Saboteur
|
|
|
43
43
|
# name must not become a workflow command in the CI log).
|
|
44
44
|
def report_failure(error, exit_code)
|
|
45
45
|
message = Text.display_safe_lines(error.message)
|
|
46
|
-
@err.puts("saboteur: #{message}")
|
|
46
|
+
@err.puts(Reporter::Palette.for(@err, mode: @color || :auto).paint(:fail, "saboteur: #{message}"))
|
|
47
47
|
@err.puts(verdict_line(EXIT_BASELINE_RED)) if exit_code == EXIT_BASELINE_RED
|
|
48
48
|
agent_failure(message, exit_code)
|
|
49
49
|
end
|
data/lib/saboteur/cli/presets.rb
CHANGED
|
@@ -14,6 +14,11 @@ module Saboteur
|
|
|
14
14
|
--html .saboteur/audit-report.html]
|
|
15
15
|
}.freeze
|
|
16
16
|
|
|
17
|
+
# What a new user types first; there is no such command, the Rails
|
|
18
|
+
# generator writes the starter files.
|
|
19
|
+
SETUP_WORDS = %w[init install setup].freeze
|
|
20
|
+
SETUP_HINT = "To set up a Rails app, run: bin/rails generate saboteur:install"
|
|
21
|
+
|
|
17
22
|
# Preset flags that take a separate value token: the pairs a user
|
|
18
23
|
# flag can override in the banner (#666).
|
|
19
24
|
VALUE_FLAGS = %w[--baseline-passes --max-duration --since --operators --json --html].freeze
|
|
@@ -21,7 +26,12 @@ module Saboteur
|
|
|
21
26
|
private
|
|
22
27
|
|
|
23
28
|
def preset_or_usage_error(command)
|
|
24
|
-
|
|
29
|
+
return preset(command) if PRESETS.key?(command)
|
|
30
|
+
|
|
31
|
+
# A typo'd command is not a run: say so before the usage line.
|
|
32
|
+
@err.puts("unknown command: #{command}. Run 'saboteur help' for the commands.") if command
|
|
33
|
+
@err.puts(SETUP_HINT) if SETUP_WORDS.include?(command)
|
|
34
|
+
usage_error
|
|
25
35
|
end
|
|
26
36
|
|
|
27
37
|
def preset(name)
|
|
@@ -30,7 +30,10 @@ module Saboteur
|
|
|
30
30
|
validate_run_options
|
|
31
31
|
warn_dirty_tree
|
|
32
32
|
|
|
33
|
-
|
|
33
|
+
notices = Reporter::Palette.for(@err, mode: @color || :auto, overrides: config["palette"])
|
|
34
|
+
config.deprecation_warnings.each do |warning|
|
|
35
|
+
@err.puts(Reporter::Notice.paint(notices, "saboteur: #{warning}"))
|
|
36
|
+
end
|
|
34
37
|
|
|
35
38
|
return preview if options[:list_operators] || options[:dry_run]
|
|
36
39
|
|