saboteur 0.10.0 → 0.11.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +4 -4
- data/CHANGELOG.md +95 -0
- data/README.md +109 -248
- data/exe/saboteur +2 -2
- data/lib/saboteur/baseline/command.rb +46 -2
- data/lib/saboteur/baseline/comparison.rb +6 -1
- data/lib/saboteur/baseline/completeness.rb +21 -0
- data/lib/saboteur/baseline/evidence.rb +11 -3
- data/lib/saboteur/baseline/loader.rb +54 -0
- data/lib/saboteur/baseline/manifest.rb +13 -4
- data/lib/saboteur/baseline/run_evidence.rb +4 -2
- data/lib/saboteur/baseline/run_gate.rb +51 -0
- data/lib/saboteur/baseline/workflow.rb +59 -24
- data/lib/saboteur/baseline_health.rb +4 -1
- data/lib/saboteur/cli/baseline_gate.rb +58 -0
- data/lib/saboteur/cli/dirty_tree_notice.rb +7 -1
- data/lib/saboteur/cli/doctor_command.rb +10 -1
- data/lib/saboteur/cli/error_cause.rb +16 -0
- data/lib/saboteur/cli/exit_status.rb +18 -16
- data/lib/saboteur/cli/explicit_sources.rb +1 -1
- data/lib/saboteur/cli/failure_output.rb +25 -2
- data/lib/saboteur/cli/output_options.rb +22 -2
- data/lib/saboteur/cli/presets.rb +11 -1
- data/lib/saboteur/cli/run_budget_options.rb +6 -0
- data/lib/saboteur/cli/run_command.rb +18 -2
- data/lib/saboteur/cli/run_reporting.rb +33 -6
- data/lib/saboteur/cli/run_validation.rb +36 -3
- data/lib/saboteur/cli/selection_options.rb +2 -0
- data/lib/saboteur/cli/survivors_command.rb +81 -17
- data/lib/saboteur/cli.rb +6 -1
- data/lib/saboteur/config.rb +23 -2
- data/lib/saboteur/crash_report.rb +18 -8
- data/lib/saboteur/doctor.rb +3 -1
- data/lib/saboteur/engine/baseline_guard.rb +2 -2
- data/lib/saboteur/engine/boot.rb +4 -2
- data/lib/saboteur/engine/coverage_partial.rb +8 -1
- data/lib/saboteur/engine/diagnostics.rb +120 -15
- data/lib/saboteur/engine/result_cache.rb +4 -2
- data/lib/saboteur/engine/run_fingerprint.rb +1 -1
- data/lib/saboteur/engine/scheduler.rb +22 -0
- data/lib/saboteur/engine/selection_announcement.rb +8 -1
- data/lib/saboteur/engine/subject_selection.rb +1 -1
- data/lib/saboteur/engine/survivor_confirmation.rb +159 -0
- data/lib/saboteur/engine/test_guess.rb +1 -1
- data/lib/saboteur/engine/thread_warnings.rb +9 -4
- data/lib/saboteur/engine.rb +64 -9
- data/lib/saboteur/execution/fiddle_loader.rb +29 -2
- data/lib/saboteur/help.rb +6 -2
- data/lib/saboteur/ignore_list.rb +15 -2
- data/lib/saboteur/install.rb +18 -3
- data/lib/saboteur/mutation_source/builtin.rb +4 -0
- data/lib/saboteur/preview.rb +6 -1
- data/lib/saboteur/reporter/agent_commands.rb +6 -1
- data/lib/saboteur/reporter/agent_summary.rb +3 -1
- data/lib/saboteur/reporter/assets/mutation_report.LICENSE +201 -0
- data/lib/saboteur/reporter/assets/mutation_report.js +331 -0
- data/lib/saboteur/reporter/baseline_tail.rb +95 -0
- data/lib/saboteur/reporter/diff.rb +10 -0
- data/lib/saboteur/reporter/html.rb +207 -17
- data/lib/saboteur/reporter/kill_causes.rb +53 -0
- data/lib/saboteur/reporter/next_steps.rb +11 -8
- data/lib/saboteur/reporter/notice.rb +24 -0
- data/lib/saboteur/reporter/palette.rb +64 -13
- data/lib/saboteur/reporter/reuse_note.rb +37 -0
- data/lib/saboteur/reporter/review_comments.rb +11 -7
- data/lib/saboteur/reporter/safe_out.rb +1 -1
- data/lib/saboteur/reporter/score_table.rb +53 -18
- data/lib/saboteur/reporter/stryker.rb +15 -2
- data/lib/saboteur/reporter/survivor_hints.rb +70 -0
- data/lib/saboteur/reporter/survivor_list.rb +207 -99
- data/lib/saboteur/reporter/survivor_ranking.rb +13 -5
- data/lib/saboteur/reporter/terminal.rb +306 -69
- data/lib/saboteur/result.rb +56 -8
- data/lib/saboteur/score.rb +6 -0
- data/lib/saboteur/selection/git_diff.rb +19 -1
- data/lib/saboteur/survivor_guidance.rb +33 -0
- data/lib/saboteur/version.rb +1 -1
- metadata +13 -2
|
@@ -6,8 +6,15 @@ module Saboteur
|
|
|
6
6
|
# Exact planned/result reconciliation, independent of survivor acceptance.
|
|
7
7
|
class Completeness
|
|
8
8
|
EXECUTED = %w[killed survived timeout].freeze
|
|
9
|
+
# Soft gaps limit what the baseline can speak for; they never refuse a
|
|
10
|
+
# proposal. Everything else is a hard blocker.
|
|
11
|
+
SOFT_PREFIXES = %w[subject_skipped: omitted:].freeze
|
|
9
12
|
attr_reader :issues
|
|
10
13
|
|
|
14
|
+
def self.soft_gap?(issue)
|
|
15
|
+
SOFT_PREFIXES.any? { |prefix| issue.start_with?(prefix) }
|
|
16
|
+
end
|
|
17
|
+
|
|
11
18
|
def initialize(planned:, results:, subjects:, stable_suite:, stable_inputs:, skipped_subjects:, omitted:,
|
|
12
19
|
generation_issues:)
|
|
13
20
|
@issues = []
|
|
@@ -25,6 +32,20 @@ module Saboteur
|
|
|
25
32
|
issues.empty?
|
|
26
33
|
end
|
|
27
34
|
|
|
35
|
+
def blockers
|
|
36
|
+
issues.reject { |issue| self.class.soft_gap?(issue) }
|
|
37
|
+
end
|
|
38
|
+
|
|
39
|
+
def gaps
|
|
40
|
+
issues.select { |issue| self.class.soft_gap?(issue) }
|
|
41
|
+
end
|
|
42
|
+
|
|
43
|
+
# A proposal may be written over soft gaps (they are listed in it);
|
|
44
|
+
# only hard blockers refuse.
|
|
45
|
+
def proposable?
|
|
46
|
+
blockers.empty?
|
|
47
|
+
end
|
|
48
|
+
|
|
28
49
|
private
|
|
29
50
|
|
|
30
51
|
def validate_inventory(planned, subjects)
|
|
@@ -10,8 +10,8 @@ module Saboteur
|
|
|
10
10
|
# Validates the producer document before it can enter the acceptance path.
|
|
11
11
|
# A claimed complete flag alone is never evidence of complete execution.
|
|
12
12
|
class Evidence
|
|
13
|
-
FIELDS = %w[compatibility snapshot subjects candidates planned complete issues
|
|
14
|
-
omissions discovery_failures inline_suppressions].freeze
|
|
13
|
+
FIELDS = %w[compatibility snapshot subjects candidates planned complete proposable issues blockers gaps
|
|
14
|
+
generation_events omissions discovery_failures inline_suppressions].freeze
|
|
15
15
|
COMPATIBILITY = %w[generator runtime config scope].freeze
|
|
16
16
|
PLAN_FIELDS = %w[id identity context].freeze
|
|
17
17
|
|
|
@@ -27,6 +27,13 @@ module Saboteur
|
|
|
27
27
|
validate_compatibility
|
|
28
28
|
validate_generation
|
|
29
29
|
validate_plan
|
|
30
|
+
blockers = @document.fetch("blockers")
|
|
31
|
+
gaps = @document.fetch("gaps")
|
|
32
|
+
unless @document.fetch("proposable") == blockers.empty? &&
|
|
33
|
+
(blockers + gaps).sort == @document.fetch("issues")
|
|
34
|
+
raise ConfigError, "baseline evidence: completeness tiers disagree"
|
|
35
|
+
end
|
|
36
|
+
|
|
30
37
|
# Manifest owns exact candidate, identity, subject and acceptance shapes.
|
|
31
38
|
Manifest.new(manifest(reasons: "validation"))
|
|
32
39
|
rescue ArgumentError, KeyError, TypeError, NoMethodError => e
|
|
@@ -34,7 +41,8 @@ module Saboteur
|
|
|
34
41
|
end
|
|
35
42
|
|
|
36
43
|
def manifest(reasons: "")
|
|
37
|
-
metadata = document.fetch("compatibility").merge(document.slice("snapshot", "subjects", "complete",
|
|
44
|
+
metadata = document.fetch("compatibility").merge(document.slice("snapshot", "subjects", "complete",
|
|
45
|
+
"proposable", "issues"))
|
|
38
46
|
accepted = document.fetch("candidates").select { |entry| entry.fetch("status") == "survived" }.map do |entry|
|
|
39
47
|
entry.slice("id", "context").merge("reason" => reasons)
|
|
40
48
|
end
|
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
# SPDX-License-Identifier: MIT
|
|
2
|
+
# frozen_string_literal: true
|
|
3
|
+
|
|
4
|
+
require "digest"
|
|
5
|
+
require "json"
|
|
6
|
+
|
|
7
|
+
require_relative "acceptance"
|
|
8
|
+
require_relative "evidence"
|
|
9
|
+
require_relative "manifest"
|
|
10
|
+
require_relative "../text"
|
|
11
|
+
|
|
12
|
+
module Saboteur
|
|
13
|
+
module Baseline
|
|
14
|
+
# Loading an accepted baseline manifest, shared by the baseline workflow
|
|
15
|
+
# and the `run --baseline` gate: size cap, strict parse, then manifest and
|
|
16
|
+
# evidence validation. Returns [manifest, sha256 of the exact bytes], so a
|
|
17
|
+
# caller can notice the file changing while the run executes.
|
|
18
|
+
module Loader
|
|
19
|
+
module_function
|
|
20
|
+
|
|
21
|
+
def call(path)
|
|
22
|
+
raise ConfigError, "baseline file too large" if File.size(path) > Acceptance::MAX_BYTES
|
|
23
|
+
|
|
24
|
+
bytes = File.read(path)
|
|
25
|
+
digest = Digest::SHA256.hexdigest(bytes)
|
|
26
|
+
manifest = Manifest.new(JSON.parse(Text.utf8_strict(bytes)))
|
|
27
|
+
Evidence.new(restored_evidence(manifest))
|
|
28
|
+
[manifest, digest]
|
|
29
|
+
rescue Errno::ENOENT
|
|
30
|
+
raise ConfigError, "baseline file #{path} does not exist; " \
|
|
31
|
+
"create it with saboteur baseline propose --baseline #{path}"
|
|
32
|
+
rescue SystemCallError, JSON::ParserError, ArgumentError => e
|
|
33
|
+
raise ConfigError, "baseline file: #{e.message}"
|
|
34
|
+
end
|
|
35
|
+
|
|
36
|
+
# The evidence snapshot a manifest stands for: its metadata, candidates
|
|
37
|
+
# and plan, with no generation events (nothing was generated).
|
|
38
|
+
def restored_evidence(manifest)
|
|
39
|
+
metadata = manifest.to_h.fetch("metadata")
|
|
40
|
+
candidates = manifest.candidates.values
|
|
41
|
+
issues = metadata.fetch("issues")
|
|
42
|
+
metadata.slice("snapshot", "subjects", "complete", "issues").merge(
|
|
43
|
+
"proposable" => metadata.fetch("proposable", metadata["complete"]),
|
|
44
|
+
"blockers" => issues.reject { |issue| Completeness.soft_gap?(issue) },
|
|
45
|
+
"gaps" => issues.select { |issue| Completeness.soft_gap?(issue) }
|
|
46
|
+
).merge(
|
|
47
|
+
"compatibility" => manifest.compatibility, "candidates" => candidates,
|
|
48
|
+
"planned" => candidates.map { |entry| entry.slice(*Evidence::PLAN_FIELDS) },
|
|
49
|
+
"generation_events" => [], "omissions" => [], "discovery_failures" => [], "inline_suppressions" => []
|
|
50
|
+
)
|
|
51
|
+
end
|
|
52
|
+
end
|
|
53
|
+
end
|
|
54
|
+
end
|
|
@@ -4,12 +4,17 @@
|
|
|
4
4
|
require "digest"
|
|
5
5
|
require "json"
|
|
6
6
|
|
|
7
|
+
require_relative "completeness"
|
|
8
|
+
|
|
7
9
|
module Saboteur
|
|
8
10
|
module Baseline
|
|
9
11
|
# Validated, immutable human acceptance and the complete run it describes.
|
|
10
12
|
class Manifest
|
|
11
13
|
FORMAT = 1
|
|
12
|
-
METADATA = %w[generator runtime config scope snapshot subjects complete issues].freeze
|
|
14
|
+
METADATA = %w[generator runtime config scope snapshot subjects complete proposable issues].freeze
|
|
15
|
+
# Baselines written before the two-tier completeness split carry no
|
|
16
|
+
# proposable key; their strict complete flag is the proposability gate.
|
|
17
|
+
LEGACY_METADATA = (METADATA - %w[proposable]).freeze
|
|
13
18
|
STATUSES = %w[killed survived timeout].freeze
|
|
14
19
|
|
|
15
20
|
attr_reader :candidates, :accepted
|
|
@@ -48,8 +53,12 @@ module Saboteur
|
|
|
48
53
|
|
|
49
54
|
def validate_metadata
|
|
50
55
|
metadata = @document["metadata"]
|
|
51
|
-
|
|
52
|
-
|
|
56
|
+
fail!("metadata fields") unless metadata.is_a?(Hash)
|
|
57
|
+
keys = metadata.keys.sort
|
|
58
|
+
fail!("metadata fields") unless [METADATA.sort, LEGACY_METADATA.sort].include?(keys)
|
|
59
|
+
proposable = metadata.fetch("proposable", metadata["complete"])
|
|
60
|
+
hard = metadata.fetch("issues").reject { |issue| Completeness.soft_gap?(issue) }
|
|
61
|
+
fail!("incomplete evidence") unless proposable == true && hard.empty?
|
|
53
62
|
%w[generator runtime config scope snapshot].each do |name|
|
|
54
63
|
fail!("metadata #{name}") unless metadata[name].is_a?(Hash) && !metadata[name].empty?
|
|
55
64
|
end
|
|
@@ -121,7 +130,7 @@ module Saboteur
|
|
|
121
130
|
current = candidates[entry["id"]]
|
|
122
131
|
fail!("acceptance candidate") unless current && current["status"] == "survived"
|
|
123
132
|
fail!("acceptance context") unless current["context"] == entry["context"]
|
|
124
|
-
text!(entry["reason"], "reason")
|
|
133
|
+
text!(entry["reason"], "reason on acceptance #{entry['id'][0, 16]} (write a non-blank reason)")
|
|
125
134
|
end
|
|
126
135
|
|
|
127
136
|
def fields!(value, fields, name)
|
|
@@ -12,7 +12,7 @@ module Saboteur
|
|
|
12
12
|
# This does not attest external database, network or environment state.
|
|
13
13
|
class RunEvidence
|
|
14
14
|
SAFE_DROPS = %w[unchanged duplicate].freeze
|
|
15
|
-
OUTPUT_SETTINGS = Config::OUTPUT_KEYS
|
|
15
|
+
OUTPUT_SETTINGS = Config::OUTPUT_KEYS + Config::PRESENTATION_KEYS
|
|
16
16
|
|
|
17
17
|
def initialize(config:, extra_files: [])
|
|
18
18
|
@config = config
|
|
@@ -46,7 +46,9 @@ module Saboteur
|
|
|
46
46
|
completeness = assess(observed, stable_suite, skipped_subjects, omissions, errors)
|
|
47
47
|
document = { "compatibility" => compatibility, "snapshot" => @snapshot.to_h, "subjects" => @subjects,
|
|
48
48
|
"candidates" => observed, "planned" => @planned,
|
|
49
|
-
"complete" => completeness.complete?, "
|
|
49
|
+
"complete" => completeness.complete?, "proposable" => completeness.proposable?,
|
|
50
|
+
"issues" => completeness.issues, "blockers" => completeness.blockers,
|
|
51
|
+
"gaps" => completeness.gaps,
|
|
50
52
|
"generation_events" => generation, "omissions" => omissions.map(&:to_h),
|
|
51
53
|
"discovery_failures" => discovery, "inline_suppressions" => suppressions }
|
|
52
54
|
Engine::RunFingerprint.canonical_value(document)
|
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
# SPDX-License-Identifier: MIT
|
|
2
|
+
# frozen_string_literal: true
|
|
3
|
+
|
|
4
|
+
require_relative "comparison"
|
|
5
|
+
|
|
6
|
+
module Saboteur
|
|
7
|
+
module Baseline
|
|
8
|
+
# The `saboteur run --baseline PATH` verdict: after a run, its execution
|
|
9
|
+
# evidence is compared with the accepted baseline. Comparison owns the
|
|
10
|
+
# classification and the decimal score gate; this names the run-surface
|
|
11
|
+
# outcome: new survivors by digest or a score drop beyond the epsilon
|
|
12
|
+
# fail the run, while accepted legacy survivors no longer do.
|
|
13
|
+
class RunGate
|
|
14
|
+
attr_reader :comparison
|
|
15
|
+
|
|
16
|
+
def initialize(comparison)
|
|
17
|
+
@comparison = comparison
|
|
18
|
+
end
|
|
19
|
+
|
|
20
|
+
def self.build(manifest:, report:, epsilon: nil)
|
|
21
|
+
evidence = report.run_evidence
|
|
22
|
+
raise Error, "--baseline needs the run's execution evidence; the run captured none" unless evidence
|
|
23
|
+
|
|
24
|
+
new(Comparison.new(baseline: { "compatibility" => manifest.compatibility,
|
|
25
|
+
"accepted" => manifest.accepted.values,
|
|
26
|
+
"candidates" => manifest.candidates.values },
|
|
27
|
+
current: evidence, epsilon: epsilon))
|
|
28
|
+
end
|
|
29
|
+
|
|
30
|
+
def new_survivor_ids
|
|
31
|
+
comparison.categories.fetch("new").map { |entry| entry.fetch("id") }
|
|
32
|
+
end
|
|
33
|
+
|
|
34
|
+
def incompatible?
|
|
35
|
+
comparison.state == "incompatible"
|
|
36
|
+
end
|
|
37
|
+
|
|
38
|
+
def incomplete?
|
|
39
|
+
comparison.state == "incomplete"
|
|
40
|
+
end
|
|
41
|
+
|
|
42
|
+
def score_exceeded?
|
|
43
|
+
comparison.score ? comparison.score.fetch("exceeded") : false
|
|
44
|
+
end
|
|
45
|
+
|
|
46
|
+
def failed?
|
|
47
|
+
new_survivor_ids.any? || score_exceeded?
|
|
48
|
+
end
|
|
49
|
+
end
|
|
50
|
+
end
|
|
51
|
+
end
|
|
@@ -3,6 +3,8 @@
|
|
|
3
3
|
|
|
4
4
|
require_relative "acceptance"
|
|
5
5
|
require_relative "comparison"
|
|
6
|
+
require_relative "identity"
|
|
7
|
+
require_relative "loader"
|
|
6
8
|
require_relative "../engine"
|
|
7
9
|
require_relative "../text"
|
|
8
10
|
|
|
@@ -40,21 +42,48 @@ module Saboteur
|
|
|
40
42
|
payload
|
|
41
43
|
end
|
|
42
44
|
|
|
45
|
+
# Blockers carry only a digest and a status. Name the subject, operator
|
|
46
|
+
# and the engine's recorded reason, and say how to unblock the proposal.
|
|
47
|
+
def refuse_proposal(evidence, report)
|
|
48
|
+
blockers = evidence.fetch("blockers")
|
|
49
|
+
by_id = report.results.to_h do |result|
|
|
50
|
+
[Identity.new(subject: result.subject, mutation: result.mutation, root: @root).entry.fetch("id"), result]
|
|
51
|
+
end
|
|
52
|
+
blockers.each { |issue| @err.puts("saboteur: baseline proposal blocker: #{describe_blocker(issue, by_id)}") }
|
|
53
|
+
if blockers.any? { |issue| issue.start_with?("unexecuted:") }
|
|
54
|
+
@err.puts("saboteur: cover the skipped code with specs or narrow include, then propose again")
|
|
55
|
+
end
|
|
56
|
+
raise Error, "baseline proposal needs a proposable run"
|
|
57
|
+
end
|
|
58
|
+
|
|
59
|
+
def describe_blocker(issue, by_id)
|
|
60
|
+
match = /\Aunexecuted:([0-9a-f]{64}):(\w+)\z/.match(issue)
|
|
61
|
+
result = match && by_id[match[1]]
|
|
62
|
+
return issue unless result
|
|
63
|
+
|
|
64
|
+
detail = "#{result.subject.id} #{result.mutation.operator}"
|
|
65
|
+
detail += " - #{result.reason}" if result.reason
|
|
66
|
+
"#{issue} (#{detail})"
|
|
67
|
+
end
|
|
68
|
+
|
|
43
69
|
def propose(proposal:, destination:, previous: nil)
|
|
44
70
|
guard_artifacts(proposal, destination)
|
|
71
|
+
preflight_outputs(proposal, destination, previous)
|
|
45
72
|
report = Engine.new(@config, err: @err, capture_evidence: true).call
|
|
46
|
-
|
|
47
|
-
|
|
48
|
-
"baseline proposal requires complete execution evidence"
|
|
49
|
-
end
|
|
73
|
+
evidence = report.run_evidence
|
|
74
|
+
refuse_proposal(evidence, report) unless evidence["proposable"] == true
|
|
50
75
|
|
|
51
|
-
captured = Evidence.new(
|
|
76
|
+
captured = Evidence.new(evidence)
|
|
52
77
|
store(captured.document.fetch("snapshot")).propose(document: captured.manifest,
|
|
53
78
|
evidence: captured.document, proposal: proposal,
|
|
54
79
|
destination: destination, previous: previous)
|
|
55
|
-
{ "state" => "proposal", "proposal" => File.expand_path(proposal),
|
|
56
|
-
|
|
57
|
-
|
|
80
|
+
payload = { "state" => "proposal", "proposal" => File.expand_path(proposal),
|
|
81
|
+
"destination" => File.expand_path(destination),
|
|
82
|
+
"survivors" => captured.manifest.fetch("accepted").size,
|
|
83
|
+
"candidates" => captured.manifest.fetch("candidates").size }
|
|
84
|
+
gaps = evidence.fetch("gaps")
|
|
85
|
+
payload["gaps"] = gaps unless gaps.empty?
|
|
86
|
+
payload
|
|
58
87
|
end
|
|
59
88
|
|
|
60
89
|
def accept(proposal:, destination:, previous: nil)
|
|
@@ -67,22 +96,8 @@ module Saboteur
|
|
|
67
96
|
private
|
|
68
97
|
|
|
69
98
|
def load_manifest(path)
|
|
70
|
-
|
|
71
|
-
|
|
72
|
-
bytes = File.read(path)
|
|
73
|
-
@baseline_digest = Digest::SHA256.hexdigest(bytes)
|
|
74
|
-
manifest = Manifest.new(JSON.parse(Text.utf8_strict(bytes)))
|
|
75
|
-
metadata = manifest.to_h.fetch("metadata")
|
|
76
|
-
candidates = manifest.candidates.values
|
|
77
|
-
restored = metadata.slice("snapshot", "subjects", "complete", "issues").merge(
|
|
78
|
-
"compatibility" => manifest.compatibility, "candidates" => candidates,
|
|
79
|
-
"planned" => candidates.map { |entry| entry.slice(*Evidence::PLAN_FIELDS) },
|
|
80
|
-
"generation_events" => [], "omissions" => [], "discovery_failures" => [], "inline_suppressions" => []
|
|
81
|
-
)
|
|
82
|
-
Evidence.new(restored)
|
|
99
|
+
manifest, @baseline_digest = Loader.call(path)
|
|
83
100
|
manifest
|
|
84
|
-
rescue SystemCallError, JSON::ParserError, ArgumentError => e
|
|
85
|
-
raise ConfigError, "baseline file: #{e.message}"
|
|
86
101
|
end
|
|
87
102
|
|
|
88
103
|
def store(snapshot)
|
|
@@ -93,12 +108,20 @@ module Saboteur
|
|
|
93
108
|
extras = saved.inventory.select { |entry| entry.kind == "extra" }.map do |entry|
|
|
94
109
|
{ path: input_path(entry.path), required: entry.required }
|
|
95
110
|
end
|
|
96
|
-
Engine::RunFingerprint.new(config: @config, extra_files: extras).verify(saved)
|
|
111
|
+
raise_unless_unchanged(Engine::RunFingerprint.new(config: @config, extra_files: extras).verify(saved))
|
|
97
112
|
end
|
|
98
113
|
Acceptance.new(directory: File.join(Identity.canonical_path(@config.path("cache_dir")), "baseline-evidence"),
|
|
99
114
|
protected_paths: protected, verify: verifier, require_evidence: true)
|
|
100
115
|
end
|
|
101
116
|
|
|
117
|
+
def raise_unless_unchanged(result)
|
|
118
|
+
return true if result.unchanged
|
|
119
|
+
|
|
120
|
+
detail = result.changes.map { |change| "#{change.path} #{change.reason}" }.join(", ")
|
|
121
|
+
detail = "verification incomplete" if detail.empty?
|
|
122
|
+
raise ConfigError, "baseline inputs changed or unverified: #{detail}"
|
|
123
|
+
end
|
|
124
|
+
|
|
102
125
|
def input_paths(snapshot)
|
|
103
126
|
snapshot.inventory.map { |entry| input_path(entry.path) }
|
|
104
127
|
end
|
|
@@ -107,6 +130,18 @@ module Saboteur
|
|
|
107
130
|
path.start_with?("external:") ? path.delete_prefix("external:") : File.join(@root, path)
|
|
108
131
|
end
|
|
109
132
|
|
|
133
|
+
# Cheap pre-checks before the full run; Acceptance re-verifies exclusively after it.
|
|
134
|
+
def preflight_outputs(proposal, destination, previous)
|
|
135
|
+
raise ConfigError, "baseline output exists: #{Identity.canonical_path(proposal)}" if File.exist?(proposal)
|
|
136
|
+
|
|
137
|
+
# Mirrors Acceptance#verify_previous so a doomed proposal fails before the run.
|
|
138
|
+
if previous.nil?
|
|
139
|
+
raise ConfigError, "baseline destination exists" if File.exist?(destination)
|
|
140
|
+
elsif !File.file?(destination) || Digest::SHA256.file(destination).hexdigest != previous
|
|
141
|
+
raise ConfigError, "baseline previous destination changed"
|
|
142
|
+
end
|
|
143
|
+
end
|
|
144
|
+
|
|
110
145
|
def guard_artifacts(*paths)
|
|
111
146
|
paths.each do |path|
|
|
112
147
|
expanded = Identity.canonical_path(path)
|
|
@@ -35,7 +35,10 @@ module Saboteur
|
|
|
35
35
|
|
|
36
36
|
def call
|
|
37
37
|
expected = @integration.all_tests.map(&:id)
|
|
38
|
-
|
|
38
|
+
if expected.empty?
|
|
39
|
+
raise Error, "baseline has no tests; check rspec_args and the suite filters, " \
|
|
40
|
+
"then run saboteur doctor --suite again"
|
|
41
|
+
end
|
|
39
42
|
|
|
40
43
|
first = check_run(1, expected)
|
|
41
44
|
second = check_run(2, expected)
|
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
# SPDX-License-Identifier: MIT
|
|
2
|
+
# frozen_string_literal: true
|
|
3
|
+
|
|
4
|
+
require "digest"
|
|
5
|
+
|
|
6
|
+
module Saboteur
|
|
7
|
+
class CLI
|
|
8
|
+
# The `saboteur run --baseline PATH` gate: loading the accepted baseline
|
|
9
|
+
# (fail fast, before the suite boots) and mapping the comparison to the
|
|
10
|
+
# run's exit code. Mixed into the CLI, which supplies `config`.
|
|
11
|
+
module BaselineGate
|
|
12
|
+
private
|
|
13
|
+
|
|
14
|
+
# The --baseline comparison replaces the ordinary quality-gate verdict.
|
|
15
|
+
def exit_code(report) = config["baseline"] ? baseline_verdict(report) : super
|
|
16
|
+
|
|
17
|
+
# --baseline: the comparison against the accepted baseline is the
|
|
18
|
+
# verdict. Accepted legacy survivors no longer fail the run; new
|
|
19
|
+
# survivors or a score drop beyond --epsilon do (exit 1). An
|
|
20
|
+
# incompatible baseline is a usage error; an incomplete comparison
|
|
21
|
+
# means the run's own evidence cannot judge it (exit 3).
|
|
22
|
+
def baseline_verdict(report)
|
|
23
|
+
gate = baseline_gate(report)
|
|
24
|
+
raise ConfigError, gate.comparison.issues.join("; ") if gate.incompatible?
|
|
25
|
+
return EXIT_TOOL if gate.incomplete?
|
|
26
|
+
|
|
27
|
+
gate.failed? ? EXIT_SURVIVORS : EXIT_OK
|
|
28
|
+
end
|
|
29
|
+
|
|
30
|
+
def baseline_gate(report)
|
|
31
|
+
@baseline_gate ||= begin
|
|
32
|
+
path = config["baseline"]
|
|
33
|
+
if Digest::SHA256.file(path).hexdigest != @baseline_digest
|
|
34
|
+
raise ConfigError, "baseline changed during the run"
|
|
35
|
+
end
|
|
36
|
+
|
|
37
|
+
Baseline::RunGate.build(manifest: baseline_manifest, report: report, epsilon: config["epsilon"] || 0.0)
|
|
38
|
+
end
|
|
39
|
+
end
|
|
40
|
+
|
|
41
|
+
# The accepted baseline at config["baseline"], loaded once (and before
|
|
42
|
+
# the suite boots) so a missing or corrupt file fails fast as a usage
|
|
43
|
+
# error naming the fix.
|
|
44
|
+
def baseline_manifest
|
|
45
|
+
return @baseline_manifest if @baseline_manifest
|
|
46
|
+
|
|
47
|
+
path = config["baseline"]
|
|
48
|
+
raise UsageError, "--baseline file not found: #{path} - run `saboteur baseline propose` first" unless
|
|
49
|
+
File.exist?(path)
|
|
50
|
+
|
|
51
|
+
@baseline_manifest, @baseline_digest = Baseline::Loader.call(path)
|
|
52
|
+
@baseline_manifest
|
|
53
|
+
rescue ConfigError => e
|
|
54
|
+
raise UsageError, "#{e.message} - run `saboteur baseline propose` first"
|
|
55
|
+
end
|
|
56
|
+
end
|
|
57
|
+
end
|
|
58
|
+
end
|
|
@@ -1,6 +1,9 @@
|
|
|
1
1
|
# SPDX-License-Identifier: MIT
|
|
2
2
|
# frozen_string_literal: true
|
|
3
3
|
|
|
4
|
+
require_relative "../reporter/palette"
|
|
5
|
+
require_relative "../reporter/notice"
|
|
6
|
+
|
|
4
7
|
module Saboteur
|
|
5
8
|
class CLI
|
|
6
9
|
# The uncommitted-work warning printed before a run (#334). Extracted
|
|
@@ -21,7 +24,10 @@ module Saboteur
|
|
|
21
24
|
else
|
|
22
25
|
config
|
|
23
26
|
end
|
|
24
|
-
DirtyTreeWarning.for(warning, config_path: options[:config])&.then
|
|
27
|
+
DirtyTreeWarning.for(warning, config_path: options[:config])&.then do |text|
|
|
28
|
+
palette = Reporter::Palette.for(@err, mode: @color || :auto, overrides: warning["palette"])
|
|
29
|
+
@err.puts(Reporter::Notice.paint(palette, text))
|
|
30
|
+
end
|
|
25
31
|
end
|
|
26
32
|
end
|
|
27
33
|
end
|
|
@@ -22,13 +22,22 @@ module Saboteur
|
|
|
22
22
|
|
|
23
23
|
def doctor_command = help_requested? ? show_help("doctor") : doctor
|
|
24
24
|
|
|
25
|
+
# The environment is healthy only if a run would have something to
|
|
26
|
+
# mutate: the same empty source set `run` refuses (exit 2).
|
|
27
|
+
def doctor_verdict
|
|
28
|
+
return EXIT_OK unless config.included_files.empty?
|
|
29
|
+
|
|
30
|
+
@err.puts("saboteur: #{empty_source_set_message}")
|
|
31
|
+
EXIT_USAGE
|
|
32
|
+
end
|
|
33
|
+
|
|
25
34
|
def doctor
|
|
26
35
|
doctor_parser.parse!(@argv)
|
|
27
36
|
raise UsageError, "unexpected arguments: #{@argv.join(' ')}" if @argv.any?
|
|
28
37
|
|
|
29
38
|
options[:tests]&.each { |test| require_existing_test(test) }
|
|
30
39
|
Doctor.new(config, out: @out, suite: options[:suite] || options.key?(:tests)).call
|
|
31
|
-
|
|
40
|
+
doctor_verdict
|
|
32
41
|
rescue ConfigError => e
|
|
33
42
|
@err.puts("saboteur: #{Text.display_safe_lines(e.message)}")
|
|
34
43
|
EXIT_USAGE
|
|
@@ -15,6 +15,22 @@ module Saboteur
|
|
|
15
15
|
|
|
16
16
|
LIMIT = 200
|
|
17
17
|
|
|
18
|
+
# The machine cause and the one-line human reason for the two exits no
|
|
19
|
+
# report can describe, because the run never produced one. The verdict
|
|
20
|
+
# line names the reason for a nonzero exit; the report-driven codes get
|
|
21
|
+
# theirs from the terminal summary's verdict line.
|
|
22
|
+
# Keyed by exit code. Literals for EXIT_BASELINE_RED and EXIT_CRASH:
|
|
23
|
+
# this file loads before the CLI defines them; the cause pins spec
|
|
24
|
+
# asserts the keys are those two constants.
|
|
25
|
+
CAUSE_LINES = {
|
|
26
|
+
5 => ["baselineRed", "baseline suite red: fix your tests first"],
|
|
27
|
+
70 => ["crash", "internal crash (a saboteur bug, never your code)"]
|
|
28
|
+
}.freeze
|
|
29
|
+
|
|
30
|
+
def self.cause(code) = CAUSE_LINES.fetch(code).first
|
|
31
|
+
|
|
32
|
+
def self.verdict_line(code) = "exit #{code} - #{CAUSE_LINES.fetch(code).last}"
|
|
33
|
+
|
|
18
34
|
# "" when no result carries a reason.
|
|
19
35
|
def self.line(errored)
|
|
20
36
|
reasons = errored.filter_map { |result| first_line(result.reason) }.reject(&:empty?)
|
|
@@ -8,22 +8,11 @@ module Saboteur
|
|
|
8
8
|
module ExitStatus
|
|
9
9
|
private
|
|
10
10
|
|
|
11
|
-
# Last resort: a bug must not escape as a bare stack trace and an accidental exit 1.
|
|
12
|
-
def internal_error(error)
|
|
13
|
-
CrashReport.call(error, verbose: @verbose, agent: agent?, err: @err, out: @out, log_path: crash_log_path)
|
|
14
|
-
end
|
|
15
|
-
|
|
16
|
-
# The crash log lives with the other run state. A broken config (or a
|
|
17
|
-
# nil cache_dir) must not swallow the crash report itself: no log.
|
|
18
|
-
def crash_log_path
|
|
19
|
-
config.path("cache_dir")&.then { |dir| File.join(dir, CrashReport::LOG_NAME) }
|
|
20
|
-
rescue StandardError
|
|
21
|
-
nil
|
|
22
|
-
end
|
|
23
|
-
|
|
24
11
|
def exit_code(report)
|
|
25
12
|
return EXIT_BUDGET if budget_exhausted?(report)
|
|
26
13
|
|
|
14
|
+
return EXIT_OK if ignore_empty_diff?(report)
|
|
15
|
+
|
|
27
16
|
return EXIT_TOOL if untested_partial?(report)
|
|
28
17
|
|
|
29
18
|
return EXIT_TOOL if unverifiable_scope?(report)
|
|
@@ -36,6 +25,17 @@ module Saboteur
|
|
|
36
25
|
gates.failed?(report, @err, record_cause: !quality_failed) || quality_failed ? EXIT_SURVIVORS : EXIT_OK
|
|
37
26
|
end
|
|
38
27
|
|
|
28
|
+
# --ignore-empty: a --since diff that selected no mutants exits 0 with
|
|
29
|
+
# a note; suppressed, omitted or undiscovered mutants still gate.
|
|
30
|
+
def ignore_empty_diff?(report)
|
|
31
|
+
return false unless config["ignore_empty"] && config["since"] && report.results.empty?
|
|
32
|
+
return false unless report.omissions.empty? && report.inline_suppressions.empty? &&
|
|
33
|
+
report.discovery_failures.empty?
|
|
34
|
+
|
|
35
|
+
@err.puts("saboteur: no mutants in the diff - nothing to gate (--ignore-empty)")
|
|
36
|
+
true
|
|
37
|
+
end
|
|
38
|
+
|
|
39
39
|
# A spent --max-duration budget exits 4 on its own, before the gates:
|
|
40
40
|
# the run is partial by design, and --max-duration-ok opts back into
|
|
41
41
|
# the plain verdict.
|
|
@@ -60,9 +60,10 @@ module Saboteur
|
|
|
60
60
|
# must not exit 0 with score n/a and green-light an untested change.
|
|
61
61
|
def unverifiable_scope?(report)
|
|
62
62
|
if unverifiable_review?(report)
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
63
|
+
# A text run already printed each skip up front; json and agent
|
|
64
|
+
# runs read this sentence as the exit explanation (#1227).
|
|
65
|
+
skips = %w[json agent].include?(@format) ? report.skipped_subjects : report.unannounced_skips
|
|
66
|
+
skips.each { |subject, reason| @err.puts(subject.skipped_gate_sentence(reason, report.root)) }
|
|
66
67
|
return true
|
|
67
68
|
end
|
|
68
69
|
return false unless unverifiable_unreachable?(report)
|
|
@@ -112,6 +113,7 @@ module Saboteur
|
|
|
112
113
|
when EXIT_OK then "ok"
|
|
113
114
|
when EXIT_BUDGET then "budgetExhausted"
|
|
114
115
|
when EXIT_SURVIVORS then survivors_cause(report)
|
|
116
|
+
when EXIT_BASELINE_RED, EXIT_CRASH then ErrorCause.cause(code)
|
|
115
117
|
else
|
|
116
118
|
if report.majority_errored?
|
|
117
119
|
"errorMajority"
|
|
@@ -39,7 +39,7 @@ module Saboteur
|
|
|
39
39
|
# A test may carry an example selector: spec/a_spec.rb:12 or spec/a_spec.rb[1:2].
|
|
40
40
|
def require_existing_test(test)
|
|
41
41
|
path = test.sub(/(:\d+|\[[\d:,]+\])+\z/, "")
|
|
42
|
-
raise UsageError, "test file not found: #{test}" unless File.exist?(path)
|
|
42
|
+
raise UsageError, "test file not found: #{test}; check the path or drop --test" unless File.exist?(path)
|
|
43
43
|
end
|
|
44
44
|
end
|
|
45
45
|
end
|
|
@@ -1,6 +1,8 @@
|
|
|
1
1
|
# SPDX-License-Identifier: MIT
|
|
2
2
|
# frozen_string_literal: true
|
|
3
3
|
|
|
4
|
+
require_relative "../reporter/palette"
|
|
5
|
+
|
|
4
6
|
module Saboteur
|
|
5
7
|
class CLI
|
|
6
8
|
# Where a failed command speaks: stderr for people, and in agent mode the
|
|
@@ -8,8 +10,22 @@ module Saboteur
|
|
|
8
10
|
module FailureOutput
|
|
9
11
|
private
|
|
10
12
|
|
|
13
|
+
# Last resort: a bug must not escape as a bare stack trace and an accidental exit 1.
|
|
14
|
+
def internal_error(error)
|
|
15
|
+
CrashReport.call(error, verbose: @verbose, agent: agent?, err: @err, out: @out,
|
|
16
|
+
log_path: crash_log_path, color: @color || :auto)
|
|
17
|
+
end
|
|
18
|
+
|
|
19
|
+
# The crash log lives with the other run state. A broken config (or a
|
|
20
|
+
# nil cache_dir) must not swallow the crash report itself: no log.
|
|
21
|
+
def crash_log_path
|
|
22
|
+
config.path("cache_dir")&.then { |dir| File.join(dir, CrashReport::LOG_NAME) }
|
|
23
|
+
rescue StandardError
|
|
24
|
+
nil
|
|
25
|
+
end
|
|
26
|
+
|
|
11
27
|
def usage_error(message = nil)
|
|
12
|
-
@err.puts(message) if message
|
|
28
|
+
@err.puts(Reporter::Palette.for(@err, mode: @color || :auto).paint(:fail, message)) if message
|
|
13
29
|
@err.puts(USAGE)
|
|
14
30
|
agent_failure(message || USAGE, EXIT_USAGE)
|
|
15
31
|
end
|
|
@@ -27,9 +43,16 @@ module Saboteur
|
|
|
27
43
|
# name must not become a workflow command in the CI log).
|
|
28
44
|
def report_failure(error, exit_code)
|
|
29
45
|
message = Text.display_safe_lines(error.message)
|
|
30
|
-
@err.puts("saboteur: #{message}")
|
|
46
|
+
@err.puts(Reporter::Palette.for(@err, mode: @color || :auto).paint(:fail, "saboteur: #{message}"))
|
|
47
|
+
@err.puts(verdict_line(EXIT_BASELINE_RED)) if exit_code == EXIT_BASELINE_RED
|
|
31
48
|
agent_failure(message, exit_code)
|
|
32
49
|
end
|
|
50
|
+
|
|
51
|
+
# The terminal verdict names the reason for a nonzero exit. Painted
|
|
52
|
+
# red under the same colour rules as the report.
|
|
53
|
+
def verdict_line(code)
|
|
54
|
+
Reporter::Palette.for(@err, mode: @color || :auto).paint(:fail, ErrorCause.verdict_line(code))
|
|
55
|
+
end
|
|
33
56
|
end
|
|
34
57
|
end
|
|
35
58
|
end
|