saboteur 0.10.0 → 0.11.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (78) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +95 -0
  3. data/README.md +109 -248
  4. data/exe/saboteur +2 -2
  5. data/lib/saboteur/baseline/command.rb +46 -2
  6. data/lib/saboteur/baseline/comparison.rb +6 -1
  7. data/lib/saboteur/baseline/completeness.rb +21 -0
  8. data/lib/saboteur/baseline/evidence.rb +11 -3
  9. data/lib/saboteur/baseline/loader.rb +54 -0
  10. data/lib/saboteur/baseline/manifest.rb +13 -4
  11. data/lib/saboteur/baseline/run_evidence.rb +4 -2
  12. data/lib/saboteur/baseline/run_gate.rb +51 -0
  13. data/lib/saboteur/baseline/workflow.rb +59 -24
  14. data/lib/saboteur/baseline_health.rb +4 -1
  15. data/lib/saboteur/cli/baseline_gate.rb +58 -0
  16. data/lib/saboteur/cli/dirty_tree_notice.rb +7 -1
  17. data/lib/saboteur/cli/doctor_command.rb +10 -1
  18. data/lib/saboteur/cli/error_cause.rb +16 -0
  19. data/lib/saboteur/cli/exit_status.rb +18 -16
  20. data/lib/saboteur/cli/explicit_sources.rb +1 -1
  21. data/lib/saboteur/cli/failure_output.rb +25 -2
  22. data/lib/saboteur/cli/output_options.rb +22 -2
  23. data/lib/saboteur/cli/presets.rb +11 -1
  24. data/lib/saboteur/cli/run_budget_options.rb +6 -0
  25. data/lib/saboteur/cli/run_command.rb +18 -2
  26. data/lib/saboteur/cli/run_reporting.rb +33 -6
  27. data/lib/saboteur/cli/run_validation.rb +36 -3
  28. data/lib/saboteur/cli/selection_options.rb +2 -0
  29. data/lib/saboteur/cli/survivors_command.rb +81 -17
  30. data/lib/saboteur/cli.rb +6 -1
  31. data/lib/saboteur/config.rb +23 -2
  32. data/lib/saboteur/crash_report.rb +18 -8
  33. data/lib/saboteur/doctor.rb +3 -1
  34. data/lib/saboteur/engine/baseline_guard.rb +2 -2
  35. data/lib/saboteur/engine/boot.rb +4 -2
  36. data/lib/saboteur/engine/coverage_partial.rb +8 -1
  37. data/lib/saboteur/engine/diagnostics.rb +120 -15
  38. data/lib/saboteur/engine/result_cache.rb +4 -2
  39. data/lib/saboteur/engine/run_fingerprint.rb +1 -1
  40. data/lib/saboteur/engine/scheduler.rb +22 -0
  41. data/lib/saboteur/engine/selection_announcement.rb +8 -1
  42. data/lib/saboteur/engine/subject_selection.rb +1 -1
  43. data/lib/saboteur/engine/survivor_confirmation.rb +159 -0
  44. data/lib/saboteur/engine/test_guess.rb +1 -1
  45. data/lib/saboteur/engine/thread_warnings.rb +9 -4
  46. data/lib/saboteur/engine.rb +64 -9
  47. data/lib/saboteur/execution/fiddle_loader.rb +29 -2
  48. data/lib/saboteur/help.rb +6 -2
  49. data/lib/saboteur/ignore_list.rb +15 -2
  50. data/lib/saboteur/install.rb +18 -3
  51. data/lib/saboteur/mutation_source/builtin.rb +4 -0
  52. data/lib/saboteur/preview.rb +6 -1
  53. data/lib/saboteur/reporter/agent_commands.rb +6 -1
  54. data/lib/saboteur/reporter/agent_summary.rb +3 -1
  55. data/lib/saboteur/reporter/assets/mutation_report.LICENSE +201 -0
  56. data/lib/saboteur/reporter/assets/mutation_report.js +331 -0
  57. data/lib/saboteur/reporter/baseline_tail.rb +95 -0
  58. data/lib/saboteur/reporter/diff.rb +10 -0
  59. data/lib/saboteur/reporter/html.rb +207 -17
  60. data/lib/saboteur/reporter/kill_causes.rb +53 -0
  61. data/lib/saboteur/reporter/next_steps.rb +11 -8
  62. data/lib/saboteur/reporter/notice.rb +24 -0
  63. data/lib/saboteur/reporter/palette.rb +64 -13
  64. data/lib/saboteur/reporter/reuse_note.rb +37 -0
  65. data/lib/saboteur/reporter/review_comments.rb +11 -7
  66. data/lib/saboteur/reporter/safe_out.rb +1 -1
  67. data/lib/saboteur/reporter/score_table.rb +53 -18
  68. data/lib/saboteur/reporter/stryker.rb +15 -2
  69. data/lib/saboteur/reporter/survivor_hints.rb +70 -0
  70. data/lib/saboteur/reporter/survivor_list.rb +207 -99
  71. data/lib/saboteur/reporter/survivor_ranking.rb +13 -5
  72. data/lib/saboteur/reporter/terminal.rb +306 -69
  73. data/lib/saboteur/result.rb +56 -8
  74. data/lib/saboteur/score.rb +6 -0
  75. data/lib/saboteur/selection/git_diff.rb +19 -1
  76. data/lib/saboteur/survivor_guidance.rb +33 -0
  77. data/lib/saboteur/version.rb +1 -1
  78. metadata +13 -2
@@ -6,8 +6,15 @@ module Saboteur
6
6
  # Exact planned/result reconciliation, independent of survivor acceptance.
7
7
  class Completeness
8
8
  EXECUTED = %w[killed survived timeout].freeze
9
+ # Soft gaps limit what the baseline can speak for; they never refuse a
10
+ # proposal. Everything else is a hard blocker.
11
+ SOFT_PREFIXES = %w[subject_skipped: omitted:].freeze
9
12
  attr_reader :issues
10
13
 
14
+ def self.soft_gap?(issue)
15
+ SOFT_PREFIXES.any? { |prefix| issue.start_with?(prefix) }
16
+ end
17
+
11
18
  def initialize(planned:, results:, subjects:, stable_suite:, stable_inputs:, skipped_subjects:, omitted:,
12
19
  generation_issues:)
13
20
  @issues = []
@@ -25,6 +32,20 @@ module Saboteur
25
32
  issues.empty?
26
33
  end
27
34
 
35
+ def blockers
36
+ issues.reject { |issue| self.class.soft_gap?(issue) }
37
+ end
38
+
39
+ def gaps
40
+ issues.select { |issue| self.class.soft_gap?(issue) }
41
+ end
42
+
43
+ # A proposal may be written over soft gaps (they are listed in it);
44
+ # only hard blockers refuse.
45
+ def proposable?
46
+ blockers.empty?
47
+ end
48
+
28
49
  private
29
50
 
30
51
  def validate_inventory(planned, subjects)
@@ -10,8 +10,8 @@ module Saboteur
10
10
  # Validates the producer document before it can enter the acceptance path.
11
11
  # A claimed complete flag alone is never evidence of complete execution.
12
12
  class Evidence
13
- FIELDS = %w[compatibility snapshot subjects candidates planned complete issues generation_events
14
- omissions discovery_failures inline_suppressions].freeze
13
+ FIELDS = %w[compatibility snapshot subjects candidates planned complete proposable issues blockers gaps
14
+ generation_events omissions discovery_failures inline_suppressions].freeze
15
15
  COMPATIBILITY = %w[generator runtime config scope].freeze
16
16
  PLAN_FIELDS = %w[id identity context].freeze
17
17
 
@@ -27,6 +27,13 @@ module Saboteur
27
27
  validate_compatibility
28
28
  validate_generation
29
29
  validate_plan
30
+ blockers = @document.fetch("blockers")
31
+ gaps = @document.fetch("gaps")
32
+ unless @document.fetch("proposable") == blockers.empty? &&
33
+ (blockers + gaps).sort == @document.fetch("issues")
34
+ raise ConfigError, "baseline evidence: completeness tiers disagree"
35
+ end
36
+
30
37
  # Manifest owns exact candidate, identity, subject and acceptance shapes.
31
38
  Manifest.new(manifest(reasons: "validation"))
32
39
  rescue ArgumentError, KeyError, TypeError, NoMethodError => e
@@ -34,7 +41,8 @@ module Saboteur
34
41
  end
35
42
 
36
43
  def manifest(reasons: "")
37
- metadata = document.fetch("compatibility").merge(document.slice("snapshot", "subjects", "complete", "issues"))
44
+ metadata = document.fetch("compatibility").merge(document.slice("snapshot", "subjects", "complete",
45
+ "proposable", "issues"))
38
46
  accepted = document.fetch("candidates").select { |entry| entry.fetch("status") == "survived" }.map do |entry|
39
47
  entry.slice("id", "context").merge("reason" => reasons)
40
48
  end
@@ -0,0 +1,54 @@
1
+ # SPDX-License-Identifier: MIT
2
+ # frozen_string_literal: true
3
+
4
+ require "digest"
5
+ require "json"
6
+
7
+ require_relative "acceptance"
8
+ require_relative "evidence"
9
+ require_relative "manifest"
10
+ require_relative "../text"
11
+
12
+ module Saboteur
13
+ module Baseline
14
+ # Loading an accepted baseline manifest, shared by the baseline workflow
15
+ # and the `run --baseline` gate: size cap, strict parse, then manifest and
16
+ # evidence validation. Returns [manifest, sha256 of the exact bytes], so a
17
+ # caller can notice the file changing while the run executes.
18
+ module Loader
19
+ module_function
20
+
21
+ def call(path)
22
+ raise ConfigError, "baseline file too large" if File.size(path) > Acceptance::MAX_BYTES
23
+
24
+ bytes = File.read(path)
25
+ digest = Digest::SHA256.hexdigest(bytes)
26
+ manifest = Manifest.new(JSON.parse(Text.utf8_strict(bytes)))
27
+ Evidence.new(restored_evidence(manifest))
28
+ [manifest, digest]
29
+ rescue Errno::ENOENT
30
+ raise ConfigError, "baseline file #{path} does not exist; " \
31
+ "create it with saboteur baseline propose --baseline #{path}"
32
+ rescue SystemCallError, JSON::ParserError, ArgumentError => e
33
+ raise ConfigError, "baseline file: #{e.message}"
34
+ end
35
+
36
+ # The evidence snapshot a manifest stands for: its metadata, candidates
37
+ # and plan, with no generation events (nothing was generated).
38
+ def restored_evidence(manifest)
39
+ metadata = manifest.to_h.fetch("metadata")
40
+ candidates = manifest.candidates.values
41
+ issues = metadata.fetch("issues")
42
+ metadata.slice("snapshot", "subjects", "complete", "issues").merge(
43
+ "proposable" => metadata.fetch("proposable", metadata["complete"]),
44
+ "blockers" => issues.reject { |issue| Completeness.soft_gap?(issue) },
45
+ "gaps" => issues.select { |issue| Completeness.soft_gap?(issue) }
46
+ ).merge(
47
+ "compatibility" => manifest.compatibility, "candidates" => candidates,
48
+ "planned" => candidates.map { |entry| entry.slice(*Evidence::PLAN_FIELDS) },
49
+ "generation_events" => [], "omissions" => [], "discovery_failures" => [], "inline_suppressions" => []
50
+ )
51
+ end
52
+ end
53
+ end
54
+ end
@@ -4,12 +4,17 @@
4
4
  require "digest"
5
5
  require "json"
6
6
 
7
+ require_relative "completeness"
8
+
7
9
  module Saboteur
8
10
  module Baseline
9
11
  # Validated, immutable human acceptance and the complete run it describes.
10
12
  class Manifest
11
13
  FORMAT = 1
12
- METADATA = %w[generator runtime config scope snapshot subjects complete issues].freeze
14
+ METADATA = %w[generator runtime config scope snapshot subjects complete proposable issues].freeze
15
+ # Baselines written before the two-tier completeness split carry no
16
+ # proposable key; their strict complete flag is the proposability gate.
17
+ LEGACY_METADATA = (METADATA - %w[proposable]).freeze
13
18
  STATUSES = %w[killed survived timeout].freeze
14
19
 
15
20
  attr_reader :candidates, :accepted
@@ -48,8 +53,12 @@ module Saboteur
48
53
 
49
54
  def validate_metadata
50
55
  metadata = @document["metadata"]
51
- fields!(metadata, METADATA, "metadata")
52
- fail!("incomplete evidence") unless metadata["complete"] == true && metadata["issues"] == []
56
+ fail!("metadata fields") unless metadata.is_a?(Hash)
57
+ keys = metadata.keys.sort
58
+ fail!("metadata fields") unless [METADATA.sort, LEGACY_METADATA.sort].include?(keys)
59
+ proposable = metadata.fetch("proposable", metadata["complete"])
60
+ hard = metadata.fetch("issues").reject { |issue| Completeness.soft_gap?(issue) }
61
+ fail!("incomplete evidence") unless proposable == true && hard.empty?
53
62
  %w[generator runtime config scope snapshot].each do |name|
54
63
  fail!("metadata #{name}") unless metadata[name].is_a?(Hash) && !metadata[name].empty?
55
64
  end
@@ -121,7 +130,7 @@ module Saboteur
121
130
  current = candidates[entry["id"]]
122
131
  fail!("acceptance candidate") unless current && current["status"] == "survived"
123
132
  fail!("acceptance context") unless current["context"] == entry["context"]
124
- text!(entry["reason"], "reason")
133
+ text!(entry["reason"], "reason on acceptance #{entry['id'][0, 16]} (write a non-blank reason)")
125
134
  end
126
135
 
127
136
  def fields!(value, fields, name)
@@ -12,7 +12,7 @@ module Saboteur
12
12
  # This does not attest external database, network or environment state.
13
13
  class RunEvidence
14
14
  SAFE_DROPS = %w[unchanged duplicate].freeze
15
- OUTPUT_SETTINGS = Config::OUTPUT_KEYS
15
+ OUTPUT_SETTINGS = Config::OUTPUT_KEYS + Config::PRESENTATION_KEYS
16
16
 
17
17
  def initialize(config:, extra_files: [])
18
18
  @config = config
@@ -46,7 +46,9 @@ module Saboteur
46
46
  completeness = assess(observed, stable_suite, skipped_subjects, omissions, errors)
47
47
  document = { "compatibility" => compatibility, "snapshot" => @snapshot.to_h, "subjects" => @subjects,
48
48
  "candidates" => observed, "planned" => @planned,
49
- "complete" => completeness.complete?, "issues" => completeness.issues,
49
+ "complete" => completeness.complete?, "proposable" => completeness.proposable?,
50
+ "issues" => completeness.issues, "blockers" => completeness.blockers,
51
+ "gaps" => completeness.gaps,
50
52
  "generation_events" => generation, "omissions" => omissions.map(&:to_h),
51
53
  "discovery_failures" => discovery, "inline_suppressions" => suppressions }
52
54
  Engine::RunFingerprint.canonical_value(document)
@@ -0,0 +1,51 @@
1
+ # SPDX-License-Identifier: MIT
2
+ # frozen_string_literal: true
3
+
4
+ require_relative "comparison"
5
+
6
+ module Saboteur
7
+ module Baseline
8
+ # The `saboteur run --baseline PATH` verdict: after a run, its execution
9
+ # evidence is compared with the accepted baseline. Comparison owns the
10
+ # classification and the decimal score gate; this names the run-surface
11
+ # outcome: new survivors by digest or a score drop beyond the epsilon
12
+ # fail the run, while accepted legacy survivors no longer do.
13
+ class RunGate
14
+ attr_reader :comparison
15
+
16
+ def initialize(comparison)
17
+ @comparison = comparison
18
+ end
19
+
20
+ def self.build(manifest:, report:, epsilon: nil)
21
+ evidence = report.run_evidence
22
+ raise Error, "--baseline needs the run's execution evidence; the run captured none" unless evidence
23
+
24
+ new(Comparison.new(baseline: { "compatibility" => manifest.compatibility,
25
+ "accepted" => manifest.accepted.values,
26
+ "candidates" => manifest.candidates.values },
27
+ current: evidence, epsilon: epsilon))
28
+ end
29
+
30
+ def new_survivor_ids
31
+ comparison.categories.fetch("new").map { |entry| entry.fetch("id") }
32
+ end
33
+
34
+ def incompatible?
35
+ comparison.state == "incompatible"
36
+ end
37
+
38
+ def incomplete?
39
+ comparison.state == "incomplete"
40
+ end
41
+
42
+ def score_exceeded?
43
+ comparison.score ? comparison.score.fetch("exceeded") : false
44
+ end
45
+
46
+ def failed?
47
+ new_survivor_ids.any? || score_exceeded?
48
+ end
49
+ end
50
+ end
51
+ end
@@ -3,6 +3,8 @@
3
3
 
4
4
  require_relative "acceptance"
5
5
  require_relative "comparison"
6
+ require_relative "identity"
7
+ require_relative "loader"
6
8
  require_relative "../engine"
7
9
  require_relative "../text"
8
10
 
@@ -40,21 +42,48 @@ module Saboteur
40
42
  payload
41
43
  end
42
44
 
45
+ # Blockers carry only a digest and a status. Name the subject, operator
46
+ # and the engine's recorded reason, and say how to unblock the proposal.
47
+ def refuse_proposal(evidence, report)
48
+ blockers = evidence.fetch("blockers")
49
+ by_id = report.results.to_h do |result|
50
+ [Identity.new(subject: result.subject, mutation: result.mutation, root: @root).entry.fetch("id"), result]
51
+ end
52
+ blockers.each { |issue| @err.puts("saboteur: baseline proposal blocker: #{describe_blocker(issue, by_id)}") }
53
+ if blockers.any? { |issue| issue.start_with?("unexecuted:") }
54
+ @err.puts("saboteur: cover the skipped code with specs or narrow include, then propose again")
55
+ end
56
+ raise Error, "baseline proposal needs a proposable run"
57
+ end
58
+
59
+ def describe_blocker(issue, by_id)
60
+ match = /\Aunexecuted:([0-9a-f]{64}):(\w+)\z/.match(issue)
61
+ result = match && by_id[match[1]]
62
+ return issue unless result
63
+
64
+ detail = "#{result.subject.id} #{result.mutation.operator}"
65
+ detail += " - #{result.reason}" if result.reason
66
+ "#{issue} (#{detail})"
67
+ end
68
+
43
69
  def propose(proposal:, destination:, previous: nil)
44
70
  guard_artifacts(proposal, destination)
71
+ preflight_outputs(proposal, destination, previous)
45
72
  report = Engine.new(@config, err: @err, capture_evidence: true).call
46
- unless report.run_evidence["complete"] == true
47
- raise Error,
48
- "baseline proposal requires complete execution evidence"
49
- end
73
+ evidence = report.run_evidence
74
+ refuse_proposal(evidence, report) unless evidence["proposable"] == true
50
75
 
51
- captured = Evidence.new(report.run_evidence)
76
+ captured = Evidence.new(evidence)
52
77
  store(captured.document.fetch("snapshot")).propose(document: captured.manifest,
53
78
  evidence: captured.document, proposal: proposal,
54
79
  destination: destination, previous: previous)
55
- { "state" => "proposal", "proposal" => File.expand_path(proposal),
56
- "destination" => File.expand_path(destination),
57
- "survivors" => captured.manifest.fetch("accepted").size }
80
+ payload = { "state" => "proposal", "proposal" => File.expand_path(proposal),
81
+ "destination" => File.expand_path(destination),
82
+ "survivors" => captured.manifest.fetch("accepted").size,
83
+ "candidates" => captured.manifest.fetch("candidates").size }
84
+ gaps = evidence.fetch("gaps")
85
+ payload["gaps"] = gaps unless gaps.empty?
86
+ payload
58
87
  end
59
88
 
60
89
  def accept(proposal:, destination:, previous: nil)
@@ -67,22 +96,8 @@ module Saboteur
67
96
  private
68
97
 
69
98
  def load_manifest(path)
70
- raise ConfigError, "baseline file too large" if File.size(path) > Acceptance::MAX_BYTES
71
-
72
- bytes = File.read(path)
73
- @baseline_digest = Digest::SHA256.hexdigest(bytes)
74
- manifest = Manifest.new(JSON.parse(Text.utf8_strict(bytes)))
75
- metadata = manifest.to_h.fetch("metadata")
76
- candidates = manifest.candidates.values
77
- restored = metadata.slice("snapshot", "subjects", "complete", "issues").merge(
78
- "compatibility" => manifest.compatibility, "candidates" => candidates,
79
- "planned" => candidates.map { |entry| entry.slice(*Evidence::PLAN_FIELDS) },
80
- "generation_events" => [], "omissions" => [], "discovery_failures" => [], "inline_suppressions" => []
81
- )
82
- Evidence.new(restored)
99
+ manifest, @baseline_digest = Loader.call(path)
83
100
  manifest
84
- rescue SystemCallError, JSON::ParserError, ArgumentError => e
85
- raise ConfigError, "baseline file: #{e.message}"
86
101
  end
87
102
 
88
103
  def store(snapshot)
@@ -93,12 +108,20 @@ module Saboteur
93
108
  extras = saved.inventory.select { |entry| entry.kind == "extra" }.map do |entry|
94
109
  { path: input_path(entry.path), required: entry.required }
95
110
  end
96
- Engine::RunFingerprint.new(config: @config, extra_files: extras).verify(saved).unchanged
111
+ raise_unless_unchanged(Engine::RunFingerprint.new(config: @config, extra_files: extras).verify(saved))
97
112
  end
98
113
  Acceptance.new(directory: File.join(Identity.canonical_path(@config.path("cache_dir")), "baseline-evidence"),
99
114
  protected_paths: protected, verify: verifier, require_evidence: true)
100
115
  end
101
116
 
117
+ def raise_unless_unchanged(result)
118
+ return true if result.unchanged
119
+
120
+ detail = result.changes.map { |change| "#{change.path} #{change.reason}" }.join(", ")
121
+ detail = "verification incomplete" if detail.empty?
122
+ raise ConfigError, "baseline inputs changed or unverified: #{detail}"
123
+ end
124
+
102
125
  def input_paths(snapshot)
103
126
  snapshot.inventory.map { |entry| input_path(entry.path) }
104
127
  end
@@ -107,6 +130,18 @@ module Saboteur
107
130
  path.start_with?("external:") ? path.delete_prefix("external:") : File.join(@root, path)
108
131
  end
109
132
 
133
+ # Cheap pre-checks before the full run; Acceptance re-verifies exclusively after it.
134
+ def preflight_outputs(proposal, destination, previous)
135
+ raise ConfigError, "baseline output exists: #{Identity.canonical_path(proposal)}" if File.exist?(proposal)
136
+
137
+ # Mirrors Acceptance#verify_previous so a doomed proposal fails before the run.
138
+ if previous.nil?
139
+ raise ConfigError, "baseline destination exists" if File.exist?(destination)
140
+ elsif !File.file?(destination) || Digest::SHA256.file(destination).hexdigest != previous
141
+ raise ConfigError, "baseline previous destination changed"
142
+ end
143
+ end
144
+
110
145
  def guard_artifacts(*paths)
111
146
  paths.each do |path|
112
147
  expanded = Identity.canonical_path(path)
@@ -35,7 +35,10 @@ module Saboteur
35
35
 
36
36
  def call
37
37
  expected = @integration.all_tests.map(&:id)
38
- raise Error, "baseline has no tests; check rspec_args and spec paths" if expected.empty?
38
+ if expected.empty?
39
+ raise Error, "baseline has no tests; check rspec_args and the suite filters, " \
40
+ "then run saboteur doctor --suite again"
41
+ end
39
42
 
40
43
  first = check_run(1, expected)
41
44
  second = check_run(2, expected)
@@ -0,0 +1,58 @@
1
+ # SPDX-License-Identifier: MIT
2
+ # frozen_string_literal: true
3
+
4
+ require "digest"
5
+
6
+ module Saboteur
7
+ class CLI
8
+ # The `saboteur run --baseline PATH` gate: loading the accepted baseline
9
+ # (fail fast, before the suite boots) and mapping the comparison to the
10
+ # run's exit code. Mixed into the CLI, which supplies `config`.
11
+ module BaselineGate
12
+ private
13
+
14
+ # The --baseline comparison replaces the ordinary quality-gate verdict.
15
+ def exit_code(report) = config["baseline"] ? baseline_verdict(report) : super
16
+
17
+ # --baseline: the comparison against the accepted baseline is the
18
+ # verdict. Accepted legacy survivors no longer fail the run; new
19
+ # survivors or a score drop beyond --epsilon do (exit 1). An
20
+ # incompatible baseline is a usage error; an incomplete comparison
21
+ # means the run's own evidence cannot judge it (exit 3).
22
+ def baseline_verdict(report)
23
+ gate = baseline_gate(report)
24
+ raise ConfigError, gate.comparison.issues.join("; ") if gate.incompatible?
25
+ return EXIT_TOOL if gate.incomplete?
26
+
27
+ gate.failed? ? EXIT_SURVIVORS : EXIT_OK
28
+ end
29
+
30
+ def baseline_gate(report)
31
+ @baseline_gate ||= begin
32
+ path = config["baseline"]
33
+ if Digest::SHA256.file(path).hexdigest != @baseline_digest
34
+ raise ConfigError, "baseline changed during the run"
35
+ end
36
+
37
+ Baseline::RunGate.build(manifest: baseline_manifest, report: report, epsilon: config["epsilon"] || 0.0)
38
+ end
39
+ end
40
+
41
+ # The accepted baseline at config["baseline"], loaded once (and before
42
+ # the suite boots) so a missing or corrupt file fails fast as a usage
43
+ # error naming the fix.
44
+ def baseline_manifest
45
+ return @baseline_manifest if @baseline_manifest
46
+
47
+ path = config["baseline"]
48
+ raise UsageError, "--baseline file not found: #{path} - run `saboteur baseline propose` first" unless
49
+ File.exist?(path)
50
+
51
+ @baseline_manifest, @baseline_digest = Baseline::Loader.call(path)
52
+ @baseline_manifest
53
+ rescue ConfigError => e
54
+ raise UsageError, "#{e.message} - run `saboteur baseline propose` first"
55
+ end
56
+ end
57
+ end
58
+ end
@@ -1,6 +1,9 @@
1
1
  # SPDX-License-Identifier: MIT
2
2
  # frozen_string_literal: true
3
3
 
4
+ require_relative "../reporter/palette"
5
+ require_relative "../reporter/notice"
6
+
4
7
  module Saboteur
5
8
  class CLI
6
9
  # The uncommitted-work warning printed before a run (#334). Extracted
@@ -21,7 +24,10 @@ module Saboteur
21
24
  else
22
25
  config
23
26
  end
24
- DirtyTreeWarning.for(warning, config_path: options[:config])&.then { |text| @err.puts(text) }
27
+ DirtyTreeWarning.for(warning, config_path: options[:config])&.then do |text|
28
+ palette = Reporter::Palette.for(@err, mode: @color || :auto, overrides: warning["palette"])
29
+ @err.puts(Reporter::Notice.paint(palette, text))
30
+ end
25
31
  end
26
32
  end
27
33
  end
@@ -22,13 +22,22 @@ module Saboteur
22
22
 
23
23
  def doctor_command = help_requested? ? show_help("doctor") : doctor
24
24
 
25
+ # The environment is healthy only if a run would have something to
26
+ # mutate: the same empty source set `run` refuses (exit 2).
27
+ def doctor_verdict
28
+ return EXIT_OK unless config.included_files.empty?
29
+
30
+ @err.puts("saboteur: #{empty_source_set_message}")
31
+ EXIT_USAGE
32
+ end
33
+
25
34
  def doctor
26
35
  doctor_parser.parse!(@argv)
27
36
  raise UsageError, "unexpected arguments: #{@argv.join(' ')}" if @argv.any?
28
37
 
29
38
  options[:tests]&.each { |test| require_existing_test(test) }
30
39
  Doctor.new(config, out: @out, suite: options[:suite] || options.key?(:tests)).call
31
- EXIT_OK
40
+ doctor_verdict
32
41
  rescue ConfigError => e
33
42
  @err.puts("saboteur: #{Text.display_safe_lines(e.message)}")
34
43
  EXIT_USAGE
@@ -15,6 +15,22 @@ module Saboteur
15
15
 
16
16
  LIMIT = 200
17
17
 
18
+ # The machine cause and the one-line human reason for the two exits no
19
+ # report can describe, because the run never produced one. The verdict
20
+ # line names the reason for a nonzero exit; the report-driven codes get
21
+ # theirs from the terminal summary's verdict line.
22
+ # Keyed by exit code. Literals for EXIT_BASELINE_RED and EXIT_CRASH:
23
+ # this file loads before the CLI defines them; the cause pins spec
24
+ # asserts the keys are those two constants.
25
+ CAUSE_LINES = {
26
+ 5 => ["baselineRed", "baseline suite red: fix your tests first"],
27
+ 70 => ["crash", "internal crash (a saboteur bug, never your code)"]
28
+ }.freeze
29
+
30
+ def self.cause(code) = CAUSE_LINES.fetch(code).first
31
+
32
+ def self.verdict_line(code) = "exit #{code} - #{CAUSE_LINES.fetch(code).last}"
33
+
18
34
  # "" when no result carries a reason.
19
35
  def self.line(errored)
20
36
  reasons = errored.filter_map { |result| first_line(result.reason) }.reject(&:empty?)
@@ -8,22 +8,11 @@ module Saboteur
8
8
  module ExitStatus
9
9
  private
10
10
 
11
- # Last resort: a bug must not escape as a bare stack trace and an accidental exit 1.
12
- def internal_error(error)
13
- CrashReport.call(error, verbose: @verbose, agent: agent?, err: @err, out: @out, log_path: crash_log_path)
14
- end
15
-
16
- # The crash log lives with the other run state. A broken config (or a
17
- # nil cache_dir) must not swallow the crash report itself: no log.
18
- def crash_log_path
19
- config.path("cache_dir")&.then { |dir| File.join(dir, CrashReport::LOG_NAME) }
20
- rescue StandardError
21
- nil
22
- end
23
-
24
11
  def exit_code(report)
25
12
  return EXIT_BUDGET if budget_exhausted?(report)
26
13
 
14
+ return EXIT_OK if ignore_empty_diff?(report)
15
+
27
16
  return EXIT_TOOL if untested_partial?(report)
28
17
 
29
18
  return EXIT_TOOL if unverifiable_scope?(report)
@@ -36,6 +25,17 @@ module Saboteur
36
25
  gates.failed?(report, @err, record_cause: !quality_failed) || quality_failed ? EXIT_SURVIVORS : EXIT_OK
37
26
  end
38
27
 
28
+ # --ignore-empty: a --since diff that selected no mutants exits 0 with
29
+ # a note; suppressed, omitted or undiscovered mutants still gate.
30
+ def ignore_empty_diff?(report)
31
+ return false unless config["ignore_empty"] && config["since"] && report.results.empty?
32
+ return false unless report.omissions.empty? && report.inline_suppressions.empty? &&
33
+ report.discovery_failures.empty?
34
+
35
+ @err.puts("saboteur: no mutants in the diff - nothing to gate (--ignore-empty)")
36
+ true
37
+ end
38
+
39
39
  # A spent --max-duration budget exits 4 on its own, before the gates:
40
40
  # the run is partial by design, and --max-duration-ok opts back into
41
41
  # the plain verdict.
@@ -60,9 +60,10 @@ module Saboteur
60
60
  # must not exit 0 with score n/a and green-light an untested change.
61
61
  def unverifiable_scope?(report)
62
62
  if unverifiable_review?(report)
63
- report.skipped_subjects.each do |subject, reason|
64
- @err.puts(subject.skipped_gate_sentence(reason, report.root))
65
- end
63
+ # A text run already printed each skip up front; json and agent
64
+ # runs read this sentence as the exit explanation (#1227).
65
+ skips = %w[json agent].include?(@format) ? report.skipped_subjects : report.unannounced_skips
66
+ skips.each { |subject, reason| @err.puts(subject.skipped_gate_sentence(reason, report.root)) }
66
67
  return true
67
68
  end
68
69
  return false unless unverifiable_unreachable?(report)
@@ -112,6 +113,7 @@ module Saboteur
112
113
  when EXIT_OK then "ok"
113
114
  when EXIT_BUDGET then "budgetExhausted"
114
115
  when EXIT_SURVIVORS then survivors_cause(report)
116
+ when EXIT_BASELINE_RED, EXIT_CRASH then ErrorCause.cause(code)
115
117
  else
116
118
  if report.majority_errored?
117
119
  "errorMajority"
@@ -39,7 +39,7 @@ module Saboteur
39
39
  # A test may carry an example selector: spec/a_spec.rb:12 or spec/a_spec.rb[1:2].
40
40
  def require_existing_test(test)
41
41
  path = test.sub(/(:\d+|\[[\d:,]+\])+\z/, "")
42
- raise UsageError, "test file not found: #{test}" unless File.exist?(path)
42
+ raise UsageError, "test file not found: #{test}; check the path or drop --test" unless File.exist?(path)
43
43
  end
44
44
  end
45
45
  end
@@ -1,6 +1,8 @@
1
1
  # SPDX-License-Identifier: MIT
2
2
  # frozen_string_literal: true
3
3
 
4
+ require_relative "../reporter/palette"
5
+
4
6
  module Saboteur
5
7
  class CLI
6
8
  # Where a failed command speaks: stderr for people, and in agent mode the
@@ -8,8 +10,22 @@ module Saboteur
8
10
  module FailureOutput
9
11
  private
10
12
 
13
+ # Last resort: a bug must not escape as a bare stack trace and an accidental exit 1.
14
+ def internal_error(error)
15
+ CrashReport.call(error, verbose: @verbose, agent: agent?, err: @err, out: @out,
16
+ log_path: crash_log_path, color: @color || :auto)
17
+ end
18
+
19
+ # The crash log lives with the other run state. A broken config (or a
20
+ # nil cache_dir) must not swallow the crash report itself: no log.
21
+ def crash_log_path
22
+ config.path("cache_dir")&.then { |dir| File.join(dir, CrashReport::LOG_NAME) }
23
+ rescue StandardError
24
+ nil
25
+ end
26
+
11
27
  def usage_error(message = nil)
12
- @err.puts(message) if message
28
+ @err.puts(Reporter::Palette.for(@err, mode: @color || :auto).paint(:fail, message)) if message
13
29
  @err.puts(USAGE)
14
30
  agent_failure(message || USAGE, EXIT_USAGE)
15
31
  end
@@ -27,9 +43,16 @@ module Saboteur
27
43
  # name must not become a workflow command in the CI log).
28
44
  def report_failure(error, exit_code)
29
45
  message = Text.display_safe_lines(error.message)
30
- @err.puts("saboteur: #{message}")
46
+ @err.puts(Reporter::Palette.for(@err, mode: @color || :auto).paint(:fail, "saboteur: #{message}"))
47
+ @err.puts(verdict_line(EXIT_BASELINE_RED)) if exit_code == EXIT_BASELINE_RED
31
48
  agent_failure(message, exit_code)
32
49
  end
50
+
51
+ # The terminal verdict names the reason for a nonzero exit. Painted
52
+ # red under the same colour rules as the report.
53
+ def verdict_line(code)
54
+ Reporter::Palette.for(@err, mode: @color || :auto).paint(:fail, ErrorCause.verdict_line(code))
55
+ end
33
56
  end
34
57
  end
35
58
  end