saboteur 0.6.0 → 0.8.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (133) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +1024 -0
  3. data/README.md +199 -952
  4. data/exe/saboteur +34 -7
  5. data/lib/saboteur/baseline/workflow.rb +2 -1
  6. data/lib/saboteur/baseline_health.rb +40 -6
  7. data/lib/saboteur/cli/exit_status.rb +80 -4
  8. data/lib/saboteur/cli/help_options.rb +17 -0
  9. data/lib/saboteur/cli/history_command.rb +135 -0
  10. data/lib/saboteur/cli/presets.rb +40 -0
  11. data/lib/saboteur/cli/run_budget_options.rb +37 -0
  12. data/lib/saboteur/cli/run_reporting.rb +74 -5
  13. data/lib/saboteur/cli/selection_options.rb +31 -0
  14. data/lib/saboteur/cli/shard_options.rb +17 -0
  15. data/lib/saboteur/cli/source_set.rb +53 -0
  16. data/lib/saboteur/cli/survivors_command.rb +157 -0
  17. data/lib/saboteur/cli.rb +136 -98
  18. data/lib/saboteur/config.rb +163 -15
  19. data/lib/saboteur/config_file.rb +8 -0
  20. data/lib/saboteur/constant_reads.rb +182 -0
  21. data/lib/saboteur/constant_subject.rb +70 -0
  22. data/lib/saboteur/constant_subject_finder.rb +281 -0
  23. data/lib/saboteur/coverage_reads.rb +29 -0
  24. data/lib/saboteur/crash_report.rb +154 -0
  25. data/lib/saboteur/custom_map.rb +3 -1
  26. data/lib/saboteur/dirty_tree_warning.rb +128 -0
  27. data/lib/saboteur/doctor.rb +49 -1
  28. data/lib/saboteur/engine/baseline_guard.rb +136 -14
  29. data/lib/saboteur/engine/boot.rb +16 -1
  30. data/lib/saboteur/engine/canary.rb +41 -10
  31. data/lib/saboteur/engine/candidate_selection.rb +28 -1
  32. data/lib/saboteur/engine/candidates.rb +43 -3
  33. data/lib/saboteur/engine/coverage_guard.rb +29 -0
  34. data/lib/saboteur/engine/coverage_store.rb +79 -15
  35. data/lib/saboteur/engine/defined_order_prefix.rb +43 -0
  36. data/lib/saboteur/engine/diagnostics.rb +41 -2
  37. data/lib/saboteur/engine/diff_scope.rb +24 -5
  38. data/lib/saboteur/engine/distribution.rb +134 -8
  39. data/lib/saboteur/engine/early_stop_recording.rb +47 -0
  40. data/lib/saboteur/engine/gated_run.rb +35 -0
  41. data/lib/saboteur/engine/load_coverage.rb +45 -0
  42. data/lib/saboteur/engine/out_of_scope_guard.rb +21 -0
  43. data/lib/saboteur/engine/pipelines.rb +107 -0
  44. data/lib/saboteur/engine/result_cache.rb +67 -9
  45. data/lib/saboteur/engine/run_budget.rb +90 -0
  46. data/lib/saboteur/engine/run_fingerprint.rb +4 -0
  47. data/lib/saboteur/engine/sandbox_policy.rb +56 -11
  48. data/lib/saboteur/engine/scheduler.rb +222 -20
  49. data/lib/saboteur/engine/selection_announcement.rb +54 -0
  50. data/lib/saboteur/engine/shard.rb +69 -0
  51. data/lib/saboteur/engine/subject_selection.rb +373 -19
  52. data/lib/saboteur/engine/suite_files.rb +3 -1
  53. data/lib/saboteur/engine/test_guess.rb +438 -25
  54. data/lib/saboteur/engine/thread_warnings.rb +53 -0
  55. data/lib/saboteur/engine/unchanged_syntax.rb +17 -2
  56. data/lib/saboteur/engine/unscoped_suite_warning.rb +41 -0
  57. data/lib/saboteur/engine/verdict_check.rb +54 -18
  58. data/lib/saboteur/engine.rb +202 -158
  59. data/lib/saboteur/execution/external_runner.rb +471 -0
  60. data/lib/saboteur/execution/marker_sweep.rb +79 -0
  61. data/lib/saboteur/execution/peak_memory.rb +28 -0
  62. data/lib/saboteur/execution/process_table.rb +12 -0
  63. data/lib/saboteur/execution/process_tree.rb +65 -5
  64. data/lib/saboteur/execution/protocol.rb +2 -1
  65. data/lib/saboteur/execution/reevaluator.rb +68 -12
  66. data/lib/saboteur/execution/runner.rb +225 -24
  67. data/lib/saboteur/execution/sandbox.rb +21 -3
  68. data/lib/saboteur/execution/seatbelt.rb +226 -0
  69. data/lib/saboteur/execution/supervisor.rb +45 -13
  70. data/lib/saboteur/execution/worker_pool.rb +110 -8
  71. data/lib/saboteur/frameworks/sqlite_memory_database.rb +49 -0
  72. data/lib/saboteur/gates.rb +43 -2
  73. data/lib/saboteur/help.rb +108 -13
  74. data/lib/saboteur/ignore_list.rb +40 -7
  75. data/lib/saboteur/install.rb +4 -3
  76. data/lib/saboteur/integration/base.rb +7 -2
  77. data/lib/saboteur/integration/rspec.rb +175 -9
  78. data/lib/saboteur/mutation_source/argument_removal.rb +1 -1
  79. data/lib/saboteur/mutation_source/builtin.rb +44 -14
  80. data/lib/saboteur/mutation_source/chain_removal.rb +1 -1
  81. data/lib/saboteur/mutation_source/condition_negation.rb +1 -1
  82. data/lib/saboteur/mutation_source/constant_literals.rb +66 -0
  83. data/lib/saboteur/mutation_source/element_removal.rb +73 -0
  84. data/lib/saboteur/mutation_source/endpoint_replacement.rb +1 -1
  85. data/lib/saboteur/mutation_source/entry_edits.rb +44 -0
  86. data/lib/saboteur/mutation_source/filter_replacement.rb +1 -1
  87. data/lib/saboteur/mutation_source/hazard_guard.rb +129 -3
  88. data/lib/saboteur/mutation_source/interpolation_removal.rb +41 -0
  89. data/lib/saboteur/mutation_source/literal_replacement.rb +1 -1
  90. data/lib/saboteur/mutation_source/method_swap.rb +193 -0
  91. data/lib/saboteur/mutation_source/mutation.rb +5 -3
  92. data/lib/saboteur/mutation_source/operand_replacement.rb +54 -0
  93. data/lib/saboteur/mutation_source/operator_assignment.rb +1 -1
  94. data/lib/saboteur/mutation_source/operator_examples.rb +6 -1
  95. data/lib/saboteur/mutation_source/operator_registry.rb +7 -2
  96. data/lib/saboteur/mutation_source/operator_visit.rb +39 -0
  97. data/lib/saboteur/mutation_source/pair_removal.rb +13 -31
  98. data/lib/saboteur/mutation_source/range_replacement.rb +1 -1
  99. data/lib/saboteur/mutation_source/regex_replacement.rb +93 -0
  100. data/lib/saboteur/preview.rb +122 -13
  101. data/lib/saboteur/refinement_activations.rb +60 -0
  102. data/lib/saboteur/reporter/agent.rb +9 -1
  103. data/lib/saboteur/reporter/agent_commands.rb +47 -15
  104. data/lib/saboteur/reporter/agent_incomplete.rb +38 -2
  105. data/lib/saboteur/reporter/agent_summary.rb +4 -3
  106. data/lib/saboteur/reporter/findings.rb +5 -2
  107. data/lib/saboteur/reporter/history_policy.rb +11 -0
  108. data/lib/saboteur/reporter/history_reader.rb +47 -0
  109. data/lib/saboteur/reporter/html.rb +26 -4
  110. data/lib/saboteur/reporter/mutation_text.rb +6 -0
  111. data/lib/saboteur/reporter/next_steps.rb +14 -4
  112. data/lib/saboteur/reporter/score_history.rb +5 -1
  113. data/lib/saboteur/reporter/stryker.rb +40 -7
  114. data/lib/saboteur/reporter/survivor_classification.rb +101 -0
  115. data/lib/saboteur/reporter/survivor_hints.rb +3 -0
  116. data/lib/saboteur/reporter/survivor_list.rb +15 -9
  117. data/lib/saboteur/reporter/terminal.rb +74 -17
  118. data/lib/saboteur/result.rb +38 -7
  119. data/lib/saboteur/selection/coverage_map.rb +107 -11
  120. data/lib/saboteur/selection/dependency_shield.rb +80 -0
  121. data/lib/saboteur/selection/external_test_map.rb +27 -0
  122. data/lib/saboteur/selection/git_diff.rb +105 -13
  123. data/lib/saboteur/selection/layout_only_lines.rb +80 -0
  124. data/lib/saboteur/selection/recorder.rb +22 -4
  125. data/lib/saboteur/selection/source_scope.rb +22 -0
  126. data/lib/saboteur/selection/untracked_changes.rb +126 -0
  127. data/lib/saboteur/subject.rb +53 -7
  128. data/lib/saboteur/subject_finder.rb +173 -29
  129. data/lib/saboteur/text.rb +42 -0
  130. data/lib/saboteur/version.rb +1 -1
  131. data/lib/saboteur/working_tree.rb +38 -15
  132. data/lib/saboteur.rb +29 -1
  133. metadata +50 -2
data/exe/saboteur CHANGED
@@ -2,15 +2,42 @@
2
2
  # SPDX-License-Identifier: MIT
3
3
  # frozen_string_literal: true
4
4
 
5
- # Line coverage must be set up before any target code loads, so it happens
6
- # first. For `run` it also counts from here: what executes while the app and
5
+ # Coverage comes first because it only measures files loaded after it -
6
+ # the crash reporter must stay measurable - and it loads inside the same
7
+ # guarded begin as the reporter: every later failure, even while the CLI
8
+ # is being required, gets the full report (the bounded one-liner, the log
9
+ # file, the --verbose backtrace and agent JSON when the CLI loaded far
10
+ # enough). If even the reporter cannot load, a bare normalized one-liner
11
+ # (one line, invalid bytes scrubbed, bounded length) still exits 3.
12
+ begin
13
+ require "coverage"
14
+ Coverage.setup(lines: true, methods: true, branches: true) if Coverage.state == :idle
15
+
16
+ require "saboteur/crash_report"
17
+ rescue LoadError, StandardError => e
18
+ message = e.message.to_s.dup.force_encoding(Encoding::UTF_8).scrub("?").gsub(/\s+/, " ").strip
19
+ message = "#{message[0...160]}..." if message.length > 160
20
+ warn "saboteur: internal error (#{e.class}: #{message}) - please report it: " \
21
+ "https://github.com/vamsipavanmahesh/saboteur/issues"
22
+ exit 3
23
+ end
24
+
25
+ # For `run` coverage counts from here: what executes while the app and
7
26
  # spec files load is how ran-at-load mutants are told apart from uncovered
8
27
  # ones. The engine suspends counting right after boot and reads the
9
28
  # counters as the map's load lines (a forked child would not inherit them).
10
- require "coverage"
11
- Coverage.setup(lines: true, methods: true, branches: true) if Coverage.state == :idle
12
- Coverage.resume if ARGV.first == "run" && Coverage.state == :suspended
29
+ begin
30
+ Coverage.resume if ARGV.first == "run" && Coverage.state == :suspended
13
31
 
14
- require "saboteur/cli"
32
+ require "saboteur/cli"
15
33
 
16
- exit Saboteur::CLI.new(ARGV).call
34
+ exit Saboteur::CLI.new(ARGV).call
35
+ rescue LoadError, StandardError => e
36
+ # Agent mode is visible in the raw arguments; the CLI may never have
37
+ # loaded, so this scan must not depend on it.
38
+ agent = ARGV.include?("--agent") || ARGV.include?("--format=agent") ||
39
+ ARGV.each_cons(2).any? { |flag, value| flag == "--format" && value == "agent" }
40
+ exit Saboteur::CrashReport.call(e, verbose: ARGV.include?("--verbose"), agent: agent,
41
+ err: $stderr, out: $stdout,
42
+ log_path: File.join(Dir.pwd, ".saboteur", "internal-error.log"))
43
+ end
@@ -4,6 +4,7 @@
4
4
  require_relative "acceptance"
5
5
  require_relative "comparison"
6
6
  require_relative "../engine"
7
+ require_relative "../text"
7
8
 
8
9
  module Saboteur
9
10
  module Baseline
@@ -70,7 +71,7 @@ module Saboteur
70
71
 
71
72
  bytes = File.read(path)
72
73
  @baseline_digest = Digest::SHA256.hexdigest(bytes)
73
- manifest = Manifest.new(JSON.parse(bytes))
74
+ manifest = Manifest.new(JSON.parse(Text.utf8_strict(bytes)))
74
75
  metadata = manifest.to_h.fetch("metadata")
75
76
  candidates = manifest.candidates.values
76
77
  restored = metadata.slice("snapshot", "subjects", "complete", "issues").merge(
@@ -4,6 +4,8 @@
4
4
  require "json"
5
5
  require "tempfile"
6
6
 
7
+ require_relative "text"
8
+
7
9
  module Saboteur
8
10
  # Two fresh supervised children check the full suite without changing methods
9
11
  # or writing caches. This samples stability; it cannot prove absence of flakes.
@@ -49,8 +51,9 @@ module Saboteur
49
51
 
50
52
  def check_run(number, expected)
51
53
  Tempfile.create("saboteur-baseline") do |log|
52
- result = @supervisor.run(timeout: @config["baseline_timeout"]) { |writer| record(writer, log.path) }
53
- data = read_result(result, number)
54
+ result = @supervisor.run(timeout: @config["baseline_timeout"],
55
+ label: "baseline child died during setup/run") { |writer| record(writer, log.path) }
56
+ data = read_result(result, number, log)
54
57
  verify_execution(data, number, expected)
55
58
  raise Error, "baseline run #{number} left processes running; fix suite cleanup" if result.leftover_pids.any?
56
59
 
@@ -64,9 +67,12 @@ module Saboteur
64
67
  raise Error, "baseline check failed: #{e.class}: #{e.message}; check suite hooks and setup"
65
68
  end
66
69
 
67
- def read_result(result, number)
70
+ def read_result(result, number, log = nil)
68
71
  raise Error, "baseline run #{number} stopped: #{result.detail}" if result.stopped_by
69
- raise Error, "baseline run #{number} exited without a result; check suite boot and exits" if result.payload.empty?
72
+ if result.payload.empty?
73
+ raise Error, "baseline child died during setup/run: baseline run #{number} exited without a " \
74
+ "result#{exit_detail(result.exit_status)}; check suite boot and exits#{log_tail(log)}"
75
+ end
70
76
  if result.child_failed?
71
77
  raise Error, "baseline run #{number} exited unsuccessfully; fix suite cleanup and exit handlers"
72
78
  end
@@ -77,6 +83,30 @@ module Saboteur
77
83
  data
78
84
  end
79
85
 
86
+ # The child's own output when it died without answering: the redirect
87
+ # puts its last words in this log, not on the terminal (#289). Bounded
88
+ # (about 2 KB, 20 lines) and scrubbed before it joins the error. Read
89
+ # through the parent's open handle, and strictly best-effort: a dead
90
+ # handle must never mask the real exit or signal error.
91
+ def log_tail(log)
92
+ return "" unless log
93
+
94
+ log.seek([log.size - 2048, 0].max)
95
+ tail = Text.utf8(log.read(2048).to_s).lines.last(20).map(&:chomp).join(" / ")
96
+ tail.empty? ? "" : "; child log tail: #{tail}"
97
+ rescue SystemCallError, IOError
98
+ ""
99
+ end
100
+
101
+ # The child's ending when the supervisor reaped one ("exit 1",
102
+ # "signal SIGKILL", ...): an empty result's cause is usually there (#255).
103
+ def exit_detail(status)
104
+ return "" unless status.is_a?(Process::Status)
105
+ return " (signal #{Signal.signame(status.termsig)})" if status.signaled?
106
+
107
+ " (exit #{status.exitstatus})"
108
+ end
109
+
80
110
  def verify_execution(data, number, expected)
81
111
  unless data["passed"] == true
82
112
  raise Error, "baseline run #{number} failed: #{data['error'] || 'fix the unchanged suite'}"
@@ -93,8 +123,12 @@ module Saboteur
93
123
  exception = example.execution_result.exception
94
124
  next unless exception
95
125
 
96
- message = exception.message.to_s.lines.map(&:strip).find { |line| !line.empty? }.to_s[0, 200]
97
- "#{example.id}: #{example.full_description.to_s[0, 200]}: #{message}"
126
+ # Exception text and example names can arrive locale-tagged or as
127
+ # raw path bytes (e.g. Errno::EILSEQ): scrub each field before any
128
+ # strip, truncation or interpolation - all three raise on bad tags.
129
+ # This detail ships in a JSON payload, so it must be valid UTF-8.
130
+ message = Text.utf8(exception.message.to_s).lines.map(&:strip).find { |line| !line.empty? }.to_s[0, 200]
131
+ "#{Text.utf8(example.id.to_s)}: #{Text.utf8(example.full_description.to_s)[0, 200]}: #{message}"
98
132
  end.first(5).join("; ")
99
133
  end
100
134
 
@@ -8,7 +8,28 @@ module Saboteur
8
8
  module ExitStatus
9
9
  private
10
10
 
11
+ # Last resort: a bug must not escape as a bare stack trace and an accidental exit 1.
12
+ def internal_error(error)
13
+ CrashReport.call(error, verbose: @verbose, agent: agent?, err: @err, out: @out, log_path: crash_log_path)
14
+ end
15
+
16
+ # The crash log lives with the other run state. A broken config (or a
17
+ # nil cache_dir) must not swallow the crash report itself, so fall
18
+ # back to no log.
19
+ def crash_log_path
20
+ dir = config.path("cache_dir")
21
+ dir && File.join(dir, CrashReport::LOG_NAME)
22
+ rescue StandardError
23
+ nil
24
+ end
25
+
11
26
  def exit_code(report)
27
+ return EXIT_BUDGET if budget_exhausted?(report)
28
+
29
+ return EXIT_TOOL if untested_partial?(report)
30
+
31
+ return EXIT_TOOL if unverifiable_scope?(report)
32
+
12
33
  return EXIT_TOOL if mostly_errored?(report)
13
34
 
14
35
  return EXIT_TOOL if incomplete_under_threshold?(report)
@@ -16,6 +37,50 @@ module Saboteur
16
37
  failed_quality_gate?(report) || gates.failed?(report, @err) ? EXIT_SURVIVORS : EXIT_OK
17
38
  end
18
39
 
40
+ # A spent --max-duration budget exits 4 on its own, before the gates:
41
+ # the run is partial by design, and --max-duration-ok opts back into
42
+ # the plain verdict.
43
+ def budget_exhausted?(report)
44
+ report.early_stop&.reason == "max_duration" && !config["max_duration_ok"]
45
+ end
46
+
47
+ # A run stopped before any mutant was tested has no verdict to fall
48
+ # back on: --max-duration-ok must not turn it green. Unverified
49
+ # results are not tested verdicts, so an unverified-only stop has no
50
+ # score either (#296). Exit 3, like the other incomplete shapes.
51
+ def untested_partial?(report)
52
+ return false unless report.early_stop && report.score_breakdown.tested.zero?
53
+
54
+ @err.puts("saboteur: the run stopped before any mutant was tested, so there is no score to judge")
55
+ true
56
+ end
57
+
58
+ # A run that verified none of the change under review - every
59
+ # subject skipped (refinements, no coverage, ..., #574) or every
60
+ # mutant unreachable (a suite re-loading mutated files, #597) -
61
+ # must not exit 0 with score n/a and green-light an untested change.
62
+ def unverifiable_scope?(report)
63
+ if unverifiable_review?(report)
64
+ report.skipped_subjects.each do |subject, reason|
65
+ @err.puts("saboteur: #{subject.id} was skipped (#{reason}), so the change under test was not verified")
66
+ end
67
+ return true
68
+ end
69
+ return false unless unverifiable_unreachable?(report)
70
+
71
+ @err.puts("saboteur: all mutants ran unreachable, so the change under test was not verified")
72
+ true
73
+ end
74
+
75
+ def unverifiable_review?(report)
76
+ report.review? && report.subjects.any? &&
77
+ report.subjects.all? { |subject| report.skipped_subjects.key?(subject) }
78
+ end
79
+
80
+ def unverifiable_unreachable?(report)
81
+ report.executed.any? && report.executed.all? { |result| result.status == :unreachable }
82
+ end
83
+
19
84
  def gates = (@gates ||= Gates.new(config))
20
85
 
21
86
  def mostly_errored?(report)
@@ -33,9 +98,11 @@ module Saboteur
33
98
  # incomplete run that mostly errored keeps its distinct cause, so the
34
99
  # harness rule means the same thing in both modes.
35
100
  def agent_exit_cause(report, payload)
101
+ return "budgetExhausted" if budget_exhausted?(report)
102
+
36
103
  case payload[:verdict]
37
104
  when "clean" then "ok"
38
- when "survivors" then report.early_stop ? "failFast" : "survivors"
105
+ when "survivors" then report.early_stop&.reason == "fail_fast" ? "failFast" : "survivors"
39
106
  else report.majority_errored? ? "errorMajority" : "incomplete"
40
107
  end
41
108
  end
@@ -45,22 +112,31 @@ module Saboteur
45
112
  def exit_cause(report, code)
46
113
  case code
47
114
  when EXIT_OK then "ok"
115
+ when EXIT_BUDGET then "budgetExhausted"
48
116
  when EXIT_SURVIVORS then survivors_cause(report)
49
- else report.majority_errored? ? "errorMajority" : "incompleteUnderThreshold"
117
+ else
118
+ if report.majority_errored?
119
+ "errorMajority"
120
+ elsif unverifiable_review?(report) || unverifiable_unreachable?(report) ||
121
+ (report.early_stop && report.score_breakdown.tested.zero?)
122
+ "incomplete"
123
+ else
124
+ "incompleteUnderThreshold"
125
+ end
50
126
  end
51
127
  end
52
128
 
53
129
  # A run stopped by --fail-fast always exits 1: it stopped on a survivor, and
54
130
  # its score covers only the subjects that ran, so no threshold can pass it.
55
131
  def survivors_cause(report)
56
- return "failFast" if report.early_stop
132
+ return "failFast" if report.early_stop&.reason == "fail_fast"
57
133
 
58
134
  gates.cause || (config["threshold"].nil? ? "survivors" : "belowThreshold")
59
135
  end
60
136
 
61
137
  # With --threshold the score decides; without it any unignored survivor does.
62
138
  def failed_quality_gate?(report)
63
- return true if report.early_stop
139
+ return true if report.early_stop&.reason == "fail_fast"
64
140
 
65
141
  threshold = config["threshold"]
66
142
  return report.unignored_survivors.any? if threshold.nil?
@@ -0,0 +1,17 @@
1
+ # SPDX-License-Identifier: MIT
2
+ # frozen_string_literal: true
3
+
4
+ module Saboteur
5
+ class CLI
6
+ # The option map `saboteur help` prints per command. run, pr and audit
7
+ # share the run parser: the presets take every run option.
8
+ module HelpOptions
9
+ def option_lines
10
+ run = parser.summarize
11
+ { "run" => run, "pr" => run, "audit" => run, "doctor" => doctor_parser.summarize,
12
+ "baseline" => baseline_parser.summarize, "badge" => badge_parser.summarize,
13
+ "trend" => trend_parser.summarize }
14
+ end
15
+ end
16
+ end
17
+ end
@@ -0,0 +1,135 @@
1
+ # SPDX-License-Identifier: MIT
2
+ # frozen_string_literal: true
3
+
4
+ require_relative "../reporter/history_reader"
5
+
6
+ module Saboteur
7
+ class CLI
8
+ # `saboteur badge` and `saboteur trend` (#474): a local README badge and
9
+ # a trend read-out over the opt-in score history, no hosted service.
10
+ module HistoryCommand
11
+ DEFAULT_HIGH = 80.0
12
+ DEFAULT_LOW = 60.0
13
+
14
+ def badge_parser
15
+ OptionParser.new do |opts|
16
+ opts.banner = "Usage: saboteur badge [--history PATH] [--output PATH] [--mark-partial] " \
17
+ "[--high PERCENT] [--low PERCENT]"
18
+ opts.on("--history PATH", "Score history JSONL (default: c.score_history)") { |v| @history_path = v }
19
+ opts.on("--output PATH", "Badge JSON destination (default: stdout)") { |v| @output = v }
20
+ opts.on("--mark-partial", "Badge an incomplete run, labeled as partial") { @mark_partial = true }
21
+ opts.on("--high PERCENT", Float, "brightgreen at or above (default #{DEFAULT_HIGH.to_i})") { |v| @high = v }
22
+ opts.on("--low PERCENT", Float, "yellow at or above (default #{DEFAULT_LOW.to_i})") { |v| @low = v }
23
+ end
24
+ end
25
+
26
+ def trend_parser
27
+ OptionParser.new do |opts|
28
+ opts.banner = "Usage: saboteur trend [--history PATH]"
29
+ opts.on("--history PATH", "Score history JSONL (default: c.score_history)") { |v| @history_path = v }
30
+ end
31
+ end
32
+
33
+ private
34
+
35
+ def history_command(command) = command == "badge" ? badge : trend
36
+
37
+ def badge
38
+ badge_parser.parse!(@argv)
39
+ raise UsageError, "unexpected arguments: #{@argv.join(' ')}" if @argv.any?
40
+
41
+ write_badge(badge_payload(read_history.last))
42
+ EXIT_OK
43
+ rescue ConfigError, Error, SystemCallError => e
44
+ history_failure(e)
45
+ end
46
+
47
+ def trend
48
+ trend_parser.parse!(@argv)
49
+ raise UsageError, "unexpected arguments: #{@argv.join(' ')}" if @argv.any?
50
+
51
+ print_trend(latest_pair)
52
+ EXIT_OK
53
+ rescue ConfigError, Error, SystemCallError => e
54
+ history_failure(e)
55
+ end
56
+
57
+ def history_failure(error)
58
+ @err.puts("saboteur: #{error.message}")
59
+ error.is_a?(ConfigError) ? EXIT_USAGE : EXIT_TOOL
60
+ end
61
+
62
+ def latest_pair
63
+ records = read_history
64
+ current = records.last
65
+ [records[0...-1].reverse.find { |record| Reporter::HistoryReader.comparable?(record, current) }, current]
66
+ end
67
+
68
+ def read_history
69
+ records = Reporter::HistoryReader.read(history_path)
70
+ if records.empty?
71
+ raise Error,
72
+ "score history is empty - record it first by running saboteur with c.score_history set"
73
+ end
74
+
75
+ records
76
+ end
77
+
78
+ def history_path
79
+ @history_path ||= config.path("score_history")
80
+ @history_path or raise UsageError, "no score history: pass --history PATH or set c.score_history"
81
+ rescue ConfigError
82
+ raise UsageError, "no score history: pass --history PATH or set c.score_history"
83
+ end
84
+
85
+ def badge_payload(record)
86
+ partial = record["incomplete"].any?
87
+ if partial && !@mark_partial
88
+ raise Error, "the latest score history record is an incomplete run " \
89
+ "(#{record['incomplete'].map(&:inspect).join(', ')}) - " \
90
+ "badge evidence needs a complete run, or pass --mark-partial to label it"
91
+ end
92
+ score = record["coveredScore"]
93
+ raise Error, "the latest score history record has no tested mutants" if score.nil?
94
+
95
+ { schemaVersion: 1, label: partial ? "mutation score (partial)" : "mutation score",
96
+ message: format("%.1f%%", score * 100), color: badge_color(score) }
97
+ end
98
+
99
+ def write_badge(payload)
100
+ json = "#{JSON.generate(payload)}\n"
101
+ return @out.print(json) unless @output
102
+
103
+ File.write(@output, json)
104
+ @out.puts(@output)
105
+ end
106
+
107
+ def badge_color(score)
108
+ percent = score * 100
109
+ return "brightgreen" if percent >= (@high || DEFAULT_HIGH)
110
+ return "yellow" if percent >= (@low || DEFAULT_LOW)
111
+
112
+ "red"
113
+ end
114
+
115
+ def print_trend((previous, current))
116
+ ([previous].compact + [current]).each { |record| @out.puts(trend_line(record)) }
117
+ unless previous && previous["coveredScore"] && current["coveredScore"]
118
+ return @out.puts("score history: no comparable earlier record (different selection/inventory, " \
119
+ "operator policy or an incomplete run)")
120
+ end
121
+
122
+ delta = (current["coveredScore"] - previous["coveredScore"]) * 100
123
+ @out.puts(format("covered score change %<delta>+.2f percentage points since %<since>s " \
124
+ "(same configured selection)", delta: delta, since: previous["date"]))
125
+ end
126
+
127
+ def trend_line(record)
128
+ suffix = record["incomplete"].empty? ? "" : " (incomplete)"
129
+ score = record["coveredScore"]
130
+ "#{record['date']} #{record['commit']&.slice(0, 8) || '-'} " \
131
+ "#{score.nil? ? 'no tested mutants' : format('%.1f%%', score * 100)}#{suffix}"
132
+ end
133
+ end
134
+ end
135
+ end
@@ -0,0 +1,40 @@
1
+ # SPDX-License-Identifier: MIT
2
+ # frozen_string_literal: true
3
+
4
+ module Saboteur
5
+ class CLI
6
+ # Opinionated bundles (#244): pr is the fast changed-lines loop (one
7
+ # baseline check, a 10-minute budget that fails an incomplete run),
8
+ # audit is the whole project with both reports. Preset flags go first,
9
+ # so an explicit flag after them wins on the usual last-flag rule.
10
+ module Presets
11
+ PRESETS = {
12
+ "pr" => %w[--baseline-passes 1 --max-duration 10m],
13
+ "audit" => %w[--all --operators all --baseline-passes 2 --json .saboteur/audit-report.json
14
+ --html .saboteur/audit-report.html]
15
+ }.freeze
16
+
17
+ private
18
+
19
+ def preset_or_usage_error(command)
20
+ PRESETS.key?(command) ? preset(command) : usage_error
21
+ end
22
+
23
+ def preset(name)
24
+ flags = PRESETS.fetch(name).dup
25
+ if name == "pr"
26
+ ref = @argv.first.nil? || @argv.first.start_with?("-") ? "main" : @argv.shift
27
+ flags += ["--since", ref]
28
+ end
29
+ # An explicit scope beats the injected whole-suite --all, which
30
+ # would otherwise reject the combination: named tests, and the
31
+ # deprecated inference alias (#245).
32
+ explicit_scope = @argv.any? { |arg| arg == "--test" || arg.start_with?("--test=") || arg == "--guess-tests" }
33
+ flags.delete("--all") if explicit_scope
34
+ @err.puts("saboteur: preset '#{name}': #{flags.join(' ')} (explicit flags override)")
35
+ @argv = flags + @argv
36
+ run_command
37
+ end
38
+ end
39
+ end
40
+ end
@@ -0,0 +1,37 @@
1
+ # SPDX-License-Identifier: MIT
2
+ # frozen_string_literal: true
3
+
4
+ module Saboteur
5
+ class CLI
6
+ # The run's clock and repeatability switches: seed, baseline check count
7
+ # (#55) and the wall-clock budget (#245). Grouped so the option table in
8
+ # the CLI stays under the house size caps.
9
+ module RunBudgetOptions
10
+ private
11
+
12
+ def add_run_budget_options(opts)
13
+ opts.on("--seed N", Integer, "Seed for Ruby's random numbers in every test run (default 1)") do |v|
14
+ set(:seed, v)
15
+ end
16
+ opts.on("--baseline-passes N", Integer, "Unchanged-suite checks (default 2)") { |v| set(:baseline_passes, v) }
17
+ opts.on("--max-duration DURATION", "Wall-clock budget for the whole run, e.g. 10m; exit 4 when spent") do |v|
18
+ set(:max_duration, parse_duration(v))
19
+ end
20
+ opts.on("--[no-]max-duration-ok", "An exhausted --max-duration budget does not fail the run") do |v|
21
+ set(:max_duration_ok, v)
22
+ end
23
+ end
24
+
25
+ # "10m" -> 600.0; bare numbers are seconds. A bad value is a usage error.
26
+ def parse_duration(value)
27
+ match = /\A(?<amount>\d+(?:\.\d+)?)(?<unit>[smh])?\z/.match(value.to_s.strip)
28
+ unless match
29
+ raise ConfigError,
30
+ "invalid --max-duration #{value.inspect}: use seconds or an s/m/h suffix, e.g. 10m"
31
+ end
32
+
33
+ match[:amount].to_f * { "s" => 1, "m" => 60, "h" => 3600, nil => 1 }.fetch(match[:unit])
34
+ end
35
+ end
36
+ end
37
+ end
@@ -10,9 +10,10 @@ module Saboteur
10
10
  module RunReporting
11
11
  private
12
12
 
13
- def text_run(report)
13
+ def text_run(report, engine)
14
14
  @err.puts
15
- Reporter::Terminal.new(@out, verbose: @verbose || false, color: @color || :auto).call(report)
15
+ Reporter::Terminal.new(@out, verbose: @verbose || false, color: @color || :auto,
16
+ replay_command: survivor_replay_command(report, engine)).call(report)
16
17
  Reporter::Replay.new(@out).call(report) if options[:id]
17
18
  code = exit_code(report)
18
19
  # The JSON carries the exit cause an outer harness trusts, so it is
@@ -50,7 +51,65 @@ module Saboteur
50
51
  validate_history
51
52
  arguments = { err: @err }
52
53
  arguments[:digest] = options[:id] if options[:id]
53
- agent? ? agent_run(arguments) : text_run(Engine.new(config, **arguments).call)
54
+ return agent_run(arguments) if agent?
55
+
56
+ # Text mode only: the engine prints a per-subject marks line when
57
+ # --verbose is on (#262); --agent output stays as it is.
58
+ engine = Engine.new(config, **arguments, verbose: @verbose || false)
59
+ text_run(engine.call, engine)
60
+ end
61
+
62
+ # The full replay command for a survivor digest: this run's own
63
+ # selection options, assembled by the same code that writes the agent
64
+ # follow-ups, so the user never reassembles them by hand (#261).
65
+ # Computed lazily: only a run with survivors asks for it.
66
+ def survivor_replay_command(report, engine)
67
+ lambda do |digest|
68
+ tree = WorkingTree.new(config.root, ignored: config.output_paths).state
69
+ Reporter::Agent::Commands.replay(agent_provenance(report, tree, engine), digest, capped: false)
70
+ end
71
+ end
72
+
73
+ # Writes the JSON report and returns its path, or nil when none is configured.
74
+ #
75
+ # An explicitly requested JSON report (--json, c.json_report) is the
76
+ # completion evidence an outer harness trusts: a failed write fails
77
+ # the run (exit 3), because a warning with exit 0 could leave a stale
78
+ # or partial file at the path and present old results as this run's.
79
+ # The default .saboteur/report.json is best-effort like the HTML
80
+ # report and the score history (#296).
81
+ def write_json(report, cause)
82
+ path = config.path("json_report")
83
+ return unless path
84
+
85
+ Reporter::Stryker.new(path).call(report, exit_cause: cause)
86
+ path
87
+ rescue SystemCallError, IOError => e
88
+ if config.explicit?("json_report")
89
+ raise Error, "could not write the JSON report to #{path}: #{e.message} - " \
90
+ "any file at that path may be stale or incomplete"
91
+ end
92
+
93
+ warn_report_write_failure("JSON report", path, e)
94
+ end
95
+
96
+ def write_html(report)
97
+ path = config.path("html_report")
98
+ return unless path
99
+
100
+ Reporter::Html.new(path).call(report)
101
+ path
102
+ rescue SystemCallError, IOError => e
103
+ warn_report_write_failure("HTML report", path, e)
104
+ end
105
+
106
+ # An auxiliary report artifact that cannot be written - the tree is
107
+ # read-only inside a sandboxed child, as in the dogfood's nested runs
108
+ # (#296) - is a warning, not a run failure: the exit code belongs to
109
+ # the results. The JSON report is not auxiliary; see write_json.
110
+ def warn_report_write_failure(what, path, error)
111
+ @err.puts "saboteur: could not write the #{what} to #{path} (#{error.message}); continuing without it"
112
+ nil
54
113
  end
55
114
 
56
115
  def validate_history
@@ -67,8 +126,18 @@ module Saboteur
67
126
 
68
127
  tree = WorkingTree.new(config.root, ignored: config.output_paths).state
69
128
  scope = config.selection_settings.except("json_report", "html_report", "score_history", "cache_dir")
70
- message = Reporter::ScoreHistory.new(path).append(report, head: tree.head, scope: scope)
71
- @err.puts("saboteur: #{message}")
129
+ # Recording is an auxiliary artifact: when the file cannot be
130
+ # written - read-only tree inside a sandboxed child (#296) - warn
131
+ # and let the run exit by its results instead of failing as a tool
132
+ # error. Only the write's IO failures are soft: validation,
133
+ # integrity, record-size and lock failures (Saboteur::Error) stay
134
+ # hard, because they say the history is unusable, not unwritable.
135
+ message = begin
136
+ Reporter::ScoreHistory.new(path).append(report, head: tree.head, scope: scope)
137
+ rescue SystemCallError, IOError => e
138
+ warn_report_write_failure("score history", path, e)
139
+ end
140
+ @err.puts("saboteur: #{message}") if message
72
141
  end
73
142
  end
74
143
  end
@@ -0,0 +1,31 @@
1
+ # SPDX-License-Identifier: MIT
2
+ # frozen_string_literal: true
3
+
4
+ module Saboteur
5
+ class CLI
6
+ # The run's subject-selection flags: operator sets, single-mutant replay,
7
+ # --since scoping, --dry-run, the flaky-check confirmation rerun (#476)
8
+ # and explicit --subject filters. Grouped so the CLI stays under the
9
+ # house size caps.
10
+ module SelectionOptions
11
+ private
12
+
13
+ def define_selection_options(opts)
14
+ opts.on("--operators NAMES", Array, "Choose operator names (default: all)") do |v|
15
+ set(:operators, v)
16
+ end
17
+ opts.on("--exclude-operators NAMES", Array, "Exclude operator names from the selected set") do |v|
18
+ set(:exclude_operators, v)
19
+ end
20
+ opts.on("--list-operators", "List operator names and descriptions") { set(:list_operators, true) }
21
+ opts.on("--dry-run", "Preview mutants without boot, tests, forks or report writes") { set(:dry_run, true) }
22
+ opts.on("--id DIGEST", "Replay exactly one current mutant by its full digest") { |v| set(:id, v) }
23
+ opts.on("--shard K/N", "Run one of N deterministic candidate shards (for example 1/4)") { |v| set(:shard, v) }
24
+ opts.on("--since REF", "Only mutate subjects changed since git REF") { |v| options[:since] = v }
25
+ flaky_desc = "Confirm a fresh verdict (survivors, decisive) with one reordered rerun"
26
+ opts.on("--flaky-check MODE", %w[off survivors decisive], flaky_desc) { |v| set(:flaky_check, v) }
27
+ opts.on("--subject EXPR", "Klass#meth, Klass.meth or a namespace; repeatable") { |v| add_subject(v) }
28
+ end
29
+ end
30
+ end
31
+ end
@@ -0,0 +1,17 @@
1
+ # SPDX-License-Identifier: MIT
2
+ # frozen_string_literal: true
3
+
4
+ module Saboteur
5
+ class CLI
6
+ # Validates the deterministic candidate-shard selection before the target boots.
7
+ module ShardOptions
8
+ private
9
+
10
+ def validate_shard_options
11
+ raise UsageError, "--id cannot be combined with --shard" if options[:id] && config["shard"]
12
+
13
+ Engine::Shard.parse(config["shard"]) if config["shard"]
14
+ end
15
+ end
16
+ end
17
+ end