saboteur 0.6.0 → 0.8.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (133) hide show
  1. checksums.yaml +4 -4
  2. data/CHANGELOG.md +1024 -0
  3. data/README.md +199 -952
  4. data/exe/saboteur +34 -7
  5. data/lib/saboteur/baseline/workflow.rb +2 -1
  6. data/lib/saboteur/baseline_health.rb +40 -6
  7. data/lib/saboteur/cli/exit_status.rb +80 -4
  8. data/lib/saboteur/cli/help_options.rb +17 -0
  9. data/lib/saboteur/cli/history_command.rb +135 -0
  10. data/lib/saboteur/cli/presets.rb +40 -0
  11. data/lib/saboteur/cli/run_budget_options.rb +37 -0
  12. data/lib/saboteur/cli/run_reporting.rb +74 -5
  13. data/lib/saboteur/cli/selection_options.rb +31 -0
  14. data/lib/saboteur/cli/shard_options.rb +17 -0
  15. data/lib/saboteur/cli/source_set.rb +53 -0
  16. data/lib/saboteur/cli/survivors_command.rb +157 -0
  17. data/lib/saboteur/cli.rb +136 -98
  18. data/lib/saboteur/config.rb +163 -15
  19. data/lib/saboteur/config_file.rb +8 -0
  20. data/lib/saboteur/constant_reads.rb +182 -0
  21. data/lib/saboteur/constant_subject.rb +70 -0
  22. data/lib/saboteur/constant_subject_finder.rb +281 -0
  23. data/lib/saboteur/coverage_reads.rb +29 -0
  24. data/lib/saboteur/crash_report.rb +154 -0
  25. data/lib/saboteur/custom_map.rb +3 -1
  26. data/lib/saboteur/dirty_tree_warning.rb +128 -0
  27. data/lib/saboteur/doctor.rb +49 -1
  28. data/lib/saboteur/engine/baseline_guard.rb +136 -14
  29. data/lib/saboteur/engine/boot.rb +16 -1
  30. data/lib/saboteur/engine/canary.rb +41 -10
  31. data/lib/saboteur/engine/candidate_selection.rb +28 -1
  32. data/lib/saboteur/engine/candidates.rb +43 -3
  33. data/lib/saboteur/engine/coverage_guard.rb +29 -0
  34. data/lib/saboteur/engine/coverage_store.rb +79 -15
  35. data/lib/saboteur/engine/defined_order_prefix.rb +43 -0
  36. data/lib/saboteur/engine/diagnostics.rb +41 -2
  37. data/lib/saboteur/engine/diff_scope.rb +24 -5
  38. data/lib/saboteur/engine/distribution.rb +134 -8
  39. data/lib/saboteur/engine/early_stop_recording.rb +47 -0
  40. data/lib/saboteur/engine/gated_run.rb +35 -0
  41. data/lib/saboteur/engine/load_coverage.rb +45 -0
  42. data/lib/saboteur/engine/out_of_scope_guard.rb +21 -0
  43. data/lib/saboteur/engine/pipelines.rb +107 -0
  44. data/lib/saboteur/engine/result_cache.rb +67 -9
  45. data/lib/saboteur/engine/run_budget.rb +90 -0
  46. data/lib/saboteur/engine/run_fingerprint.rb +4 -0
  47. data/lib/saboteur/engine/sandbox_policy.rb +56 -11
  48. data/lib/saboteur/engine/scheduler.rb +222 -20
  49. data/lib/saboteur/engine/selection_announcement.rb +54 -0
  50. data/lib/saboteur/engine/shard.rb +69 -0
  51. data/lib/saboteur/engine/subject_selection.rb +373 -19
  52. data/lib/saboteur/engine/suite_files.rb +3 -1
  53. data/lib/saboteur/engine/test_guess.rb +438 -25
  54. data/lib/saboteur/engine/thread_warnings.rb +53 -0
  55. data/lib/saboteur/engine/unchanged_syntax.rb +17 -2
  56. data/lib/saboteur/engine/unscoped_suite_warning.rb +41 -0
  57. data/lib/saboteur/engine/verdict_check.rb +54 -18
  58. data/lib/saboteur/engine.rb +202 -158
  59. data/lib/saboteur/execution/external_runner.rb +471 -0
  60. data/lib/saboteur/execution/marker_sweep.rb +79 -0
  61. data/lib/saboteur/execution/peak_memory.rb +28 -0
  62. data/lib/saboteur/execution/process_table.rb +12 -0
  63. data/lib/saboteur/execution/process_tree.rb +65 -5
  64. data/lib/saboteur/execution/protocol.rb +2 -1
  65. data/lib/saboteur/execution/reevaluator.rb +68 -12
  66. data/lib/saboteur/execution/runner.rb +225 -24
  67. data/lib/saboteur/execution/sandbox.rb +21 -3
  68. data/lib/saboteur/execution/seatbelt.rb +226 -0
  69. data/lib/saboteur/execution/supervisor.rb +45 -13
  70. data/lib/saboteur/execution/worker_pool.rb +110 -8
  71. data/lib/saboteur/frameworks/sqlite_memory_database.rb +49 -0
  72. data/lib/saboteur/gates.rb +43 -2
  73. data/lib/saboteur/help.rb +108 -13
  74. data/lib/saboteur/ignore_list.rb +40 -7
  75. data/lib/saboteur/install.rb +4 -3
  76. data/lib/saboteur/integration/base.rb +7 -2
  77. data/lib/saboteur/integration/rspec.rb +175 -9
  78. data/lib/saboteur/mutation_source/argument_removal.rb +1 -1
  79. data/lib/saboteur/mutation_source/builtin.rb +44 -14
  80. data/lib/saboteur/mutation_source/chain_removal.rb +1 -1
  81. data/lib/saboteur/mutation_source/condition_negation.rb +1 -1
  82. data/lib/saboteur/mutation_source/constant_literals.rb +66 -0
  83. data/lib/saboteur/mutation_source/element_removal.rb +73 -0
  84. data/lib/saboteur/mutation_source/endpoint_replacement.rb +1 -1
  85. data/lib/saboteur/mutation_source/entry_edits.rb +44 -0
  86. data/lib/saboteur/mutation_source/filter_replacement.rb +1 -1
  87. data/lib/saboteur/mutation_source/hazard_guard.rb +129 -3
  88. data/lib/saboteur/mutation_source/interpolation_removal.rb +41 -0
  89. data/lib/saboteur/mutation_source/literal_replacement.rb +1 -1
  90. data/lib/saboteur/mutation_source/method_swap.rb +193 -0
  91. data/lib/saboteur/mutation_source/mutation.rb +5 -3
  92. data/lib/saboteur/mutation_source/operand_replacement.rb +54 -0
  93. data/lib/saboteur/mutation_source/operator_assignment.rb +1 -1
  94. data/lib/saboteur/mutation_source/operator_examples.rb +6 -1
  95. data/lib/saboteur/mutation_source/operator_registry.rb +7 -2
  96. data/lib/saboteur/mutation_source/operator_visit.rb +39 -0
  97. data/lib/saboteur/mutation_source/pair_removal.rb +13 -31
  98. data/lib/saboteur/mutation_source/range_replacement.rb +1 -1
  99. data/lib/saboteur/mutation_source/regex_replacement.rb +93 -0
  100. data/lib/saboteur/preview.rb +122 -13
  101. data/lib/saboteur/refinement_activations.rb +60 -0
  102. data/lib/saboteur/reporter/agent.rb +9 -1
  103. data/lib/saboteur/reporter/agent_commands.rb +47 -15
  104. data/lib/saboteur/reporter/agent_incomplete.rb +38 -2
  105. data/lib/saboteur/reporter/agent_summary.rb +4 -3
  106. data/lib/saboteur/reporter/findings.rb +5 -2
  107. data/lib/saboteur/reporter/history_policy.rb +11 -0
  108. data/lib/saboteur/reporter/history_reader.rb +47 -0
  109. data/lib/saboteur/reporter/html.rb +26 -4
  110. data/lib/saboteur/reporter/mutation_text.rb +6 -0
  111. data/lib/saboteur/reporter/next_steps.rb +14 -4
  112. data/lib/saboteur/reporter/score_history.rb +5 -1
  113. data/lib/saboteur/reporter/stryker.rb +40 -7
  114. data/lib/saboteur/reporter/survivor_classification.rb +101 -0
  115. data/lib/saboteur/reporter/survivor_hints.rb +3 -0
  116. data/lib/saboteur/reporter/survivor_list.rb +15 -9
  117. data/lib/saboteur/reporter/terminal.rb +74 -17
  118. data/lib/saboteur/result.rb +38 -7
  119. data/lib/saboteur/selection/coverage_map.rb +107 -11
  120. data/lib/saboteur/selection/dependency_shield.rb +80 -0
  121. data/lib/saboteur/selection/external_test_map.rb +27 -0
  122. data/lib/saboteur/selection/git_diff.rb +105 -13
  123. data/lib/saboteur/selection/layout_only_lines.rb +80 -0
  124. data/lib/saboteur/selection/recorder.rb +22 -4
  125. data/lib/saboteur/selection/source_scope.rb +22 -0
  126. data/lib/saboteur/selection/untracked_changes.rb +126 -0
  127. data/lib/saboteur/subject.rb +53 -7
  128. data/lib/saboteur/subject_finder.rb +173 -29
  129. data/lib/saboteur/text.rb +42 -0
  130. data/lib/saboteur/version.rb +1 -1
  131. data/lib/saboteur/working_tree.rb +38 -15
  132. data/lib/saboteur.rb +29 -1
  133. metadata +50 -2
@@ -0,0 +1,53 @@
1
+ # SPDX-License-Identifier: MIT
2
+ # frozen_string_literal: true
3
+
4
+ module Saboteur
5
+ class CLI
6
+ # The effective source set a run will mutate, refused when it is empty.
7
+ module SourceSet
8
+ private
9
+
10
+ # The Rails-shaped footgun (#447): the default include targets plain
11
+ # gems, so a first run on an app/ tree selects nothing and would exit 0
12
+ # over 0 subjects - a green gate that mutates nothing, every CI run.
13
+ # Refuse like a typo'd source: exit 2, name the stale pattern and the
14
+ # way out. --dry-run still previews: it gates nothing.
15
+ def refuse_empty_source_set
16
+ return unless config.included_files.empty?
17
+
18
+ raise UsageError, empty_source_set_message
19
+ end
20
+
21
+ # A --since ref whose resolved range diffs empty yields 0 subjects
22
+ # and a green run over nothing: `saboteur pr <merge_sha>` on a
23
+ # checkout AT the merge diffs the merge against itself (#643).
24
+ # Refuse like a typo'd ref: exit 2, name the base and the range.
25
+ def refuse_empty_since_range
26
+ ref = config["since"]
27
+ return unless ref
28
+
29
+ paths = Selection::GitDiff.changed_paths(ref, root: config.root)
30
+ return unless paths.values.all?(&:empty?)
31
+
32
+ base = Selection::GitDiff.merge_base(ref, root: config.root)
33
+ raise UsageError, "--since #{ref}: the resolved range #{base}..the working tree has no changes - " \
34
+ "the run would pass with 0 subjects. A checkout AT the ref (the CI merge-sha " \
35
+ "checkout) diffs it against itself: fetch the merge's parent, or point --since " \
36
+ "at the base of the change under test"
37
+ end
38
+
39
+ def empty_source_set_message
40
+ include_patterns = config["include"]
41
+ matched = include_patterns.flat_map { |pattern| Dir.glob(pattern, base: config.root) }
42
+ .select { |file| file.end_with?(".rb") }
43
+ cause = if matched.empty?
44
+ "include: #{include_patterns.join(', ')} selects no Ruby file"
45
+ else
46
+ "exclude: #{config['exclude'].join(', ')} removes every included file"
47
+ end
48
+ "#{cause} - the run would pass with 0 subjects. Point include at your code " \
49
+ "(Rails apps: bundle exec rails generate saboteur:install writes the config) or fix the patterns"
50
+ end
51
+ end
52
+ end
53
+ end
@@ -0,0 +1,157 @@
1
+ # SPDX-License-Identifier: MIT
2
+ # frozen_string_literal: true
3
+
4
+ require "json"
5
+
6
+ module Saboteur
7
+ class CLI
8
+ # saboteur survivors <report.json> (#475): a read-only, line-oriented
9
+ # terminal browser for the Stryker JSON report. It never boots the
10
+ # project or reruns tests; it lists survivors, filters them, and for
11
+ # one digest prints the diff, the exact replay command and the
12
+ # ignore-file template. It never edits the ignore file.
13
+ module SurvivorsCommand
14
+ private
15
+
16
+ def survivors_command
17
+ survivors_command_inner
18
+ rescue UsageError => e
19
+ @err.puts("saboteur: #{e.message}")
20
+ EXIT_USAGE
21
+ end
22
+
23
+ def survivors_command_inner
24
+ parser = OptionParser.new do |opts|
25
+ opts.on("--ranked", "Review-priority order: the report's ranked findings first")
26
+ opts.on("--file SUBSTR", "Only survivors whose path contains SUBSTR")
27
+ opts.on("--operator NAME", "Only survivors of operator NAME")
28
+ opts.on("--triage LABEL", "Only survivors classified LABEL (for example missing_test)")
29
+ opts.on("--show DIGEST", "Diff, replay command and ignore template for one survivor")
30
+ end
31
+ options = {}
32
+ args = parser.parse!(@argv, into: options)
33
+ path = args.shift or raise UsageError,
34
+ "survivors expects a report path: saboteur survivors .saboteur/report.json"
35
+ raise UsageError, "unexpected arguments: #{args.join(' ')}" if args.any?
36
+
37
+ report = load_survivor_report(path)
38
+ browser = SurvivorsBrowser.new(report)
39
+ if options[:show]
40
+ @out.puts(browser.show(options[:show]))
41
+ else
42
+ @out.puts(browser.list(**options.slice(:ranked, :file, :operator, :triage)))
43
+ end
44
+ EXIT_OK
45
+ end
46
+
47
+ def load_survivor_report(path)
48
+ raise UsageError, "report not found: #{path}" unless File.file?(path)
49
+
50
+ JSON.parse(File.read(path))
51
+ rescue JSON::ParserError => e
52
+ raise UsageError, "#{path} is not a readable saboteur report: #{e.message}"
53
+ end
54
+ end
55
+
56
+ # Reads the report's files section (the complete mutant list), not the
57
+ # capped findings, so survivors beyond the findings limit still list.
58
+ class SurvivorsBrowser
59
+ def initialize(report)
60
+ @report = report
61
+ @findings = report.dig("saboteur", "findings") || {}
62
+ end
63
+
64
+ def list(ranked: false, file: nil, operator: nil, triage: nil)
65
+ survivors = filtered(file, operator, triage)
66
+ survivors = ranked_order(survivors) if ranked
67
+ return "no survivors#{filters_note(file, operator, triage)}" if survivors.empty?
68
+
69
+ [header(survivors.size, file, operator, triage), *rows(survivors)]
70
+ end
71
+
72
+ def show(digest)
73
+ survivor = all_survivors.find { |s| s[:digest].start_with?(digest) } ||
74
+ raise(UsageError, "no survivor with digest #{digest} in this report")
75
+ finding = findings_by_digest[survivor[:digest]] || {}
76
+ subject = finding["subject"]
77
+ [
78
+ "#{survivor[:path]}:#{survivor[:line]} #{survivor[:operator]} #{survivor[:digest]}",
79
+ "",
80
+ "- #{finding['original'] || original_line(survivor)}",
81
+ "+ #{finding['replacement'] || survivor[:replacement]}",
82
+ "",
83
+ "Replay exactly this mutant:",
84
+ " bundle exec saboteur run --id #{survivor[:digest]}",
85
+ "",
86
+ "Ignore-file entry (config/saboteur_ignore.rb):",
87
+ *ignore_entry(survivor, subject)
88
+ ].join("\n")
89
+ end
90
+
91
+ private
92
+
93
+ def filtered(file, operator, triage)
94
+ survivors = all_survivors
95
+ survivors = survivors.select { |s| s[:path].include?(file) } if file
96
+ survivors = survivors.select { |s| s[:operator] == operator } if operator
97
+ triage ? survivors.select { |s| s[:classification] == triage } : survivors
98
+ end
99
+
100
+ def header(size, file, operator, triage)
101
+ text = "#{size} survivor#{'s' unless size == 1}#{filters_note(file, operator, triage)}"
102
+ truncated = @findings["survivorsTruncated"].to_i
103
+ return text unless truncated.positive?
104
+
105
+ "#{text} (the report's ranked findings name only the first #{@findings['limit'] || 10})"
106
+ end
107
+
108
+ def rows(survivors)
109
+ survivors.each_with_index.map do |s, i|
110
+ format("%3<i>d. %<path>s:%<line>d %-28<operator>s %<digest>s %<classification>s",
111
+ i: i + 1, path: s[:path], line: s[:line], operator: s[:operator], digest: s[:digest],
112
+ classification: s[:classification] || "-")
113
+ end
114
+ end
115
+
116
+ def ignore_entry(survivor, subject)
117
+ note = subject ? subject.inspect : '"Klass#meth" # TODO: capped findings omit this digest - fill it in'
118
+ [%( i.mutation "#{survivor[:digest]}",), " subject: #{note},", %( reason: "...")]
119
+ end
120
+
121
+ def all_survivors
122
+ @all_survivors ||= begin
123
+ survivors = @report.fetch("files", {}).flat_map do |path, entry|
124
+ entry.fetch("mutants", []).select { |m| m["status"] == "Survived" }.map do |m|
125
+ { path: path, line: m.dig("location", "start", "line").to_i, operator: m["mutatorName"],
126
+ digest: m["id"], replacement: m["replacement"], classification: m["classification"],
127
+ covered: m["coveredBy"] || [], source: entry["source"] }
128
+ end
129
+ end
130
+ survivors.sort_by { |s| [s[:path], s[:line], s[:digest]] }
131
+ end
132
+ end
133
+
134
+ def ranked_order(survivors)
135
+ rank = @findings.fetch("survivors", []).each_with_index.to_h { |f, i| [f["digest"], i] }
136
+ survivors.sort_by { |s| [rank[s[:digest]] || Float::INFINITY, -s[:covered].size, s[:path], s[:line], s[:digest]] }
137
+ end
138
+
139
+ def findings_by_digest
140
+ @findings_by_digest ||=
141
+ (@findings.fetch("survivors", []) + @findings.fetch("accepted", [])).to_h { |f| [f["digest"], f] }
142
+ end
143
+
144
+ def original_line(survivor)
145
+ survivor[:source].to_s.lines[survivor[:line] - 1].to_s.strip
146
+ end
147
+
148
+ def filters_note(file, operator, triage)
149
+ parts = []
150
+ parts << "file~#{file}" if file
151
+ parts << "operator=#{operator}" if operator
152
+ parts << "triage=#{triage}" if triage
153
+ parts.empty? ? "" : " (#{parts.join(', ')})"
154
+ end
155
+ end
156
+ end
157
+ end
data/lib/saboteur/cli.rb CHANGED
@@ -3,6 +3,7 @@
3
3
 
4
4
  require "optparse"
5
5
  require_relative "../saboteur"
6
+ require_relative "crash_report"
6
7
  require_relative "doctor"
7
8
  require_relative "help"
8
9
  require_relative "reporter/replay"
@@ -10,14 +11,30 @@ require_relative "baseline/command"
10
11
  require_relative "cli/exit_status"
11
12
  require_relative "preview"
12
13
  require_relative "cli/run_reporting"
14
+ require_relative "cli/survivors_command"
15
+ require_relative "cli/source_set"
16
+ require_relative "cli/run_budget_options"
17
+ require_relative "cli/shard_options"
18
+ require_relative "cli/selection_options"
19
+ require_relative "cli/presets"
20
+ require_relative "cli/help_options"
21
+ require_relative "cli/history_command"
13
22
 
14
23
  module Saboteur
15
24
  class CLI
16
25
  include Baseline::Command
17
26
  include ExitStatus
18
27
  include RunReporting
19
-
20
- USAGE = "Usage: saboteur run [SOURCE...] [--test TEST...] [--threshold PERCENT] [--since REF] " \
28
+ include SurvivorsCommand
29
+ include SourceSet
30
+ include RunBudgetOptions
31
+ include ShardOptions
32
+ include SelectionOptions
33
+ include Presets
34
+ include HistoryCommand
35
+ include HelpOptions
36
+
37
+ USAGE = "Usage: saboteur run [SOURCE...] [--test TEST...] [--all] [--threshold PERCENT] [--since REF] " \
21
38
  "[--subject 'Klass#meth'] [--timeout-factor N] [--jobs N] [--seed N] [--id DIGEST]
22
39
  " \
23
40
  "Run 'saboteur help' for every command, option, config key and exit code."
@@ -26,10 +43,18 @@ module Saboteur
26
43
  EXIT_SURVIVORS = 1
27
44
  EXIT_USAGE = 2
28
45
  EXIT_TOOL = 3
46
+ EXIT_BUDGET = 4
29
47
  class UsageError < ConfigError; end
30
48
 
31
49
  attr_reader :options
32
50
 
51
+ # Found before parsing, so any error, even a bad option that comes first, still
52
+ # answers an agent with JSON. Class-level: the executable's fallback reads raw arguments.
53
+ def self.agent_requested?(argv)
54
+ argv.include?("--agent") || argv.include?("--format=agent") ||
55
+ argv.each_cons(2).any? { |flag, value| flag == "--format" && value == "agent" }
56
+ end
57
+
33
58
  def initialize(argv, out: $stdout, err: $stderr)
34
59
  @argv = argv.dup
35
60
  @out = out
@@ -39,18 +64,21 @@ module Saboteur
39
64
  end
40
65
 
41
66
  def call
42
- case @argv.shift
67
+ command = @argv.shift
68
+ case command
43
69
  when "run" then run_command
44
70
  when "baseline" then baseline
45
- when "doctor" then help_requested? ? show_help("doctor") : doctor
71
+ when "survivors" then survivors_command
72
+ when "doctor" then doctor_command
73
+ when "badge", "trend" then history_entry(command)
46
74
  when "version", "--version" then @out.puts(VERSION) || EXIT_OK
47
- when "help" then show_help(@argv.shift)
75
+ when "help", "--help", "-h" then show_help(@argv.shift)
48
76
  when "agent-skill" then @out.print(AgentSkill::TEXT) || EXIT_OK
49
- when "--help", "-h" then show_help(nil)
50
- else usage_error
77
+ else preset_or_usage_error(command)
51
78
  end
52
79
  rescue OptionParser::ParseError => e
53
80
  usage_error(e.message)
81
+ rescue StandardError => e then internal_error(e)
54
82
  end
55
83
 
56
84
  def config
@@ -68,11 +96,6 @@ module Saboteur
68
96
  end
69
97
  end
70
98
 
71
- # Each command's own option lines, as printed under OPTIONS in `saboteur help`.
72
- def option_lines
73
- { "run" => parser.summarize, "doctor" => doctor_parser.summarize, "baseline" => baseline_parser.summarize }
74
- end
75
-
76
99
  def doctor_parser
77
100
  OptionParser.new do |opts|
78
101
  opts.banner = "Usage: saboteur doctor [--config PATH] [--seed N] [--suite] [--test PATH]"
@@ -87,13 +110,13 @@ module Saboteur
87
110
 
88
111
  private
89
112
 
90
- def run_command
91
- help_requested? ? show_help("run") : run
92
- end
113
+ def run_command = help_requested? ? show_help("run") : run
93
114
 
94
- def help_requested?
95
- @argv.intersect?(%w[--help -h])
96
- end
115
+ def doctor_command = help_requested? ? show_help("doctor") : doctor
116
+
117
+ def history_entry(command) = help_requested? ? show_help(command) : history_command(command)
118
+
119
+ def help_requested? = @argv.intersect?(%w[--help -h])
97
120
 
98
121
  def show_help(command_name)
99
122
  help = Help.new(option_lines: option_lines)
@@ -110,16 +133,9 @@ module Saboteur
110
133
  agent_failure(message || USAGE, EXIT_USAGE)
111
134
  end
112
135
 
113
- def agent?
114
- @format == "agent"
115
- end
136
+ def agent? = @format == "agent"
116
137
 
117
- # Found before parsing, so any error, even a bad option that comes first, still
118
- # answers an agent with JSON.
119
- def agent_requested?
120
- @argv.include?("--agent") || @argv.include?("--format=agent") ||
121
- @argv.each_cons(2).any? { |flag, value| flag == "--format" && value == "agent" }
122
- end
138
+ def agent_requested? = self.class.agent_requested?(@argv)
123
139
 
124
140
  # In agent mode the failure is also on stdout, where the agent reads its answer.
125
141
  def agent_failure(message, exit_code)
@@ -129,12 +145,17 @@ module Saboteur
129
145
 
130
146
  def run
131
147
  parser.parse!(@argv)
148
+ reject_test_selection_conflicts
132
149
  use_explicit_sources
133
150
  validate_run_options
134
151
  warn_dirty_tree
135
152
 
153
+ config.deprecation_warnings.each { |warning| @err.puts("saboteur: #{warning}") }
154
+
136
155
  return preview if options[:list_operators] || options[:dry_run]
137
156
 
157
+ refuse_empty_source_set
158
+ refuse_empty_since_range
138
159
  execute_run
139
160
  rescue ConfigError => e
140
161
  @err.puts("saboteur: #{e.message}")
@@ -146,28 +167,60 @@ module Saboteur
146
167
 
147
168
  # Mutants run your code, and a mutant of code that writes or deletes files
148
169
  # can change the tree. Say so up front when uncommitted work is at stake.
170
+ # The scope the warning names is the configured one: positional source
171
+ # operands narrow the run after the warning, so the include/exclude they
172
+ # imply stay out of its config (#309, #503 review).
149
173
  def warn_dirty_tree
150
- WorkingTree.new(config.root, ignored: config.output_paths).state.warning&.then { |text| @err.puts(text) }
174
+ warning = @sources ? Config.load(path: options[:config], overrides: overrides.except(:include, :exclude)) : config
175
+ DirtyTreeWarning.for(warning, config_path: options[:config])&.then { |text| @err.puts(text) }
176
+ end
177
+
178
+ # Contradictory selection flags are a usage error before any file or
179
+ # repository check: --test wins over inference by precedence, but only
180
+ # one of them may be named. Config-file combinations are not errors:
181
+ # --test wins there, so generated commands stay valid over any config.
182
+ def reject_test_selection_conflicts
183
+ raise UsageError, "--all cannot be combined with --test" if options[:all] && options[:tests]&.any?
184
+
185
+ reject_test_command_conflicts
186
+ raise UsageError, "--all cannot be combined with --guess-tests" if options[:all] && options[:guess_tests]
187
+ return unless options[:guess_tests] && options[:tests]&.any?
188
+
189
+ raise UsageError, "--guess-tests cannot be combined with --test"
190
+ end
191
+
192
+ # --test-command replaces the whole in-process test machinery: test
193
+ # selection, coverage, parallelism and the sandbox do not apply.
194
+ def reject_test_command_conflicts
195
+ return unless options[:test_command]
196
+
197
+ # options[:tests] is nil or a non-empty list, so truthiness is the check.
198
+ raise UsageError, "--test-command cannot be combined with --test" if options[:tests]
199
+ raise UsageError, "--test-command cannot be combined with --incremental" if options[:incremental]
151
200
  end
152
201
 
153
202
  # Usage errors found before the target boots.
154
203
  def validate_run_options
155
- reject_threshold_with_agent_format
204
+ reject_format_conflicts
205
+ validate_shard_options
156
206
  Selection::GitDiff.verify_ref!(config["since"], root: config.root) if config["since"]
157
207
  end
158
208
 
159
- # Agent mode has one strict meaning for exit 0, so a score threshold cannot also decide it.
160
- def reject_threshold_with_agent_format
161
- return unless agent? && (config["threshold"] || gates.any?)
209
+ # Agent mode allows no score gates (one strict exit 0); the json inventory is --dry-run planning only.
210
+ def reject_format_conflicts
211
+ if agent? && (config["threshold"] || gates.any?)
212
+ raise UsageError, "#{config['threshold'] ? '--threshold' : gates.agent_option_name} " \
213
+ "cannot be combined with --agent"
214
+ end
215
+ return unless @format == "json" && !options[:dry_run]
162
216
 
163
- raise UsageError, "#{config['threshold'] ? '--threshold' : '--max-* gates'} cannot be combined with --agent"
217
+ raise UsageError, "--format json is the --dry-run mutant inventory; a run summary is --format agent"
164
218
  end
165
219
 
166
220
  # One line naming the reports written, paths relative to the root when
167
221
  # they sit under it (a blank line before it, as the old JSON line had).
168
222
  def announce_reports(reports)
169
- written = reports.select { |path, _label| path }
170
- return if written.empty?
223
+ return if (written = reports.select { |path, _label| path }).empty?
171
224
 
172
225
  root = File.join(File.realpath(config.root), "")
173
226
  names = written.map do |path, label|
@@ -209,16 +262,38 @@ module Saboteur
209
262
  Reporter::Agent::Provenance.new(
210
263
  ref: config["since"], seed: report.seed, head: tree.head, working_tree: tree.label,
211
264
  changed_files: tree.changed_files, mode: agent_mode, json_report: config.path("json_report"),
212
- subjects: Array(config["subjects"]), tests: Array(engine.guessed_tests || options[:tests]),
265
+ subjects: Array(config["subjects"]), tests: Array(engine.guessed_tests || config["tests"]),
213
266
  sources: @sources || [],
214
267
  config_path: options[:config], digest: options[:id], cache: config["cache"],
215
- incremental: config["incremental"], settings: agent_settings
268
+ incremental: config["incremental"], settings: agent_settings, typed_settings: typed_settings
216
269
  )
217
270
  end
218
271
 
272
+ # The settings keys typed on the command line: a typed empty operator
273
+ # list may clear a config value, so the generated commands keep it.
274
+ def typed_settings = options.keys & Reporter::Agent::Commands::SETTING_FLAGS.keys
275
+
276
+ # A whole-suite run records its scope as --all, so the generated rerun
277
+ # and replay commands do not default back to inference (which can exit
278
+ # 2 in a project with a spec-less source file).
219
279
  def agent_settings
220
- options.slice(*Reporter::Agent::Commands::SETTING_FLAGS.keys)
221
- .merge(operators: config["operators"], exclude_operators: config["exclude_operators"])
280
+ settings = options.slice(*Reporter::Agent::Commands::SETTING_FLAGS.keys)
281
+ .merge(operators: config["operators"], exclude_operators: config["exclude_operators"])
282
+ # The effective mode: named tests win over a configured whole-suite
283
+ # selection by precedence, so all is recorded only when no tests
284
+ # resolved - otherwise the follow-up commands would carry both --all
285
+ # and --test, which the CLI rejects.
286
+ settings[:all] = true if (config["all"] || config["test_selection"] == "all") && config["tests"].empty?
287
+ # CLI-only switches that change candidate availability or boot, so a
288
+ # printed rerun/replay must keep them (config-file settings like
289
+ # arid/hazard patterns, whole-file, require, rspec_args and rails ride
290
+ # along through --config or the default config path).
291
+ settings[:allow_hazardous] = true if options[:allow_hazardous]
292
+ settings[:allow_focus] = true if options[:allow_focus]
293
+ # An explicit false (--no-max-duration-ok over a config true) is a
294
+ # setting too: dropping it would silently re-enable the config value.
295
+ settings[:max_duration_ok] = options[:max_duration_ok] unless options[:max_duration_ok].nil?
296
+ settings
222
297
  end
223
298
 
224
299
  def agent_mode
@@ -228,40 +303,19 @@ module Saboteur
228
303
  @sources&.any? ? "files" : "full"
229
304
  end
230
305
 
231
- # Writes the JSON report and returns its path, or nil when none is configured.
232
- def write_json(report, cause)
233
- path = config.path("json_report")
234
- return unless path
235
-
236
- Reporter::Stryker.new(path).call(report, exit_cause: cause)
237
- path
238
- end
239
-
240
- def write_html(report)
241
- path = config.path("html_report")
242
- return unless path
243
-
244
- Reporter::Html.new(path).call(report)
245
- path
246
- end
247
-
248
306
  # Positional arguments name the files (or directories) to mutate. They
249
307
  # replace the configured include and exclude patterns. Each must exist,
250
308
  # so a typo is an error and not a run over zero subjects.
251
309
  def use_explicit_sources
252
- if @argv.empty?
253
- raise UsageError, "--test needs at least one source file" if options[:tests] && !config["since"]
254
-
255
- # With --since the changed files are the sources, so --test may
256
- # stand alone; the files must still exist.
257
- options[:tests]&.each { |test| require_existing_test(test) }
258
- return
259
- end
310
+ Selection::SourceScope.verify_no_specs!(@argv)
311
+ options[:tests]&.each { |test| require_existing_test(test) }
312
+ # --test may stand alone: the configured include (or the --since
313
+ # diff) names the sources. The files must still exist.
314
+ return if @argv.empty?
260
315
 
261
316
  @sources = @argv.dup
262
317
  options[:include] = @argv.map { |source| source_pattern(source) }
263
318
  options[:exclude] = []
264
- options[:tests]&.each { |test| require_existing_test(test) }
265
319
  end
266
320
 
267
321
  def source_pattern(source)
@@ -283,12 +337,10 @@ module Saboteur
283
337
  raise UsageError, "test file not found: #{test}" unless File.exist?(path)
284
338
  end
285
339
 
286
- def overrides
287
- options.except(:config, :id, :suite, :list_operators, :dry_run)
288
- end
340
+ def overrides = options.except(:config, :id, :suite, :list_operators, :dry_run)
289
341
 
290
342
  def preview
291
- raise UsageError, "--dry-run and --list-operators print text, not --agent JSON" if agent?
343
+ raise UsageError, "--dry-run and --list-operators print text, not --agent JSON; use --format json" if agent?
292
344
  raise UsageError, "--id replays one mutant: not with --dry-run or --list-operators" if options[:id]
293
345
 
294
346
  if options[:list_operators]
@@ -296,30 +348,14 @@ module Saboteur
296
348
  return EXIT_OK
297
349
  end
298
350
 
299
- Preview.new(config, out: @out, err: @err).call
351
+ Preview.new(config, out: @out, err: @err, format: @format).call
300
352
  EXIT_OK
301
353
  end
302
354
 
303
- def define_selection_options(opts)
304
- opts.on("--operators NAMES", Array, "Choose operator names (default: all)") do |v|
305
- set(:operators, v)
306
- end
307
- opts.on("--exclude-operators NAMES", Array, "Exclude operator names from the selected set") do |v|
308
- set(:exclude_operators, v)
309
- end
310
- opts.on("--list-operators", "List operator names and descriptions") { set(:list_operators, true) }
311
- opts.on("--dry-run", "Preview mutants without boot, tests, forks or report writes") { set(:dry_run, true) }
312
- opts.on("--id DIGEST", "Replay exactly one current mutant by its full digest") { |v| set(:id, v) }
313
- opts.on("--since REF", "Only mutate subjects changed since git REF") { |v| options[:since] = v }
314
- opts.on("--subject EXPR", "Klass#meth, Klass.meth or a namespace; repeatable") { |v| add_subject(v) }
315
- end
316
-
317
355
  def define_execution_options(opts)
318
356
  opts.on("--timeout-factor N", Float, "Multiple of baseline runtime (default 3)") { |v| set(:timeout_factor, v) }
319
357
  opts.on("--jobs N", Integer, "Concurrent workers (default 1)") { |v| set(:jobs, v) }
320
- opts.on("--seed N", Integer, "Seed for Ruby's random numbers in every test run (default 1)") do |v|
321
- set(:seed, v)
322
- end
358
+ add_run_budget_options(opts)
323
359
  opts.on("--config PATH", "Config file (default config/saboteur.rb)") { |v| set(:config, v) }
324
360
  opts.on("--integration NAME", "Test adapter (rspec)") { |v| set(:integration, v) }
325
361
  opts.on("--reeval STRATEGY", Execution::Reevaluator::STRATEGIES, "class_eval or nesting") { |v| set(:reeval, v) }
@@ -332,8 +368,14 @@ module Saboteur
332
368
  opts.on("--allow-focus", "Run even when a focus filter (fit, fdescribe, :focus) narrows the suite") do
333
369
  set(:allow_focus, true)
334
370
  end
335
- opts.on("--[no-]guess-tests", "With --since: run only the specs named after the changed files " \
336
- "(lib/a.rb -> spec/a_spec.rb); the whole suite if any has none") do |v|
371
+ opts.on("--all", "Run the whole configured test suite instead of inferring the specs named after the " \
372
+ "mutated files") { set(:all, true) }
373
+ opts.on("--test-command CMD", "Run CMD per mutant in the project root; exit 0 means the mutant survived. " \
374
+ "For apps that cannot load in this Ruby; much slower, serial, no coverage") do |v|
375
+ set(:test_command, v)
376
+ end
377
+ opts.on("--[no-]guess-tests", "Deprecated alias: inferred specs are already the default; " \
378
+ "--no-guess-tests warns and is ignored - use --all for the whole suite") do |v|
337
379
  set(:guess_tests, v)
338
380
  end
339
381
  end
@@ -348,10 +390,10 @@ module Saboteur
348
390
  opts.on("--score-history PATH", "Append score history JSONL (opt-in)") { |v| set(:score_history, v) }
349
391
  opts.on("--json PATH", "Stryker JSON report path") { |v| set(:json_report, v) }
350
392
  opts.on("--html PATH", "Self-contained HTML report path") { |v| set(:html_report, v) }
351
- opts.on("--format FORMAT", %w[text agent], "text (default) or agent: one bounded JSON summary on stdout, " \
352
- "strict exit codes") { |v| @format = v }
393
+ opts.on("--format FORMAT", %w[text agent json], "text, agent or json (--dry-run inventory)") { |v| @format = v }
353
394
  opts.on("--agent", "Same as --format agent") { @format = "agent" }
354
- opts.on("--verbose", "Text report: also print score split, timings and kill causes") { @verbose = true }
395
+ opts.on("--verbose", "Text report: also print score split, timings and kill causes;",
396
+ "a crash prints its backtrace") { @verbose = true }
355
397
  opts.on("--[no-]color", "Colour the text report (default: on for a terminal unless NO_COLOR is set)") do |v|
356
398
  @color = v ? :always : :never
357
399
  end
@@ -367,12 +409,8 @@ module Saboteur
367
409
  Gates.define_options(opts) { |key, value| set(key, value) }
368
410
  end
369
411
 
370
- def add_subject(expr)
371
- (options[:subjects] ||= []) << expr
372
- end
412
+ def add_subject(expr) = (options[:subjects] ||= []) << expr
373
413
 
374
- def set(key, value)
375
- options[key] = value
376
- end
414
+ def set(key, value) = (options[key] = value)
377
415
  end
378
416
  end