saboteur 0.5.0 → 0.7.0

data/lib/saboteur/cli/exit_status.rb

@@ -8,7 +8,26 @@ module Saboteur
     module ExitStatus
       private
 
+      # Last resort: a bug must not escape as a bare stack trace and an accidental exit 1.
+      def internal_error(error)
+        CrashReport.call(error, verbose: @verbose, agent: agent?, err: @err, out: @out, log_path: crash_log_path)
+      end
+
+      # The crash log lives with the other run state. A broken config (or a
+      # nil cache_dir) must not swallow the crash report itself, so fall
+      # back to no log.
+      def crash_log_path
+        dir = config.path("cache_dir")
+        dir && File.join(dir, CrashReport::LOG_NAME)
+      rescue StandardError
+        nil
+      end
+
       def exit_code(report)
+        return EXIT_BUDGET if budget_exhausted?(report)
+
+        return EXIT_TOOL if untested_partial?(report)
+
         return EXIT_TOOL if mostly_errored?(report)
 
         return EXIT_TOOL if incomplete_under_threshold?(report)
@@ -16,6 +35,24 @@ module Saboteur
         failed_quality_gate?(report) || gates.failed?(report, @err) ? EXIT_SURVIVORS : EXIT_OK
       end
 
+      # A spent --max-duration budget exits 4 on its own, before the gates:
+      # the run is partial by design, and --max-duration-ok opts back into
+      # the plain verdict.
+      def budget_exhausted?(report)
+        report.early_stop&.reason == "max_duration" && !config["max_duration_ok"]
+      end
+
+      # A run stopped before any mutant was tested has no verdict to fall
+      # back on: --max-duration-ok must not turn it green. Unverified
+      # results are not tested verdicts, so an unverified-only stop has no
+      # score either (#296). Exit 3, like the other incomplete shapes.
+      def untested_partial?(report)
+        return false unless report.early_stop && report.score_breakdown.tested.zero?
+
+        @err.puts("saboteur: the run stopped before any mutant was tested, so there is no score to judge")
+        true
+      end
+
       def gates = (@gates ||= Gates.new(config))
 
       def mostly_errored?(report)
@@ -33,9 +70,11 @@ module Saboteur
       # incomplete run that mostly errored keeps its distinct cause, so the
       # harness rule means the same thing in both modes.
       def agent_exit_cause(report, payload)
+        return "budgetExhausted" if budget_exhausted?(report)
+
         case payload[:verdict]
         when "clean" then "ok"
-        when "survivors" then report.early_stop ? "failFast" : "survivors"
+        when "survivors" then report.early_stop&.reason == "fail_fast" ? "failFast" : "survivors"
         else report.majority_errored? ? "errorMajority" : "incomplete"
         end
       end
@@ -45,22 +84,28 @@ module Saboteur
       def exit_cause(report, code)
         case code
         when EXIT_OK then "ok"
+        when EXIT_BUDGET then "budgetExhausted"
         when EXIT_SURVIVORS then survivors_cause(report)
-        else report.majority_errored? ? "errorMajority" : "incompleteUnderThreshold"
+        else
+          if report.majority_errored?
+            "errorMajority"
+          else
+            report.early_stop && report.score_breakdown.tested.zero? ? "incomplete" : "incompleteUnderThreshold"
+          end
         end
       end
 
       # A run stopped by --fail-fast always exits 1: it stopped on a survivor, and
       # its score covers only the subjects that ran, so no threshold can pass it.
       def survivors_cause(report)
-        return "failFast" if report.early_stop
+        return "failFast" if report.early_stop&.reason == "fail_fast"
 
         gates.cause || (config["threshold"].nil? ? "survivors" : "belowThreshold")
       end
 
       # With --threshold the score decides; without it any unignored survivor does.
       def failed_quality_gate?(report)
-        return true if report.early_stop
+        return true if report.early_stop&.reason == "fail_fast"
 
         threshold = config["threshold"]
         return report.unignored_survivors.any? if threshold.nil?

data/lib/saboteur/cli/help_options.rb

@@ -0,0 +1,16 @@
+# SPDX-License-Identifier: MIT
+# frozen_string_literal: true
+
+module Saboteur
+  class CLI
+    # The option map `saboteur help` prints per command. run, pr and audit
+    # share the run parser: the presets take every run option.
+    module HelpOptions
+      def option_lines
+        run = parser.summarize
+        { "run" => run, "pr" => run, "audit" => run, "doctor" => doctor_parser.summarize,
+          "baseline" => baseline_parser.summarize }
+      end
+    end
+  end
+end

data/lib/saboteur/cli/presets.rb

@@ -0,0 +1,36 @@
+# SPDX-License-Identifier: MIT
+# frozen_string_literal: true
+
+module Saboteur
+  class CLI
+    # Opinionated bundles (#244): pr is the fast changed-lines loop (one
+    # baseline check, a 10-minute budget that does not fail the build),
+    # audit is the whole project with both reports. Preset flags go first,
+    # so an explicit flag after them wins on the usual last-flag rule.
+    module Presets
+      PRESETS = {
+        "pr" => %w[--baseline-passes 1 --max-duration 10m --max-duration-ok],
+        "audit" => %w[--all --operators all --baseline-passes 2 --json .saboteur/audit-report.json
+                      --html .saboteur/audit-report.html]
+      }.freeze
+
+      private
+
+      def preset(name)
+        flags = PRESETS.fetch(name).dup
+        if name == "pr"
+          ref = @argv.first.nil? || @argv.first.start_with?("-") ? "main" : @argv.shift
+          flags += ["--since", ref]
+        end
+        # An explicit scope beats the injected whole-suite --all, which
+        # would otherwise reject the combination: named tests, and the
+        # deprecated inference alias (#245).
+        explicit_scope = @argv.any? { |arg| arg == "--test" || arg.start_with?("--test=") || arg == "--guess-tests" }
+        flags.delete("--all") if explicit_scope
+        @err.puts("saboteur: preset '#{name}': #{flags.join(' ')} (explicit flags override)")
+        @argv = flags + @argv
+        run_command
+      end
+    end
+  end
+end

data/lib/saboteur/cli/run_budget_options.rb

@@ -0,0 +1,37 @@
+# SPDX-License-Identifier: MIT
+# frozen_string_literal: true
+
+module Saboteur
+  class CLI
+    # The run's clock and repeatability switches: seed, baseline check count
+    # (#55) and the wall-clock budget (#245). Grouped so the option table in
+    # the CLI stays under the house size caps.
+    module RunBudgetOptions
+      private
+
+      def add_run_budget_options(opts)
+        opts.on("--seed N", Integer, "Seed for Ruby's random numbers in every test run (default 1)") do |v|
+          set(:seed, v)
+        end
+        opts.on("--baseline-passes N", Integer, "Unchanged-suite checks (default 2)") { |v| set(:baseline_passes, v) }
+        opts.on("--max-duration DURATION", "Wall-clock budget for the whole run, e.g. 10m; exit 4 when spent") do |v|
+          set(:max_duration, parse_duration(v))
+        end
+        opts.on("--[no-]max-duration-ok", "An exhausted --max-duration budget does not fail the run") do |v|
+          set(:max_duration_ok, v)
+        end
+      end
+
+      # "10m" -> 600.0; bare numbers are seconds. A bad value is a usage error.
+      def parse_duration(value)
+        match = /\A(?<amount>\d+(?:\.\d+)?)(?<unit>[smh])?\z/.match(value.to_s.strip)
+        unless match
+          raise ConfigError,
+                "invalid --max-duration #{value.inspect}: use seconds or an s/m/h suffix, e.g. 10m"
+        end
+
+        match[:amount].to_f * { "s" => 1, "m" => 60, "h" => 3600, nil => 1 }.fetch(match[:unit])
+      end
+    end
+  end
+end

data/lib/saboteur/cli/run_reporting.rb

@@ -10,9 +10,10 @@ module Saboteur
     module RunReporting
       private
 
-      def text_run(report)
+      def text_run(report, engine)
         @err.puts
-        Reporter::Terminal.new(@out, verbose: @verbose || false, color: @color || :auto).call(report)
+        Reporter::Terminal.new(@out, verbose: @verbose || false, color: @color || :auto,
+                                     replay_command: survivor_replay_command(report, engine)).call(report)
         Reporter::Replay.new(@out).call(report) if options[:id]
         code = exit_code(report)
         # The JSON carries the exit cause an outer harness trusts, so it is
@@ -50,7 +51,65 @@ module Saboteur
         validate_history
         arguments = { err: @err }
         arguments[:digest] = options[:id] if options[:id]
-        agent? ? agent_run(arguments) : text_run(Engine.new(config, **arguments).call)
+        return agent_run(arguments) if agent?
+
+        # Text mode only: the engine prints a per-subject marks line when
+        # --verbose is on (#262); --agent output stays as it is.
+        engine = Engine.new(config, **arguments, verbose: @verbose || false)
+        text_run(engine.call, engine)
+      end
+
+      # The full replay command for a survivor digest: this run's own
+      # selection options, assembled by the same code that writes the agent
+      # follow-ups, so the user never reassembles them by hand (#261).
+      # Computed lazily: only a run with survivors asks for it.
+      def survivor_replay_command(report, engine)
+        lambda do |digest|
+          tree = WorkingTree.new(config.root, ignored: config.output_paths).state
+          Reporter::Agent::Commands.replay(agent_provenance(report, tree, engine), digest, capped: false)
+        end
+      end
+
+      # Writes the JSON report and returns its path, or nil when none is configured.
+      #
+      # An explicitly requested JSON report (--json, c.json_report) is the
+      # completion evidence an outer harness trusts: a failed write fails
+      # the run (exit 3), because a warning with exit 0 could leave a stale
+      # or partial file at the path and present old results as this run's.
+      # The default .saboteur/report.json is best-effort like the HTML
+      # report and the score history (#296).
+      def write_json(report, cause)
+        path = config.path("json_report")
+        return unless path
+
+        Reporter::Stryker.new(path).call(report, exit_cause: cause)
+        path
+      rescue SystemCallError, IOError => e
+        if config.explicit?("json_report")
+          raise Error, "could not write the JSON report to #{path}: #{e.message} - " \
+                       "any file at that path may be stale or incomplete"
+        end
+
+        warn_report_write_failure("JSON report", path, e)
+      end
+
+      def write_html(report)
+        path = config.path("html_report")
+        return unless path
+
+        Reporter::Html.new(path).call(report)
+        path
+      rescue SystemCallError, IOError => e
+        warn_report_write_failure("HTML report", path, e)
+      end
+
+      # An auxiliary report artifact that cannot be written - the tree is
+      # read-only inside a sandboxed child, as in the dogfood's nested runs
+      # (#296) - is a warning, not a run failure: the exit code belongs to
+      # the results. The JSON report is not auxiliary; see write_json.
+      def warn_report_write_failure(what, path, error)
+        @err.puts "saboteur: could not write the #{what} to #{path} (#{error.message}); continuing without it"
+        nil
       end
 
       def validate_history
@@ -66,9 +125,19 @@ module Saboteur
         return unless path
 
         tree = WorkingTree.new(config.root, ignored: config.output_paths).state
-        scope = config.to_h.except("json_report", "html_report", "score_history", "cache_dir")
-        message = Reporter::ScoreHistory.new(path).append(report, head: tree.head, scope: scope)
-        @err.puts("saboteur: #{message}")
+        scope = config.selection_settings.except("json_report", "html_report", "score_history", "cache_dir")
+        # Recording is an auxiliary artifact: when the file cannot be
+        # written - read-only tree inside a sandboxed child (#296) - warn
+        # and let the run exit by its results instead of failing as a tool
+        # error. Only the write's IO failures are soft: validation,
+        # integrity, record-size and lock failures (Saboteur::Error) stay
+        # hard, because they say the history is unusable, not unwritable.
+        message = begin
+          Reporter::ScoreHistory.new(path).append(report, head: tree.head, scope: scope)
+        rescue SystemCallError, IOError => e
+          warn_report_write_failure("score history", path, e)
+        end
+        @err.puts("saboteur: #{message}") if message
       end
     end
   end

data/lib/saboteur/cli.rb

@@ -3,6 +3,7 @@
 
 require "optparse"
 require_relative "../saboteur"
+require_relative "crash_report"
 require_relative "doctor"
 require_relative "help"
 require_relative "reporter/replay"
@@ -10,14 +11,20 @@ require_relative "baseline/command"
 require_relative "cli/exit_status"
 require_relative "preview"
 require_relative "cli/run_reporting"
+require_relative "cli/run_budget_options"
+require_relative "cli/presets"
+require_relative "cli/help_options"
 
 module Saboteur
   class CLI
     include Baseline::Command
     include ExitStatus
     include RunReporting
+    include RunBudgetOptions
+    include Presets
+    include HelpOptions
 
-    USAGE = "Usage: saboteur run [SOURCE...] [--test TEST...] [--threshold PERCENT] [--since REF] " \
+    USAGE = "Usage: saboteur run [SOURCE...] [--test TEST...] [--all] [--threshold PERCENT] [--since REF] " \
             "[--subject 'Klass#meth'] [--timeout-factor N] [--jobs N] [--seed N] [--id DIGEST]
 " \
             "Run 'saboteur help' for every command, option, config key and exit code."
@@ -26,10 +33,18 @@ module Saboteur
     EXIT_SURVIVORS = 1
     EXIT_USAGE = 2
     EXIT_TOOL = 3
+    EXIT_BUDGET = 4
     class UsageError < ConfigError; end
 
     attr_reader :options
 
+    # Found before parsing, so any error, even a bad option that comes first, still
+    # answers an agent with JSON. Class-level: the executable's fallback reads raw arguments.
+    def self.agent_requested?(argv)
+      argv.include?("--agent") || argv.include?("--format=agent") ||
+        argv.each_cons(2).any? { |flag, value| flag == "--format" && value == "agent" }
+    end
+
     def initialize(argv, out: $stdout, err: $stderr)
       @argv = argv.dup
       @out = out
@@ -39,18 +54,19 @@ module Saboteur
     end
 
     def call
-      case @argv.shift
+      command = @argv.shift
+      case command
       when "run" then run_command
       when "baseline" then baseline
       when "doctor" then help_requested? ? show_help("doctor") : doctor
       when "version", "--version" then @out.puts(VERSION) || EXIT_OK
-      when "help" then show_help(@argv.shift)
+      when "help", "--help", "-h" then show_help(@argv.shift)
       when "agent-skill" then @out.print(AgentSkill::TEXT) || EXIT_OK
-      when "--help", "-h" then show_help(nil)
-      else usage_error
+      else PRESETS.key?(command) ? preset(command) : usage_error
       end
     rescue OptionParser::ParseError => e
       usage_error(e.message)
+    rescue StandardError => e then internal_error(e)
     end
 
     def config
@@ -68,11 +84,6 @@ module Saboteur
       end
     end
 
-    # Each command's own option lines, as printed under OPTIONS in `saboteur help`.
-    def option_lines
-      { "run" => parser.summarize, "doctor" => doctor_parser.summarize, "baseline" => baseline_parser.summarize }
-    end
-
     def doctor_parser
       OptionParser.new do |opts|
         opts.banner = "Usage: saboteur doctor [--config PATH] [--seed N] [--suite] [--test PATH]"
@@ -114,12 +125,7 @@ module Saboteur
       @format == "agent"
     end
 
-    # Found before parsing, so any error, even a bad option that comes first, still
-    # answers an agent with JSON.
-    def agent_requested?
-      @argv.include?("--agent") || @argv.include?("--format=agent") ||
-        @argv.each_cons(2).any? { |flag, value| flag == "--format" && value == "agent" }
-    end
+    def agent_requested? = self.class.agent_requested?(@argv)
 
     # In agent mode the failure is also on stdout, where the agent reads its answer.
     def agent_failure(message, exit_code)
@@ -129,10 +135,13 @@ module Saboteur
 
     def run
       parser.parse!(@argv)
+      reject_test_selection_conflicts
       use_explicit_sources
       validate_run_options
       warn_dirty_tree
 
+      config.deprecation_warnings.each { |warning| @err.puts("saboteur: #{warning}") }
+
       return preview if options[:list_operators] || options[:dry_run]
 
       execute_run
@@ -150,6 +159,18 @@ module Saboteur
       WorkingTree.new(config.root, ignored: config.output_paths).state.warning&.then { |text| @err.puts(text) }
     end
 
+    # Contradictory selection flags are a usage error before any file or
+    # repository check: --test wins over inference by precedence, but only
+    # one of them may be named. Config-file combinations are not errors:
+    # --test wins there, so generated commands stay valid over any config.
+    def reject_test_selection_conflicts
+      raise UsageError, "--all cannot be combined with --test" if options[:all] && options[:tests]&.any?
+      raise UsageError, "--all cannot be combined with --guess-tests" if options[:all] && options[:guess_tests]
+      return unless options[:guess_tests] && options[:tests]&.any?
+
+      raise UsageError, "--guess-tests cannot be combined with --test"
+    end
+
     # Usage errors found before the target boots.
     def validate_run_options
       reject_threshold_with_agent_format
@@ -209,13 +230,36 @@ module Saboteur
       Reporter::Agent::Provenance.new(
         ref: config["since"], seed: report.seed, head: tree.head, working_tree: tree.label,
         changed_files: tree.changed_files, mode: agent_mode, json_report: config.path("json_report"),
-        subjects: Array(config["subjects"]), tests: Array(engine.guessed_tests || options[:tests]),
+        subjects: Array(config["subjects"]), tests: Array(engine.guessed_tests || config["tests"]),
         sources: @sources || [],
         config_path: options[:config], digest: options[:id], cache: config["cache"],
-        incremental: config["incremental"], settings: options.slice(*Reporter::Agent::Commands::SETTING_FLAGS.keys)
+        incremental: config["incremental"], settings: agent_settings
       )
     end
 
+    # A whole-suite run records its scope as --all, so the generated rerun
+    # and replay commands do not default back to inference (which can exit
+    # 2 in a project with a spec-less source file).
+    def agent_settings
+      settings = options.slice(*Reporter::Agent::Commands::SETTING_FLAGS.keys)
+                        .merge(operators: config["operators"], exclude_operators: config["exclude_operators"])
+      # The effective mode: named tests win over a configured whole-suite
+      # selection by precedence, so all is recorded only when no tests
+      # resolved - otherwise the follow-up commands would carry both --all
+      # and --test, which the CLI rejects.
+      settings[:all] = true if (config["all"] || config["test_selection"] == "all") && config["tests"].empty?
+      # CLI-only switches that change candidate availability or boot, so a
+      # printed rerun/replay must keep them (config-file settings like
+      # arid/hazard patterns, whole-file, require, rspec_args and rails ride
+      # along through --config or the default config path).
+      settings[:allow_hazardous] = true if options[:allow_hazardous]
+      settings[:allow_focus] = true if options[:allow_focus]
+      # An explicit false (--no-max-duration-ok over a config true) is a
+      # setting too: dropping it would silently re-enable the config value.
+      settings[:max_duration_ok] = options[:max_duration_ok] unless options[:max_duration_ok].nil?
+      settings
+    end
+
     def agent_mode
       return "replay" if options[:id]
       return "diff" if config["since"]
@@ -223,32 +267,13 @@ module Saboteur
       @sources&.any? ? "files" : "full"
     end
 
-    # Writes the JSON report and returns its path, or nil when none is configured.
-    def write_json(report, cause)
-      path = config.path("json_report")
-      return unless path
-
-      Reporter::Stryker.new(path).call(report, exit_cause: cause)
-      path
-    end
-
-    def write_html(report)
-      path = config.path("html_report")
-      return unless path
-
-      Reporter::Html.new(path).call(report)
-      path
-    end
-
     # Positional arguments name the files (or directories) to mutate. They
     # replace the configured include and exclude patterns. Each must exist,
     # so a typo is an error and not a run over zero subjects.
     def use_explicit_sources
       if @argv.empty?
-        raise UsageError, "--test needs at least one source file" if options[:tests] && !config["since"]
-
-        # With --since the changed files are the sources, so --test may
-        # stand alone; the files must still exist.
+        # --test may stand alone: the configured include (or the --since
+        # diff) names the sources. The files must still exist.
         options[:tests]&.each { |test| require_existing_test(test) }
         return
       end
@@ -286,17 +311,23 @@ module Saboteur
       raise UsageError, "--dry-run and --list-operators print text, not --agent JSON" if agent?
       raise UsageError, "--id replays one mutant: not with --dry-run or --list-operators" if options[:id]
 
-      return MutationSource::OperatorRegistry.list(@out) || EXIT_OK if options[:list_operators]
+      if options[:list_operators]
+        MutationSource::OperatorRegistry.list(@out, names: config["operators"], excluded: config["exclude_operators"])
+        return EXIT_OK
+      end
 
       Preview.new(config, out: @out, err: @err).call
       EXIT_OK
     end
 
     def define_selection_options(opts)
-      opts.on("--operators NAMES", Array, "Choose operator names or tiers (core, extended, all)") do |v|
+      opts.on("--operators NAMES", Array, "Choose operator names (default: all)") do |v|
         set(:operators, v)
       end
-      opts.on("--list-operators", "List operator names, descriptions and tiers") { set(:list_operators, true) }
+      opts.on("--exclude-operators NAMES", Array, "Exclude operator names from the selected set") do |v|
+        set(:exclude_operators, v)
+      end
+      opts.on("--list-operators", "List operator names and descriptions") { set(:list_operators, true) }
       opts.on("--dry-run", "Preview mutants without boot, tests, forks or report writes") { set(:dry_run, true) }
       opts.on("--id DIGEST", "Replay exactly one current mutant by its full digest") { |v| set(:id, v) }
       opts.on("--since REF", "Only mutate subjects changed since git REF") { |v| options[:since] = v }
@@ -306,9 +337,7 @@ module Saboteur
     def define_execution_options(opts)
       opts.on("--timeout-factor N", Float, "Multiple of baseline runtime (default 3)") { |v| set(:timeout_factor, v) }
       opts.on("--jobs N", Integer, "Concurrent workers (default 1)") { |v| set(:jobs, v) }
-      opts.on("--seed N", Integer, "Seed for Ruby's random numbers in every test run (default 1)") do |v|
-        set(:seed, v)
-      end
+      add_run_budget_options(opts)
       opts.on("--config PATH", "Config file (default config/saboteur.rb)") { |v| set(:config, v) }
       opts.on("--integration NAME", "Test adapter (rspec)") { |v| set(:integration, v) }
       opts.on("--reeval STRATEGY", Execution::Reevaluator::STRATEGIES, "class_eval or nesting") { |v| set(:reeval, v) }
@@ -321,8 +350,10 @@ module Saboteur
       opts.on("--allow-focus", "Run even when a focus filter (fit, fdescribe, :focus) narrows the suite") do
         set(:allow_focus, true)
       end
-      opts.on("--[no-]guess-tests", "With --since: run only the specs named after the changed files " \
-                                    "(lib/a.rb -> spec/a_spec.rb); the whole suite if any has none") do |v|
+      opts.on("--all", "Run the whole configured test suite instead of inferring the specs named after the " \
+                       "mutated files") { set(:all, true) }
+      opts.on("--[no-]guess-tests", "Deprecated alias: inferred specs are already the default; " \
+                                    "--no-guess-tests warns and is ignored - use --all for the whole suite") do |v|
         set(:guess_tests, v)
       end
     end
@@ -340,7 +371,8 @@ module Saboteur
       opts.on("--format FORMAT", %w[text agent], "text (default) or agent: one bounded JSON summary on stdout, " \
                                                  "strict exit codes") { |v| @format = v }
       opts.on("--agent", "Same as --format agent") { @format = "agent" }
-      opts.on("--verbose", "Text report: also print score split, timings and kill causes") { @verbose = true }
+      opts.on("--verbose", "Text report: also print score split, timings and kill causes;",
+              "a crash prints its backtrace") { @verbose = true }
       opts.on("--[no-]color", "Colour the text report (default: on for a terminal unless NO_COLOR is set)") do |v|
         @color = v ? :always : :never
       end