saboteur 0.6.0 → 0.7.0

data/lib/saboteur/cli/run_budget_options.rb

@@ -0,0 +1,37 @@
+# SPDX-License-Identifier: MIT
+# frozen_string_literal: true
+
+module Saboteur
+  class CLI
+    # The run's clock and repeatability switches: seed, baseline check count
+    # (#55) and the wall-clock budget (#245). Grouped so the option table in
+    # the CLI stays under the house size caps.
+    module RunBudgetOptions
+      private
+
+      def add_run_budget_options(opts)
+        opts.on("--seed N", Integer, "Seed for Ruby's random numbers in every test run (default 1)") do |v|
+          set(:seed, v)
+        end
+        opts.on("--baseline-passes N", Integer, "Unchanged-suite checks (default 2)") { |v| set(:baseline_passes, v) }
+        opts.on("--max-duration DURATION", "Wall-clock budget for the whole run, e.g. 10m; exit 4 when spent") do |v|
+          set(:max_duration, parse_duration(v))
+        end
+        opts.on("--[no-]max-duration-ok", "An exhausted --max-duration budget does not fail the run") do |v|
+          set(:max_duration_ok, v)
+        end
+      end
+
+      # "10m" -> 600.0; bare numbers are seconds. A bad value is a usage error.
+      def parse_duration(value)
+        match = /\A(?<amount>\d+(?:\.\d+)?)(?<unit>[smh])?\z/.match(value.to_s.strip)
+        unless match
+          raise ConfigError,
+                "invalid --max-duration #{value.inspect}: use seconds or an s/m/h suffix, e.g. 10m"
+        end
+
+        match[:amount].to_f * { "s" => 1, "m" => 60, "h" => 3600, nil => 1 }.fetch(match[:unit])
+      end
+    end
+  end
+end

data/lib/saboteur/cli/run_reporting.rb

@@ -10,9 +10,10 @@ module Saboteur
     module RunReporting
       private
 
-      def text_run(report)
+      def text_run(report, engine)
         @err.puts
-        Reporter::Terminal.new(@out, verbose: @verbose || false, color: @color || :auto).call(report)
+        Reporter::Terminal.new(@out, verbose: @verbose || false, color: @color || :auto,
+                                     replay_command: survivor_replay_command(report, engine)).call(report)
         Reporter::Replay.new(@out).call(report) if options[:id]
         code = exit_code(report)
         # The JSON carries the exit cause an outer harness trusts, so it is
@@ -50,7 +51,65 @@ module Saboteur
         validate_history
         arguments = { err: @err }
         arguments[:digest] = options[:id] if options[:id]
-        agent? ? agent_run(arguments) : text_run(Engine.new(config, **arguments).call)
+        return agent_run(arguments) if agent?
+
+        # Text mode only: the engine prints a per-subject marks line when
+        # --verbose is on (#262); --agent output stays as it is.
+        engine = Engine.new(config, **arguments, verbose: @verbose || false)
+        text_run(engine.call, engine)
+      end
+
+      # The full replay command for a survivor digest: this run's own
+      # selection options, assembled by the same code that writes the agent
+      # follow-ups, so the user never reassembles them by hand (#261).
+      # Computed lazily: only a run with survivors asks for it.
+      def survivor_replay_command(report, engine)
+        lambda do |digest|
+          tree = WorkingTree.new(config.root, ignored: config.output_paths).state
+          Reporter::Agent::Commands.replay(agent_provenance(report, tree, engine), digest, capped: false)
+        end
+      end
+
+      # Writes the JSON report and returns its path, or nil when none is configured.
+      #
+      # An explicitly requested JSON report (--json, c.json_report) is the
+      # completion evidence an outer harness trusts: a failed write fails
+      # the run (exit 3), because a warning with exit 0 could leave a stale
+      # or partial file at the path and present old results as this run's.
+      # The default .saboteur/report.json is best-effort like the HTML
+      # report and the score history (#296).
+      def write_json(report, cause)
+        path = config.path("json_report")
+        return unless path
+
+        Reporter::Stryker.new(path).call(report, exit_cause: cause)
+        path
+      rescue SystemCallError, IOError => e
+        if config.explicit?("json_report")
+          raise Error, "could not write the JSON report to #{path}: #{e.message} - " \
+                       "any file at that path may be stale or incomplete"
+        end
+
+        warn_report_write_failure("JSON report", path, e)
+      end
+
+      def write_html(report)
+        path = config.path("html_report")
+        return unless path
+
+        Reporter::Html.new(path).call(report)
+        path
+      rescue SystemCallError, IOError => e
+        warn_report_write_failure("HTML report", path, e)
+      end
+
+      # An auxiliary report artifact that cannot be written - the tree is
+      # read-only inside a sandboxed child, as in the dogfood's nested runs
+      # (#296) - is a warning, not a run failure: the exit code belongs to
+      # the results. The JSON report is not auxiliary; see write_json.
+      def warn_report_write_failure(what, path, error)
+        @err.puts "saboteur: could not write the #{what} to #{path} (#{error.message}); continuing without it"
+        nil
       end
 
       def validate_history
@@ -67,8 +126,18 @@ module Saboteur
 
         tree = WorkingTree.new(config.root, ignored: config.output_paths).state
         scope = config.selection_settings.except("json_report", "html_report", "score_history", "cache_dir")
-        message = Reporter::ScoreHistory.new(path).append(report, head: tree.head, scope: scope)
-        @err.puts("saboteur: #{message}")
+        # Recording is an auxiliary artifact: when the file cannot be
+        # written - read-only tree inside a sandboxed child (#296) - warn
+        # and let the run exit by its results instead of failing as a tool
+        # error. Only the write's IO failures are soft: validation,
+        # integrity, record-size and lock failures (Saboteur::Error) stay
+        # hard, because they say the history is unusable, not unwritable.
+        message = begin
+          Reporter::ScoreHistory.new(path).append(report, head: tree.head, scope: scope)
+        rescue SystemCallError, IOError => e
+          warn_report_write_failure("score history", path, e)
+        end
+        @err.puts("saboteur: #{message}") if message
       end
     end
   end

data/lib/saboteur/cli.rb

@@ -3,6 +3,7 @@
 
 require "optparse"
 require_relative "../saboteur"
+require_relative "crash_report"
 require_relative "doctor"
 require_relative "help"
 require_relative "reporter/replay"
@@ -10,14 +11,20 @@ require_relative "baseline/command"
 require_relative "cli/exit_status"
 require_relative "preview"
 require_relative "cli/run_reporting"
+require_relative "cli/run_budget_options"
+require_relative "cli/presets"
+require_relative "cli/help_options"
 
 module Saboteur
   class CLI
     include Baseline::Command
     include ExitStatus
     include RunReporting
+    include RunBudgetOptions
+    include Presets
+    include HelpOptions
 
-    USAGE = "Usage: saboteur run [SOURCE...] [--test TEST...] [--threshold PERCENT] [--since REF] " \
+    USAGE = "Usage: saboteur run [SOURCE...] [--test TEST...] [--all] [--threshold PERCENT] [--since REF] " \
             "[--subject 'Klass#meth'] [--timeout-factor N] [--jobs N] [--seed N] [--id DIGEST]
 " \
             "Run 'saboteur help' for every command, option, config key and exit code."
@@ -26,10 +33,18 @@ module Saboteur
     EXIT_SURVIVORS = 1
     EXIT_USAGE = 2
     EXIT_TOOL = 3
+    EXIT_BUDGET = 4
     class UsageError < ConfigError; end
 
     attr_reader :options
 
+    # Found before parsing, so any error, even a bad option that comes first, still
+    # answers an agent with JSON. Class-level: the executable's fallback reads raw arguments.
+    def self.agent_requested?(argv)
+      argv.include?("--agent") || argv.include?("--format=agent") ||
+        argv.each_cons(2).any? { |flag, value| flag == "--format" && value == "agent" }
+    end
+
     def initialize(argv, out: $stdout, err: $stderr)
       @argv = argv.dup
       @out = out
@@ -39,18 +54,19 @@ module Saboteur
     end
 
     def call
-      case @argv.shift
+      command = @argv.shift
+      case command
       when "run" then run_command
       when "baseline" then baseline
       when "doctor" then help_requested? ? show_help("doctor") : doctor
       when "version", "--version" then @out.puts(VERSION) || EXIT_OK
-      when "help" then show_help(@argv.shift)
+      when "help", "--help", "-h" then show_help(@argv.shift)
       when "agent-skill" then @out.print(AgentSkill::TEXT) || EXIT_OK
-      when "--help", "-h" then show_help(nil)
-      else usage_error
+      else PRESETS.key?(command) ? preset(command) : usage_error
       end
     rescue OptionParser::ParseError => e
       usage_error(e.message)
+    rescue StandardError => e then internal_error(e)
     end
 
     def config
@@ -68,11 +84,6 @@ module Saboteur
       end
     end
 
-    # Each command's own option lines, as printed under OPTIONS in `saboteur help`.
-    def option_lines
-      { "run" => parser.summarize, "doctor" => doctor_parser.summarize, "baseline" => baseline_parser.summarize }
-    end
-
     def doctor_parser
       OptionParser.new do |opts|
         opts.banner = "Usage: saboteur doctor [--config PATH] [--seed N] [--suite] [--test PATH]"
@@ -114,12 +125,7 @@ module Saboteur
       @format == "agent"
     end
 
-    # Found before parsing, so any error, even a bad option that comes first, still
-    # answers an agent with JSON.
-    def agent_requested?
-      @argv.include?("--agent") || @argv.include?("--format=agent") ||
-        @argv.each_cons(2).any? { |flag, value| flag == "--format" && value == "agent" }
-    end
+    def agent_requested? = self.class.agent_requested?(@argv)
 
     # In agent mode the failure is also on stdout, where the agent reads its answer.
     def agent_failure(message, exit_code)
@@ -129,10 +135,13 @@ module Saboteur
 
     def run
       parser.parse!(@argv)
+      reject_test_selection_conflicts
       use_explicit_sources
       validate_run_options
       warn_dirty_tree
 
+      config.deprecation_warnings.each { |warning| @err.puts("saboteur: #{warning}") }
+
       return preview if options[:list_operators] || options[:dry_run]
 
       execute_run
@@ -150,6 +159,18 @@ module Saboteur
       WorkingTree.new(config.root, ignored: config.output_paths).state.warning&.then { |text| @err.puts(text) }
     end
 
+    # Contradictory selection flags are a usage error before any file or
+    # repository check: --test wins over inference by precedence, but only
+    # one of them may be named. Config-file combinations are not errors:
+    # --test wins there, so generated commands stay valid over any config.
+    def reject_test_selection_conflicts
+      raise UsageError, "--all cannot be combined with --test" if options[:all] && options[:tests]&.any?
+      raise UsageError, "--all cannot be combined with --guess-tests" if options[:all] && options[:guess_tests]
+      return unless options[:guess_tests] && options[:tests]&.any?
+
+      raise UsageError, "--guess-tests cannot be combined with --test"
+    end
+
     # Usage errors found before the target boots.
     def validate_run_options
       reject_threshold_with_agent_format
@@ -209,16 +230,34 @@ module Saboteur
       Reporter::Agent::Provenance.new(
         ref: config["since"], seed: report.seed, head: tree.head, working_tree: tree.label,
         changed_files: tree.changed_files, mode: agent_mode, json_report: config.path("json_report"),
-        subjects: Array(config["subjects"]), tests: Array(engine.guessed_tests || options[:tests]),
+        subjects: Array(config["subjects"]), tests: Array(engine.guessed_tests || config["tests"]),
         sources: @sources || [],
         config_path: options[:config], digest: options[:id], cache: config["cache"],
         incremental: config["incremental"], settings: agent_settings
       )
     end
 
+    # A whole-suite run records its scope as --all, so the generated rerun
+    # and replay commands do not default back to inference (which can exit
+    # 2 in a project with a spec-less source file).
     def agent_settings
-      options.slice(*Reporter::Agent::Commands::SETTING_FLAGS.keys)
-             .merge(operators: config["operators"], exclude_operators: config["exclude_operators"])
+      settings = options.slice(*Reporter::Agent::Commands::SETTING_FLAGS.keys)
+                        .merge(operators: config["operators"], exclude_operators: config["exclude_operators"])
+      # The effective mode: named tests win over a configured whole-suite
+      # selection by precedence, so all is recorded only when no tests
+      # resolved - otherwise the follow-up commands would carry both --all
+      # and --test, which the CLI rejects.
+      settings[:all] = true if (config["all"] || config["test_selection"] == "all") && config["tests"].empty?
+      # CLI-only switches that change candidate availability or boot, so a
+      # printed rerun/replay must keep them (config-file settings like
+      # arid/hazard patterns, whole-file, require, rspec_args and rails ride
+      # along through --config or the default config path).
+      settings[:allow_hazardous] = true if options[:allow_hazardous]
+      settings[:allow_focus] = true if options[:allow_focus]
+      # An explicit false (--no-max-duration-ok over a config true) is a
+      # setting too: dropping it would silently re-enable the config value.
+      settings[:max_duration_ok] = options[:max_duration_ok] unless options[:max_duration_ok].nil?
+      settings
     end
 
     def agent_mode
@@ -228,32 +267,13 @@ module Saboteur
       @sources&.any? ? "files" : "full"
     end
 
-    # Writes the JSON report and returns its path, or nil when none is configured.
-    def write_json(report, cause)
-      path = config.path("json_report")
-      return unless path
-
-      Reporter::Stryker.new(path).call(report, exit_cause: cause)
-      path
-    end
-
-    def write_html(report)
-      path = config.path("html_report")
-      return unless path
-
-      Reporter::Html.new(path).call(report)
-      path
-    end
-
     # Positional arguments name the files (or directories) to mutate. They
     # replace the configured include and exclude patterns. Each must exist,
     # so a typo is an error and not a run over zero subjects.
     def use_explicit_sources
       if @argv.empty?
-        raise UsageError, "--test needs at least one source file" if options[:tests] && !config["since"]
-
-        # With --since the changed files are the sources, so --test may
-        # stand alone; the files must still exist.
+        # --test may stand alone: the configured include (or the --since
+        # diff) names the sources. The files must still exist.
         options[:tests]&.each { |test| require_existing_test(test) }
         return
       end
@@ -317,9 +337,7 @@ module Saboteur
     def define_execution_options(opts)
       opts.on("--timeout-factor N", Float, "Multiple of baseline runtime (default 3)") { |v| set(:timeout_factor, v) }
       opts.on("--jobs N", Integer, "Concurrent workers (default 1)") { |v| set(:jobs, v) }
-      opts.on("--seed N", Integer, "Seed for Ruby's random numbers in every test run (default 1)") do |v|
-        set(:seed, v)
-      end
+      add_run_budget_options(opts)
       opts.on("--config PATH", "Config file (default config/saboteur.rb)") { |v| set(:config, v) }
       opts.on("--integration NAME", "Test adapter (rspec)") { |v| set(:integration, v) }
       opts.on("--reeval STRATEGY", Execution::Reevaluator::STRATEGIES, "class_eval or nesting") { |v| set(:reeval, v) }
@@ -332,8 +350,10 @@ module Saboteur
       opts.on("--allow-focus", "Run even when a focus filter (fit, fdescribe, :focus) narrows the suite") do
         set(:allow_focus, true)
       end
-      opts.on("--[no-]guess-tests", "With --since: run only the specs named after the changed files " \
-                                    "(lib/a.rb -> spec/a_spec.rb); the whole suite if any has none") do |v|
+      opts.on("--all", "Run the whole configured test suite instead of inferring the specs named after the " \
+                       "mutated files") { set(:all, true) }
+      opts.on("--[no-]guess-tests", "Deprecated alias: inferred specs are already the default; " \
+                                    "--no-guess-tests warns and is ignored - use --all for the whole suite") do |v|
         set(:guess_tests, v)
       end
     end
@@ -351,7 +371,8 @@ module Saboteur
       opts.on("--format FORMAT", %w[text agent], "text (default) or agent: one bounded JSON summary on stdout, " \
                                                  "strict exit codes") { |v| @format = v }
       opts.on("--agent", "Same as --format agent") { @format = "agent" }
-      opts.on("--verbose", "Text report: also print score split, timings and kill causes") { @verbose = true }
+      opts.on("--verbose", "Text report: also print score split, timings and kill causes;",
+              "a crash prints its backtrace") { @verbose = true }
       opts.on("--[no-]color", "Colour the text report (default: on for a terminal unless NO_COLOR is set)") do |v|
         @color = v ? :always : :never
       end

data/lib/saboteur/config.rb

@@ -1,6 +1,8 @@
 # SPDX-License-Identifier: MIT
 # frozen_string_literal: true
 
+require_relative "text"
+
 require_relative "mutation_source/builtin"
 require_relative "mutation_source/operator_registry"
 require_relative "mutation_source/hazard_guard"
@@ -57,8 +59,13 @@ module Saboteur
       "allow_focus" => false,
       "fail_fast" => false,
       "guess_tests" => false,
+      "all" => false,
+      "test_selection" => "auto",
       "matrix" => false,
       "seed" => 1,
+      "baseline_passes" => 2,
+      "max_duration" => nil,
+      "max_duration_ok" => false,
       "tests" => [],
       "fingerprint_extra" => [],
       "threshold" => nil,
@@ -66,8 +73,9 @@ module Saboteur
       "max_errors" => nil
     }.freeze
 
-    INTEGER_KEYS = %w[jobs max_memory_mb max_processes max_mutants_per_method seed].freeze
+    INTEGER_KEYS = %w[jobs max_memory_mb max_processes max_mutants_per_method seed baseline_passes].freeze
     FLOAT_KEYS = %w[timeout_factor timeout_min baseline_timeout coverage_timeout].freeze
+    NILABLE_FLOAT_KEYS = %w[threshold max_duration].freeze
     THRESHOLD_RANGE = (0.0..100.0)
 
     LIST_KEYS = %w[require include exclude arid hazard sources hooks rspec_args subjects tests
@@ -96,11 +104,19 @@ module Saboteur
       hash.to_h { |key, value| [key.to_s, value] }.compact
     end
 
-    def initialize(values = {}, root: Dir.pwd)
-      @root = File.expand_path(root)
+    # Deprecations found while loading: printed by the CLI, never fatal.
+    attr_reader :deprecation_warnings
+
+    def initialize(values = {}, root: Dir.pwd, explicit: nil)
+      # Retag the root losslessly: under a non-UTF-8 locale it arrives
+      # locale-tagged, and every glob, join and expand below would mix
+      # encodings with the UTF-8 patterns from the config file (#231).
+      @root = Text.utf8_identity(File.expand_path(root))
       unknown = values.keys - DEFAULTS.keys
       raise ConfigError, "unknown config key(s): #{unknown.join(', ')}" if unknown.any?
 
+      @deprecation_warnings = deprecations(values)
+      @explicit_keys = (explicit || values.keys).map(&:to_s).freeze
       @values = DEFAULTS.merge(values)
       coerce!
       validate!
@@ -115,6 +131,26 @@ module Saboteur
       @values.fetch(key.to_s, default)
     end
 
+    # Keys the user set (config file or CLI flag) rather than relying on
+    # the default; carried through Engine's config clone so provenance
+    # survives (#296).
+    attr_reader :explicit_keys
+
+    # A copy with some values replaced or dropped, keeping which keys the
+    # user set explicitly: the replacements count as explicit, the dropped
+    # keys lose it (#296).
+    def with(values, except: [])
+      merged = to_h.except(*except).merge(values)
+      Config.new(merged, root: root, explicit: (explicit_keys - except.map(&:to_s)) | values.keys.map(&:to_s))
+    end
+
+    # Whether the user set a key (config file or CLI flag) rather than
+    # relying on the default. An explicitly requested report artifact is
+    # completion evidence, so its write failure is a hard error (#296).
+    def explicit?(key)
+      @explicit_keys.include?(key.to_s)
+    end
+
     def to_h
       @values.dup
     end
@@ -124,7 +160,8 @@ module Saboteur
       matched = self["include"].flat_map { |pattern| Dir.glob(pattern, base: root) }
       excluded = self["exclude"].flat_map { |pattern| Dir.glob(pattern, base: root) }
       (matched - excluded).select { |file| file.end_with?(".rb") }
-                          .map { |file| File.realpath(File.expand_path(file, root)) }.uniq.sort
+                          .map { |file| File.realpath(File.expand_path(Text.utf8_identity(file), root)) }
+                          .uniq.sort
     end
 
     # Patterns that match no file. A pattern goes stale when files move;
@@ -151,11 +188,23 @@ module Saboteur
 
     private
 
+    # An explicit guess_tests = false (config file or --no-guess-tests) used
+    # to keep the whole suite; inference is the default now, so the opt-out
+    # is --all / c.test_selection = :all. guess_tests = true is the default
+    # behavior spelled out, so it stays silent.
+    def deprecations(values)
+      return [] unless values["guess_tests"] == false
+
+      ["guess_tests = false is deprecated: inferred specs are the default and it no longer restores the " \
+       "whole suite; use --all or c.test_selection = :all to run every spec"]
+    end
+
     def coerce!
       INTEGER_KEYS.each { |key| @values[key] = Integer(@values[key]) }
       FLOAT_KEYS.each { |key| @values[key] = Float(@values[key]) }
       LIST_KEYS.each { |key| @values[key] = Array(@values[key]).map(&:to_s) }
-      @values["threshold"] = Float(@values["threshold"]) unless @values["threshold"].nil?
+      NILABLE_FLOAT_KEYS.each { |key| @values[key] = Float(@values[key]) unless @values[key].nil? }
+      @values["test_selection"] = @values["test_selection"].to_s
     rescue ArgumentError, TypeError => e
       raise ConfigError, "invalid config value: #{e.message}"
     end
@@ -164,6 +213,13 @@ module Saboteur
       raise ConfigError, "max_mutants_per_method must be > 0" unless self["max_mutants_per_method"].positive?
     end
 
+    # The unchanged-suite checks: how many, and the whole run's clock (#55, #245).
+    def validate_baseline!
+      raise ConfigError, "seed must be >= 0" if self["seed"].negative?
+      raise ConfigError, "baseline_passes must be >= 1" if self["baseline_passes"] < 1
+      raise ConfigError, "max_duration must be > 0" if self["max_duration"] && !self["max_duration"].positive?
+    end
+
     def validate_report_paths!
       json = path("json_report")
       if json && json == path("html_report")
@@ -179,7 +235,7 @@ module Saboteur
     def validate_modes!
       validate_fail_fast!
       validate_matrix!
-      validate_guess_tests!
+      validate_test_selection!
       validate_gates!
     end
 
@@ -208,11 +264,10 @@ module Saboteur
       raise ConfigError, "threshold must be between 0 and 100"
     end
 
-    def validate_guess_tests!
-      return unless self["guess_tests"]
+    def validate_test_selection!
+      return if %w[auto all].include?(self["test_selection"])
 
-      raise ConfigError, "guess_tests needs since" unless self["since"]
-      raise ConfigError, "guess_tests cannot be combined with tests" if self["tests"].any?
+      raise ConfigError, "test_selection must be one of: auto, all"
     end
 
     def validate_reeval!
@@ -229,7 +284,7 @@ module Saboteur
       validate_mutant_limit!
       validate_threshold!
       validate_modes!
-      raise ConfigError, "seed must be >= 0" if self["seed"].negative?
+      validate_baseline!
       raise ConfigError, "max_memory_mb must be > 0" unless self["max_memory_mb"].positive?
       raise ConfigError, "max_processes must be > 0" unless self["max_processes"].positive?
       raise ConfigError, "timeout_factor must be > 0" unless self["timeout_factor"].positive?