saboteur 0.7.0 → 0.8.1

data/lib/saboteur/cli.rb

@@ -11,17 +11,27 @@ require_relative "baseline/command"
 require_relative "cli/exit_status"
 require_relative "preview"
 require_relative "cli/run_reporting"
+require_relative "cli/survivors_command"
+require_relative "cli/source_set"
 require_relative "cli/run_budget_options"
+require_relative "cli/shard_options"
+require_relative "cli/selection_options"
 require_relative "cli/presets"
 require_relative "cli/help_options"
+require_relative "cli/history_command"
 
 module Saboteur
   class CLI
     include Baseline::Command
     include ExitStatus
     include RunReporting
+    include SurvivorsCommand
+    include SourceSet
     include RunBudgetOptions
+    include ShardOptions
+    include SelectionOptions
     include Presets
+    include HistoryCommand
     include HelpOptions
 
     USAGE = "Usage: saboteur run [SOURCE...] [--test TEST...] [--all] [--threshold PERCENT] [--since REF] " \
@@ -58,11 +68,13 @@ module Saboteur
       case command
       when "run" then run_command
       when "baseline" then baseline
-      when "doctor" then help_requested? ? show_help("doctor") : doctor
+      when "survivors" then survivors_command
+      when "doctor" then doctor_command
+      when "badge", "trend" then history_entry(command)
       when "version", "--version" then @out.puts(VERSION) || EXIT_OK
       when "help", "--help", "-h" then show_help(@argv.shift)
       when "agent-skill" then @out.print(AgentSkill::TEXT) || EXIT_OK
-      else PRESETS.key?(command) ? preset(command) : usage_error
+      else preset_or_usage_error(command)
       end
     rescue OptionParser::ParseError => e
       usage_error(e.message)
@@ -98,13 +110,13 @@ module Saboteur
 
     private
 
-    def run_command
-      help_requested? ? show_help("run") : run
-    end
+    def run_command = help_requested? ? show_help("run") : run
 
-    def help_requested?
-      @argv.intersect?(%w[--help -h])
-    end
+    def doctor_command = help_requested? ? show_help("doctor") : doctor
+
+    def history_entry(command) = help_requested? ? show_help(command) : history_command(command)
+
+    def help_requested? = @argv.intersect?(%w[--help -h])
 
     def show_help(command_name)
       help = Help.new(option_lines: option_lines)
@@ -121,9 +133,7 @@ module Saboteur
       agent_failure(message || USAGE, EXIT_USAGE)
     end
 
-    def agent?
-      @format == "agent"
-    end
+    def agent? = @format == "agent"
 
     def agent_requested? = self.class.agent_requested?(@argv)
 
@@ -144,6 +154,7 @@ module Saboteur
 
       return preview if options[:list_operators] || options[:dry_run]
 
+      refuse_empty_source_set
       execute_run
     rescue ConfigError => e
       @err.puts("saboteur: #{e.message}")
@@ -155,8 +166,12 @@ module Saboteur
 
     # Mutants run your code, and a mutant of code that writes or deletes files
     # can change the tree. Say so up front when uncommitted work is at stake.
+    # The scope the warning names is the configured one: positional source
+    # operands narrow the run after the warning, so the include/exclude they
+    # imply stay out of its config (#309, #503 review).
     def warn_dirty_tree
-      WorkingTree.new(config.root, ignored: config.output_paths).state.warning&.then { |text| @err.puts(text) }
+      warning = @sources ? Config.load(path: options[:config], overrides: overrides.except(:include, :exclude)) : config
+      DirtyTreeWarning.for(warning, config_path: options[:config])&.then { |text| @err.puts(text) }
     end
 
     # Contradictory selection flags are a usage error before any file or
@@ -165,30 +180,46 @@ module Saboteur
     # --test wins there, so generated commands stay valid over any config.
     def reject_test_selection_conflicts
       raise UsageError, "--all cannot be combined with --test" if options[:all] && options[:tests]&.any?
+
+      reject_test_command_conflicts
       raise UsageError, "--all cannot be combined with --guess-tests" if options[:all] && options[:guess_tests]
       return unless options[:guess_tests] && options[:tests]&.any?
 
       raise UsageError, "--guess-tests cannot be combined with --test"
     end
 
+    # --test-command replaces the whole in-process test machinery: test
+    # selection, coverage, parallelism and the sandbox do not apply.
+    def reject_test_command_conflicts
+      return unless options[:test_command]
+
+      # options[:tests] is nil or a non-empty list, so truthiness is the check.
+      raise UsageError, "--test-command cannot be combined with --test" if options[:tests]
+      raise UsageError, "--test-command cannot be combined with --incremental" if options[:incremental]
+    end
+
     # Usage errors found before the target boots.
     def validate_run_options
-      reject_threshold_with_agent_format
+      reject_format_conflicts
+      validate_shard_options
       Selection::GitDiff.verify_ref!(config["since"], root: config.root) if config["since"]
     end
 
-    # Agent mode has one strict meaning for exit 0, so a score threshold cannot also decide it.
-    def reject_threshold_with_agent_format
-      return unless agent? && (config["threshold"] || gates.any?)
+    # Agent mode allows no score gates (one strict exit 0); the json inventory is --dry-run planning only.
+    def reject_format_conflicts
+      if agent? && (config["threshold"] || gates.any?)
+        raise UsageError, "#{config['threshold'] ? '--threshold' : gates.agent_option_name} " \
+                          "cannot be combined with --agent"
+      end
+      return unless @format == "json" && !options[:dry_run]
 
-      raise UsageError, "#{config['threshold'] ? '--threshold' : '--max-* gates'} cannot be combined with --agent"
+      raise UsageError, "--format json is the --dry-run mutant inventory; a run summary is --format agent"
     end
 
     # One line naming the reports written, paths relative to the root when
     # they sit under it (a blank line before it, as the old JSON line had).
     def announce_reports(reports)
-      written = reports.select { |path, _label| path }
-      return if written.empty?
+      return if (written = reports.select { |path, _label| path }).empty?
 
       root = File.join(File.realpath(config.root), "")
       names = written.map do |path, label|
@@ -233,10 +264,14 @@ module Saboteur
         subjects: Array(config["subjects"]), tests: Array(engine.guessed_tests || config["tests"]),
         sources: @sources || [],
         config_path: options[:config], digest: options[:id], cache: config["cache"],
-        incremental: config["incremental"], settings: agent_settings
+        incremental: config["incremental"], settings: agent_settings, typed_settings: typed_settings
       )
     end
 
+    # The settings keys typed on the command line: a typed empty operator
+    # list may clear a config value, so the generated commands keep it.
+    def typed_settings = options.keys & Reporter::Agent::Commands::SETTING_FLAGS.keys
+
     # A whole-suite run records its scope as --all, so the generated rerun
     # and replay commands do not default back to inference (which can exit
     # 2 in a project with a spec-less source file).
@@ -271,17 +306,15 @@ module Saboteur
     # replace the configured include and exclude patterns. Each must exist,
     # so a typo is an error and not a run over zero subjects.
     def use_explicit_sources
-      if @argv.empty?
-        # --test may stand alone: the configured include (or the --since
-        # diff) names the sources. The files must still exist.
-        options[:tests]&.each { |test| require_existing_test(test) }
-        return
-      end
+      Selection::SourceScope.verify_no_specs!(@argv)
+      options[:tests]&.each { |test| require_existing_test(test) }
+      # --test may stand alone: the configured include (or the --since
+      # diff) names the sources. The files must still exist.
+      return if @argv.empty?
 
       @sources = @argv.dup
       options[:include] = @argv.map { |source| source_pattern(source) }
       options[:exclude] = []
-      options[:tests]&.each { |test| require_existing_test(test) }
     end
 
     def source_pattern(source)
@@ -303,12 +336,10 @@ module Saboteur
       raise UsageError, "test file not found: #{test}" unless File.exist?(path)
     end
 
-    def overrides
-      options.except(:config, :id, :suite, :list_operators, :dry_run)
-    end
+    def overrides = options.except(:config, :id, :suite, :list_operators, :dry_run)
 
     def preview
-      raise UsageError, "--dry-run and --list-operators print text, not --agent JSON" if agent?
+      raise UsageError, "--dry-run and --list-operators print text, not --agent JSON; use --format json" if agent?
       raise UsageError, "--id replays one mutant: not with --dry-run or --list-operators" if options[:id]
 
       if options[:list_operators]
@@ -316,24 +347,10 @@ module Saboteur
         return EXIT_OK
       end
 
-      Preview.new(config, out: @out, err: @err).call
+      Preview.new(config, out: @out, err: @err, format: @format).call
       EXIT_OK
     end
 
-    def define_selection_options(opts)
-      opts.on("--operators NAMES", Array, "Choose operator names (default: all)") do |v|
-        set(:operators, v)
-      end
-      opts.on("--exclude-operators NAMES", Array, "Exclude operator names from the selected set") do |v|
-        set(:exclude_operators, v)
-      end
-      opts.on("--list-operators", "List operator names and descriptions") { set(:list_operators, true) }
-      opts.on("--dry-run", "Preview mutants without boot, tests, forks or report writes") { set(:dry_run, true) }
-      opts.on("--id DIGEST", "Replay exactly one current mutant by its full digest") { |v| set(:id, v) }
-      opts.on("--since REF", "Only mutate subjects changed since git REF") { |v| options[:since] = v }
-      opts.on("--subject EXPR", "Klass#meth, Klass.meth or a namespace; repeatable") { |v| add_subject(v) }
-    end
-
     def define_execution_options(opts)
       opts.on("--timeout-factor N", Float, "Multiple of baseline runtime (default 3)") { |v| set(:timeout_factor, v) }
       opts.on("--jobs N", Integer, "Concurrent workers (default 1)") { |v| set(:jobs, v) }
@@ -352,6 +369,10 @@ module Saboteur
       end
       opts.on("--all", "Run the whole configured test suite instead of inferring the specs named after the " \
                        "mutated files") { set(:all, true) }
+      opts.on("--test-command CMD", "Run CMD per mutant in the project root; exit 0 means the mutant survived. " \
+                                    "For apps that cannot load in this Ruby; much slower, serial, no coverage") do |v|
+        set(:test_command, v)
+      end
       opts.on("--[no-]guess-tests", "Deprecated alias: inferred specs are already the default; " \
                                     "--no-guess-tests warns and is ignored - use --all for the whole suite") do |v|
         set(:guess_tests, v)
@@ -368,11 +389,13 @@ module Saboteur
       opts.on("--score-history PATH", "Append score history JSONL (opt-in)") { |v| set(:score_history, v) }
       opts.on("--json PATH", "Stryker JSON report path") { |v| set(:json_report, v) }
       opts.on("--html PATH", "Self-contained HTML report path") { |v| set(:html_report, v) }
-      opts.on("--format FORMAT", %w[text agent], "text (default) or agent: one bounded JSON summary on stdout, " \
-                                                 "strict exit codes") { |v| @format = v }
+      opts.on("--format FORMAT", %w[text agent json], "text, agent or json (--dry-run inventory)") { |v| @format = v }
       opts.on("--agent", "Same as --format agent") { @format = "agent" }
       opts.on("--verbose", "Text report: also print score split, timings and kill causes;",
-              "a crash prints its backtrace") { @verbose = true }
+              "a crash prints its backtrace") do
+        @verbose = true
+        set(:verbose, true)
+      end
       opts.on("--[no-]color", "Colour the text report (default: on for a terminal unless NO_COLOR is set)") do |v|
         @color = v ? :always : :never
       end
@@ -388,12 +411,8 @@ module Saboteur
       Gates.define_options(opts) { |key, value| set(key, value) }
     end
 
-    def add_subject(expr)
-      (options[:subjects] ||= []) << expr
-    end
+    def add_subject(expr) = (options[:subjects] ||= []) << expr
 
-    def set(key, value)
-      options[key] = value
-    end
+    def set(key, value) = (options[key] = value)
   end
 end

data/lib/saboteur/config.rb

@@ -22,13 +22,27 @@ module Saboteur
       "integration" => "rspec",
       "require" => [],
       "include" => ["lib/**/*.rb"],
-      "exclude" => [],
+      # Spec and test files vendored under lib/ (capybara ships its driver
+      # suite at lib/capybara/spec/**) are never mutation subjects (#504).
+      # A config's own c.exclude replaces this list wholesale.
+      "exclude" => ["lib/**/spec/**/*", "lib/**/test/**/*", "lib/**/tests/**/*",
+                    "lib/**/*_spec.rb", "lib/**/*_test.rb", "lib/**/test_*.rb"],
       "arid" => MutationSource::Builtin::DEFAULT_ARID,
       "whole_file" => [],
+      # [{ "pattern" => "app/payments/**/*", "min_score" => 95.0 }], from
+      # repeatable c.score_gate calls (#473). --threshold stays the global
+      # fallback.
+      "score_gates" => [],
       "hazard" => MutationSource::HazardGuard::DEFAULT_HAZARD,
       "allow_hazardous" => false,
-      "sandbox" => true, # true, false, or "strict": strict refuses to run without full Landlock
+      # true, false, "strict" or "seatbelt": strict refuses to run without full Landlock;
+      # seatbelt opts into the macOS Seatbelt sandbox (on Linux it behaves like true).
+      "sandbox" => true,
       "sandbox_writable" => [],
+      # off, survivors, or decisive (#476): one reordered confirmation run
+      # per new survivor (and per fresh kill in decisive); a disagreement
+      # becomes unverified, outside the score.
+      "flaky_check" => "off",
       "timeout_factor" => 3.0,
       "timeout_min" => 5.0,
       "baseline_timeout" => 300.0,
@@ -56,6 +70,7 @@ module Saboteur
       "since" => nil,
       "cache" => true,
       "incremental" => false,
+      "shard" => nil,
       "allow_focus" => false,
       "fail_fast" => false,
       "guess_tests" => false,
@@ -67,10 +82,14 @@ module Saboteur
       "max_duration" => nil,
       "max_duration_ok" => false,
       "tests" => [],
+      "test_command" => nil,
       "fingerprint_extra" => [],
       "threshold" => nil,
       "max_timeouts" => nil,
-      "max_errors" => nil
+      "max_errors" => nil,
+      # --verbose also lands here, so boot-time output filters (the rspec
+      # load-error report, #608) can keep their raw text behind the flag.
+      "verbose" => false
     }.freeze
 
     INTEGER_KEYS = %w[jobs max_memory_mb max_processes max_mutants_per_method seed baseline_passes].freeze
@@ -160,14 +179,30 @@ module Saboteur
       matched = self["include"].flat_map { |pattern| Dir.glob(pattern, base: root) }
       excluded = self["exclude"].flat_map { |pattern| Dir.glob(pattern, base: root) }
       (matched - excluded).select { |file| file.end_with?(".rb") }
-                          .map { |file| File.realpath(File.expand_path(Text.utf8_identity(file), root)) }
+                          .filter_map { |file| real_source_path(file) }
                           .uniq.sort
     end
 
+    # A broken symlink matches Dir.glob but has no real path: dangling
+    # (ENOENT), a loop (ELOOP) or through a regular file (ENOTDIR). It is
+    # not mutation source, so it is skipped rather than crashing the run
+    # (#503). A regular file that cannot be resolved still raises.
+    def real_source_path(file)
+      path = File.expand_path(Text.utf8_identity(file), root)
+      File.realpath(path)
+    rescue SystemCallError
+      raise unless File.symlink?(path)
+    end
+
     # Patterns that match no file. A pattern goes stale when files move;
     # an exclude that silently stops excluding is a hazard, so say so.
     def unmatched_patterns
       %w[include exclude].flat_map do |key|
+        # The default exclude shields (#504) match nothing on a healthy
+        # project - that is their job - so only a configured exclude can go
+        # stale enough to warn about.
+        next [] if key == "exclude" && !explicit?("exclude")
+
         self[key].select { |pattern| Dir.glob(pattern, base: root).empty? }.map { |pattern| "#{key}: #{pattern}" }
       end
     end
@@ -237,6 +272,41 @@ module Saboteur
       validate_matrix!
       validate_test_selection!
       validate_gates!
+      validate_score_gates!
+    end
+
+    # Per-path gates (#473): normalize each entry to string pattern /
+    # float floor, and fail loud on a pattern no included file matches -
+    # a gate that matches nothing would pass vacuously forever.
+    def validate_score_gates!
+      gates = self["score_gates"]
+      return if gates.empty?
+
+      normalized = gates.map { |gate| normalize_score_gate(gate) }
+      prefix = File.join(File.realpath(root), "")
+      relative = included_files.map { |file| file.delete_prefix(prefix) }
+      flags = File::FNM_PATHNAME | File::FNM_EXTGLOB
+      normalized.each do |gate|
+        next if relative.any? { |file| File.fnmatch?(gate["pattern"], file, flags) }
+
+        raise ConfigError, "score_gate #{gate['pattern']} matches no included file"
+      end
+      @values["score_gates"] = normalized
+    end
+
+    def normalize_score_gate(gate)
+      pattern = gate["pattern"].to_s
+      begin
+        floor = Float(gate["min_score"])
+      rescue ArgumentError, TypeError
+        raise ConfigError, "score_gate min_score must be between 0 and 100, got #{gate['min_score'].inspect}"
+      end
+      unless floor.finite? && floor.between?(0, 100)
+        raise ConfigError, "score_gate min_score must be between 0 and 100, got #{gate['min_score'].inspect}"
+      end
+      raise ConfigError, "score_gate pattern must not be empty" if pattern.empty?
+
+      { "pattern" => pattern, "min_score" => floor }
     end
 
     def validate_gates!
@@ -270,6 +340,28 @@ module Saboteur
       raise ConfigError, "test_selection must be one of: auto, all"
     end
 
+    # --test-command replaces the whole in-process machinery, on the
+    # EFFECTIVE config: CLI flags and config-file entries alike (#305).
+    # The command runs unsandboxed, so "strict" cannot be honored.
+    def validate_test_command!
+      command = self["test_command"]
+      # nil alone means unset: false, arrays and numbers are all rejected,
+      # never coerced and never sent to String methods (#305 QA).
+      raise ConfigError, "test_command must be a string" unless command.nil? || command.is_a?(String)
+      raise ConfigError, "test_command must not be blank" if command&.strip&.empty?
+
+      return if command.to_s.empty?
+      raise ConfigError, "test_command cannot be combined with tests" unless self["tests"].empty?
+      raise ConfigError, "test_command cannot be combined with incremental" if self["incremental"]
+      unless self["flaky_check"] == "off"
+        raise ConfigError, "test_command cannot be combined with flaky_check: an external run has no " \
+                           "in-process scheduler to rerun the decisive tests through"
+      end
+      return unless self["sandbox"] == "strict"
+
+      raise ConfigError, 'test_command runs the command unsandboxed, so sandbox "strict" cannot be honored'
+    end
+
     def validate_reeval!
       raise ConfigError, "reeval must be one of: #{Execution::Reevaluator::STRATEGIES.join(', ')}" unless
         Execution::Reevaluator::STRATEGIES.include?(self["reeval"].to_s)
@@ -277,8 +369,11 @@ module Saboteur
 
     def validate!
       raise ConfigError, "jobs must be >= 1" if self["jobs"] < 1
-      unless [true, false, "strict"].include?(self["sandbox"])
-        raise ConfigError, 'sandbox must be true, false, or "strict"'
+      unless [true, false, "strict", "seatbelt"].include?(self["sandbox"])
+        raise ConfigError, 'sandbox must be true, false, "strict", or "seatbelt"'
+      end
+      unless %w[off survivors decisive].include?(self["flaky_check"])
+        raise ConfigError, "flaky_check must be one of: off, survivors, decisive"
       end
 
       validate_mutant_limit!
@@ -292,6 +387,7 @@ module Saboteur
 
       validate_reeval!
       validate_operator_config!
+      validate_test_command!
     end
 
     public

data/lib/saboteur/config_file.rb

@@ -9,23 +9,36 @@ module Saboteur
   # boots, so it cannot reference app constants - plain Ruby and ENV only.
   module ConfigFile
     # The assignment target. One setter per Config key; anything else is an
-    # unknown setting. Records the line of each assignment so validation
-    # errors can point at it.
+    # unknown setting.
     class Settings
       def initialize
         @values = {}
-        @lines = {}
       end
 
-      attr_reader :values, :lines
+      attr_reader :values
 
-      Config::DEFAULTS.each_key do |key|
+      (Config::DEFAULTS.keys - ["integration"]).each do |key|
         define_method("#{key}=") do |value|
           @values[key] = value
-          @lines[key] = caller_locations(1, 1).first.lineno
         end
       end
 
+      def integration=(value)
+        line = caller_locations(1, 1).fetch(0).lineno
+        unless value.is_a?(String)
+          raise ConfigError, "#{line}: integration must be a String, got #{value.inspect}; use \"rspec\""
+        end
+
+        @values["integration"] = value
+      end
+
+      # Repeatable per-path score gates (#473), the one setting that takes
+      # arguments: c.score_gate "app/payments/**/*", min_score: 95. Values
+      # are validated later, on the effective Config.
+      def score_gate(pattern, min_score:)
+        (@values["score_gates"] ||= []) << { "pattern" => pattern, "min_score" => min_score }
+      end
+
       Config::INTEGER_KEYS.each do |key|
         define_method("#{key}=") do |value|
           unless value.is_a?(Integer)
@@ -33,7 +46,6 @@ module Saboteur
             raise ConfigError, "#{line}: #{key} must be an Integer, got #{value.inspect}"
           end
           @values[key] = value
-          @lines[key] = caller_locations(1, 1).first.lineno
         end
       end
     end