saboteur 0.8.0 → 0.9.0

data/lib/saboteur/cli/execution_options.rb

@@ -0,0 +1,46 @@
+# SPDX-License-Identifier: MIT
+# frozen_string_literal: true
+
+module Saboteur
+  class CLI
+    # Execution, cache and hazard flags for `saboteur run` (#334).
+    module ExecutionOptions
+      private
+
+      def define_execution_options(opts)
+        opts.on("--timeout-factor N", Float, "Multiple of baseline runtime (default 3)") { |v| set(:timeout_factor, v) }
+        opts.on("--jobs N", Integer, "Concurrent workers (default 1)") { |v| set(:jobs, v) }
+        add_run_budget_options(opts)
+        opts.on("--config PATH", "Config file (default config/saboteur.rb)") { |v| set(:config, v) }
+        opts.on("--integration NAME", "Test adapter (rspec)") { |v| set(:integration, v) }
+        opts.on("--reeval STRATEGY", Execution::Reevaluator::STRATEGIES, "class_eval or nesting") { |v| set(:reeval, v) }
+        opts.on("--no-cache", "Rebuild the coverage map") { set(:cache, false) }
+        opts.on("--allow-hazardous",
+                "Also run mutants inside destructive calls (rm_rf, system, ...)") { set(:allow_hazardous, true) }
+        opts.on("--fail-fast",
+                "Stop after the first subject with a survivor (needs --jobs 1)") { set(:fail_fast, true) }
+        add_reuse_options(opts)
+        opts.on("--allow-focus", "Run even when a focus filter (fit, fdescribe, :focus) narrows the suite") do
+          set(:allow_focus, true)
+        end
+        opts.on("--all", "Run the whole configured test suite instead of inferring the specs named after the " \
+                         "mutated files") { set(:all, true) }
+        opts.on("--test-command CMD", "Run CMD per mutant in the project root; exit 0 means the mutant survived. " \
+                                      "For apps that cannot load in this Ruby; much slower, serial, no coverage") do |v|
+          set(:test_command, v)
+        end
+        opts.on("--[no-]guess-tests", "Deprecated alias: inferred specs are already the default; " \
+                                      "--no-guess-tests warns and is ignored - use --all for the whole suite") do |v|
+          set(:guess_tests, v)
+        end
+      end
+
+      def add_reuse_options(opts)
+        opts.on("--incremental", "Reuse earlier verdicts whose inputs did not change") { set(:incremental, true) }
+        opts.on("--resume DIR", "Keep verdicts in DIR and reuse them in a later run with the same inputs") do |v|
+          set(:resume, v)
+        end
+      end
+    end
+  end
+end

data/lib/saboteur/cli/exit_status.rb

@@ -14,11 +14,9 @@ module Saboteur
       end
 
       # The crash log lives with the other run state. A broken config (or a
-      # nil cache_dir) must not swallow the crash report itself, so fall
-      # back to no log.
+      # nil cache_dir) must not swallow the crash report itself: no log.
       def crash_log_path
-        dir = config.path("cache_dir")
-        dir && File.join(dir, CrashReport::LOG_NAME)
+        config.path("cache_dir")&.then { |dir| File.join(dir, CrashReport::LOG_NAME) }
       rescue StandardError
         nil
       end
@@ -34,7 +32,8 @@ module Saboteur
 
         return EXIT_TOOL if incomplete_under_threshold?(report)
 
-        failed_quality_gate?(report) || gates.failed?(report, @err) ? EXIT_SURVIVORS : EXIT_OK
+        quality_failed = failed_quality_gate?(report)
+        gates.failed?(report, @err, record_cause: !quality_failed) || quality_failed ? EXIT_SURVIVORS : EXIT_OK
       end
 
       # A spent --max-duration budget exits 4 on its own, before the gates:
@@ -62,7 +61,7 @@ module Saboteur
       def unverifiable_scope?(report)
         if unverifiable_review?(report)
           report.skipped_subjects.each do |subject, reason|
-            @err.puts("saboteur: #{subject.id} was skipped (#{reason}), so the change under test was not verified")
+            @err.puts(subject.skipped_gate_sentence(reason, report.root))
           end
           return true
         end
@@ -86,10 +85,9 @@ module Saboteur
       def mostly_errored?(report)
         return false unless report.majority_errored?
 
-        executed = report.executed
-        errors = executed.count { |result| result.status == :error }
-        @err.puts("saboteur: #{errors} of #{executed.size} mutants errored, so the run tested almost nothing; " \
-                  "fix the errors and run again")
+        errored = report.executed.select { |result| result.status == :error }
+        @err.puts("saboteur: fix the errors and run again\n  #{errored.size} of #{report.executed.size} " \
+                  "mutants errored, so the run tested almost nothing#{ErrorCause.line(errored)}")
         true
       end
 
@@ -159,10 +157,10 @@ module Saboteur
       def incomplete_under_threshold?(report)
         return false if config["threshold"].nil?
 
-        count = report.counts.values_at(*(gates.judges_errors? ? %i[resource_limit] : %i[error resource_limit])).sum
-        @err.puts("saboteur: #{count} mutant(s) errored or hit a resource limit, so the score is incomplete") if
-          count.positive?
-        count.positive?
+        statuses = gates.judges_errors? ? %i[resource_limit] : %i[error resource_limit]
+        problem = report.results.select { |result| statuses.include?(result.status) }
+        @err.puts(ErrorCause.incomplete(problem)) if problem.any?
+        problem.any?
       end
     end
   end

data/lib/saboteur/cli/explicit_file_options.rb

@@ -0,0 +1,21 @@
+# SPDX-License-Identifier: MIT
+# frozen_string_literal: true
+
+module Saboteur
+  class CLI
+    # Explicit test selection and quality-gate flags (#334).
+    module ExplicitFileOptions
+      private
+
+      def define_explicit_file_options(opts)
+        opts.on("--test PATH", "Spec file or directory to run; repeatable. Replaces the configured spec paths") do |v|
+          (options[:tests] ||= []) << v
+        end
+        opts.on("--threshold PERCENT", Float, "Exit 1 when the score is below PERCENT (0-100)") do |v|
+          set(:threshold, v)
+        end
+        Gates.define_options(opts) { |key, value| set(key, value) }
+      end
+    end
+  end
+end

data/lib/saboteur/cli/explicit_sources.rb

@@ -0,0 +1,46 @@
+# SPDX-License-Identifier: MIT
+# frozen_string_literal: true
+
+module Saboteur
+  class CLI
+    # The source and test operands of `saboteur run`, checked against the
+    # file system before anything boots (#334). Extracted from CLI.
+    module ExplicitSources
+      private
+
+      # Positional arguments name the files (or directories) to mutate. They
+      # replace the configured include and exclude patterns. Each must exist,
+      # so a typo is an error and not a run over zero subjects.
+      def use_explicit_sources
+        Selection::SourceScope.verify_no_specs!(@argv)
+        options[:tests]&.each { |test| require_existing_test(test) }
+        # --test may stand alone: the configured include (or the --since
+        # diff) names the sources. The files must still exist.
+        return if @argv.empty?
+
+        @sources = @argv.dup
+        options[:include] = @argv.map { |source| source_pattern(source) }
+        options[:exclude] = []
+      end
+
+      def source_pattern(source)
+        return "#{glob_escape(source.chomp('/'))}/**/*.rb" if File.directory?(source)
+        raise UsageError, "source is not a Ruby file: #{source}" if File.file?(source) && !source.end_with?(".rb")
+        raise UsageError, "source file not found: #{source}" unless File.file?(source)
+
+        glob_escape(source)
+      end
+
+      # include is a list of glob patterns; a named file must match only itself.
+      def glob_escape(path)
+        path.gsub(/[\\*?\[\]{}]/) { |character| "\\#{character}" }
+      end
+
+      # A test may carry an example selector: spec/a_spec.rb:12 or spec/a_spec.rb[1:2].
+      def require_existing_test(test)
+        path = test.sub(/(:\d+|\[[\d:,]+\])+\z/, "")
+        raise UsageError, "test file not found: #{test}" unless File.exist?(path)
+      end
+    end
+  end
+end

data/lib/saboteur/cli/failure_output.rb

@@ -0,0 +1,35 @@
+# SPDX-License-Identifier: MIT
+# frozen_string_literal: true
+
+module Saboteur
+  class CLI
+    # Where a failed command speaks: stderr for people, and in agent mode the
+    # same failure as JSON on stdout (#334). Extracted from CLI.
+    module FailureOutput
+      private
+
+      def usage_error(message = nil)
+        @err.puts(message) if message
+        @err.puts(USAGE)
+        agent_failure(message || USAGE, EXIT_USAGE)
+      end
+
+      def agent? = @format == "agent"
+
+      # In agent mode the failure is also on stdout, where the agent reads its answer.
+      def agent_failure(message, exit_code)
+        @out.print(Reporter::Agent.render(Reporter::Agent.failure(message, exit_code))) if agent?
+        exit_code
+      end
+
+      # Error messages interpolate repository-controlled names, so each line is
+      # made safe before it is printed (a newline then "::error::" in a file
+      # name must not become a workflow command in the CI log).
+      def report_failure(error, exit_code)
+        message = Text.display_safe_lines(error.message)
+        @err.puts("saboteur: #{message}")
+        agent_failure(message, exit_code)
+      end
+    end
+  end
+end

data/lib/saboteur/cli/help_command.rb

@@ -0,0 +1,23 @@
+# SPDX-License-Identifier: MIT
+# frozen_string_literal: true
+
+module Saboteur
+  class CLI
+    # `saboteur help` and the `--help` of every command (#334). Extracted
+    # from CLI by command.
+    module HelpCommand
+      private
+
+      def help_requested? = @argv.intersect?(%w[--help -h])
+
+      def show_help(command_name)
+        help = Help.new(option_lines: option_lines)
+        text = command_name ? help.command(command_name) : help.overview
+        return @out.puts(text) || EXIT_OK if text
+
+        @err.puts("unknown command: #{command_name}. Commands: #{help.command_names.join(', ')}")
+        EXIT_USAGE
+      end
+    end
+  end
+end

data/lib/saboteur/cli/matrix_option.rb

@@ -0,0 +1,17 @@
+# SPDX-License-Identifier: MIT
+# frozen_string_literal: true
+
+module Saboteur
+  class CLI
+    # The --matrix flag for `saboteur run` (#334).
+    module MatrixOption
+      private
+
+      def define_matrix_option(opts)
+        opts.on("--matrix", "Run every covering test per mutant and list blind and redundant tests") do
+          set(:matrix, true)
+        end
+      end
+    end
+  end
+end

data/lib/saboteur/cli/output_options.rb

@@ -0,0 +1,27 @@
+# SPDX-License-Identifier: MIT
+# frozen_string_literal: true
+
+module Saboteur
+  class CLI
+    # Report destinations and text output flags for `saboteur run` (#334).
+    module OutputOptions
+      private
+
+      def define_output_options(opts)
+        opts.on("--score-history PATH", "Append score history JSONL (opt-in)") { |v| set(:score_history, v) }
+        opts.on("--json PATH", "Stryker JSON report path") { |v| set(:json_report, v) }
+        opts.on("--html PATH", "Self-contained HTML report path") { |v| set(:html_report, v) }
+        opts.on("--format FORMAT", %w[text agent json], "text, agent or json (--dry-run inventory)") { |v| @format = v }
+        opts.on("--agent", "Same as --format agent") { @format = "agent" }
+        opts.on("--verbose", "Text report: also print score split, timings and kill causes;",
+                "a crash prints its backtrace") do
+          @verbose = true
+          set(:verbose, true)
+        end
+        opts.on("--[no-]color", "Colour the text report (default: on for a terminal unless NO_COLOR is set)") do |v|
+          @color = v ? :always : :never
+        end
+      end
+    end
+  end
+end

data/lib/saboteur/cli/presets.rb

@@ -14,6 +14,10 @@ module Saboteur
                       --html .saboteur/audit-report.html]
       }.freeze
 
+      # Preset flags that take a separate value token: the pairs a user
+      # flag can override in the banner (#666).
+      VALUE_FLAGS = %w[--baseline-passes --max-duration --since --operators --json --html].freeze
+
       private
 
       def preset_or_usage_error(command)
@@ -31,10 +35,46 @@ module Saboteur
         # deprecated inference alias (#245).
         explicit_scope = @argv.any? { |arg| arg == "--test" || arg.start_with?("--test=") || arg == "--guess-tests" }
         flags.delete("--all") if explicit_scope
-        @err.puts("saboteur: preset '#{name}': #{flags.join(' ')} (explicit flags override)")
+        @err.puts("saboteur: preset '#{name}': #{preset_banner(flags).join(' ')} (explicit flags override)")
         @argv = flags + @argv
         run_command
       end
+
+      # The banner is the reader's only record of the effective settings,
+      # so a preset flag the user also passed shows the user's spelling -
+      # the later occurrence wins (#666). Flags the user did not override
+      # print as the preset declares them.
+      def preset_banner(flags)
+        overrides = user_flag_overrides
+        banner = []
+        index = 0
+        while index < flags.size
+          if VALUE_FLAGS.include?(flags[index])
+            banner.concat(overrides.fetch(flags[index], [flags[index], flags[index + 1]]))
+            index += 2
+          else
+            banner << flags[index]
+            index += 1
+          end
+        end
+        banner
+      end
+
+      # Each long flag the user passed, mapped to its effective spelling:
+      # the last occurrence, verbatim, with its separate value token when
+      # the flag takes one.
+      def user_flag_overrides
+        overrides = {}
+        @argv.each_with_index do |arg, index|
+          name = arg[/\A--[^=]+/] or next
+          if arg.include?("=")
+            overrides[name] = [arg]
+          elsif VALUE_FLAGS.include?(name) && @argv[index + 1]
+            overrides[name] = [name, @argv[index + 1]]
+          end
+        end
+        overrides
+      end
     end
   end
 end

data/lib/saboteur/cli/preview_command.rb

@@ -0,0 +1,26 @@
+# SPDX-License-Identifier: MIT
+# frozen_string_literal: true
+
+module Saboteur
+  class CLI
+    # `run --dry-run` and `run --list-operators`: plan output, no mutations
+    # (#334). Extracted from CLI by command.
+    module PreviewCommand
+      private
+
+      def preview
+        raise UsageError, "--dry-run and --list-operators print text, not --agent JSON; use --format json" if agent?
+        raise UsageError, "--id replays one mutant: not with --dry-run or --list-operators" if options[:id]
+
+        if options[:list_operators]
+          MutationSource::OperatorRegistry.list(@out, names: config["operators"],
+                                                      excluded: config["exclude_operators"])
+          return EXIT_OK
+        end
+
+        Preview.new(config, out: @out, err: @err, format: @format).call
+        EXIT_OK
+      end
+    end
+  end
+end

data/lib/saboteur/cli/run_budget_options.rb

@@ -13,8 +13,13 @@ module Saboteur
         opts.on("--seed N", Integer, "Seed for Ruby's random numbers in every test run (default 1)") do |v|
           set(:seed, v)
         end
+        opts.on("--coverage-parts N", Integer, "Build the coverage map in N parallel parts (default 1)") do |v|
+          set(:coverage_parts, v)
+        end
         opts.on("--baseline-passes N", Integer, "Unchanged-suite checks (default 2)") { |v| set(:baseline_passes, v) }
-        opts.on("--max-duration DURATION", "Wall-clock budget for the whole run, e.g. 10m; exit 4 when spent") do |v|
+        opts.on("--max-duration DURATION", "Wall-clock budget for the whole run, e.g. 10m; exit 4 when spent - " \
+                                           "the clock starts at engine entry, so boot, baselines and the " \
+                                           "coverage map spend it too") do |v|
           set(:max_duration, parse_duration(v))
         end
         opts.on("--[no-]max-duration-ok", "An exhausted --max-duration budget does not fail the run") do |v|

data/lib/saboteur/cli/run_command.rb

@@ -0,0 +1,33 @@
+# SPDX-License-Identifier: MIT
+# frozen_string_literal: true
+
+module Saboteur
+  class CLI
+    # `saboteur run`: parse, validate, then preview or execute (#334).
+    # Extracted from CLI by command.
+    module RunCommand
+      private
+
+      def run_command = help_requested? ? show_help("run") : run
+
+      def run
+        parser.parse!(@argv)
+        reject_test_selection_conflicts
+        use_explicit_sources
+        validate_run_options
+        warn_dirty_tree
+
+        config.deprecation_warnings.each { |warning| @err.puts("saboteur: #{warning}") }
+
+        return preview if options[:list_operators] || options[:dry_run]
+
+        refuse_empty_source_set
+        execute_run
+      rescue ConfigError => e
+        report_failure(e, EXIT_USAGE)
+      rescue Error => e
+        report_failure(e, EXIT_TOOL)
+      end
+    end
+  end
+end

data/lib/saboteur/cli/run_output.rb

@@ -0,0 +1,38 @@
+# SPDX-License-Identifier: MIT
+# frozen_string_literal: true
+
+module Saboteur
+  class CLI
+    # Where a run's own output goes: the report names it prints and the
+    # stdout redirect that keeps agent JSON clean (#334). Extracted from CLI.
+    module RunOutput
+      private
+
+      # One line naming the reports written, paths relative to the root when
+      # they sit under it (a blank line before it, as the old JSON line had).
+      def announce_reports(reports)
+        return if (written = reports.select { |path, _label| path }).empty?
+
+        root = File.join(File.realpath(config.root), "")
+        names = written.map do |path, label|
+          shown = File.expand_path(path).delete_prefix(root)
+          "#{shown} (#{label})"
+        end
+        @out.puts "\nReports: #{names.join(', ')}"
+      end
+
+      # Agent mode: stdout carries exactly one JSON payload. The target's own output (boot
+      # files, hooks, spec loading, child processes) goes to stderr while the engine runs.
+      # Descriptor 1 points at stderr for the block, so Kernel#puts, system and forked
+      # children cannot write into the JSON on stdout.
+      def stdout_on_stderr
+        saved = $stdout.dup
+        $stdout.reopen($stderr)
+        yield
+      ensure
+        $stdout.reopen(saved) if saved
+        saved&.close
+      end
+    end
+  end
+end

data/lib/saboteur/cli/run_reporting.rb

@@ -30,7 +30,7 @@ module Saboteur
       end
 
       def agent_run(arguments)
-        tree = WorkingTree.new(config.root, ignored: config.output_paths).state
+        tree = WorkingTree.new(config.root, ignored: config.ignored_tree_paths).state
         engine = Engine.new(config, **arguments)
         report = stdout_on_stderr { engine.call }
         payload = Reporter::Agent.call(report, agent_provenance(report, tree, engine))
@@ -65,7 +65,7 @@ module Saboteur
       # Computed lazily: only a run with survivors asks for it.
       def survivor_replay_command(report, engine)
         lambda do |digest|
-          tree = WorkingTree.new(config.root, ignored: config.output_paths).state
+          tree = WorkingTree.new(config.root, ignored: config.ignored_tree_paths).state
           Reporter::Agent::Commands.replay(agent_provenance(report, tree, engine), digest, capped: false)
         end
       end
@@ -82,7 +82,8 @@ module Saboteur
         path = config.path("json_report")
         return unless path
 
-        Reporter::Stryker.new(path).call(report, exit_cause: cause)
+        Reporter::Stryker.new(path).call(report, exit_cause: cause,
+                                                 score_gates: gates.score_gate_breakdowns(report))
         path
       rescue SystemCallError, IOError => e
         if config.explicit?("json_report")
@@ -124,8 +125,8 @@ module Saboteur
         path = config.path("score_history")
         return unless path
 
-        tree = WorkingTree.new(config.root, ignored: config.output_paths).state
-        scope = config.selection_settings.except("json_report", "html_report", "score_history", "cache_dir")
+        tree = WorkingTree.new(config.root, ignored: config.ignored_tree_paths).state
+        scope = config.selection_settings.except(*Config::OUTPUT_KEYS)
         # Recording is an auxiliary artifact: when the file cannot be
         # written - read-only tree inside a sandboxed child (#296) - warn
         # and let the run exit by its results instead of failing as a tool

data/lib/saboteur/cli/run_validation.rb

@@ -0,0 +1,54 @@
+# SPDX-License-Identifier: MIT
+# frozen_string_literal: true
+
+module Saboteur
+  class CLI
+    # The usage errors `saboteur run` finds before the target boots: option
+    # combinations that contradict each other (#334). Extracted from CLI.
+    module RunValidation
+      private
+
+      # Contradictory selection flags are a usage error before any file or
+      # repository check: --test wins over inference by precedence, but only
+      # one of them may be named. Config-file combinations are not errors:
+      # --test wins there, so generated commands stay valid over any config.
+      def reject_test_selection_conflicts
+        raise UsageError, "--all cannot be combined with --test" if options[:all] && options[:tests]&.any?
+
+        reject_test_command_conflicts
+        raise UsageError, "--all cannot be combined with --guess-tests" if options[:all] && options[:guess_tests]
+        return unless options[:guess_tests] && options[:tests]&.any?
+
+        raise UsageError, "--guess-tests cannot be combined with --test"
+      end
+
+      # --test-command replaces the whole in-process test machinery: test
+      # selection, coverage, parallelism and the sandbox do not apply.
+      def reject_test_command_conflicts
+        return unless options[:test_command]
+
+        # options[:tests] is nil or a non-empty list, so truthiness is the check.
+        raise UsageError, "--test-command cannot be combined with --test" if options[:tests]
+        raise UsageError, "--test-command cannot be combined with --incremental" if options[:incremental]
+      end
+
+      # Usage errors found before the target boots.
+      def validate_run_options
+        reject_format_conflicts
+        validate_shard_options
+        Selection::GitDiff.verify_ref!(config["since"], root: config.root) if config["since"]
+      end
+
+      # Agent mode allows no score gates (one strict exit 0); the json inventory is --dry-run planning only.
+      def reject_format_conflicts
+        if agent? && (config["threshold"] || gates.any?)
+          raise UsageError, "#{config['threshold'] ? '--threshold' : gates.agent_option_name} " \
+                            "cannot be combined with --agent"
+        end
+        return unless @format == "json" && !options[:dry_run]
+
+        raise UsageError, "--format json is the --dry-run mutant inventory; a run summary is --format agent"
+      end
+    end
+  end
+end

data/lib/saboteur/cli/source_set.rb

@@ -18,24 +18,6 @@ module Saboteur
         raise UsageError, empty_source_set_message
       end
 
-      # A --since ref whose resolved range diffs empty yields 0 subjects
-      # and a green run over nothing: `saboteur pr <merge_sha>` on a
-      # checkout AT the merge diffs the merge against itself (#643).
-      # Refuse like a typo'd ref: exit 2, name the base and the range.
-      def refuse_empty_since_range
-        ref = config["since"]
-        return unless ref
-
-        paths = Selection::GitDiff.changed_paths(ref, root: config.root)
-        return unless paths.values.all?(&:empty?)
-
-        base = Selection::GitDiff.merge_base(ref, root: config.root)
-        raise UsageError, "--since #{ref}: the resolved range #{base}..the working tree has no changes - " \
-                          "the run would pass with 0 subjects. A checkout AT the ref (the CI merge-sha " \
-                          "checkout) diffs it against itself: fetch the merge's parent, or point --since " \
-                          "at the base of the change under test"
-      end
-
       def empty_source_set_message
         include_patterns = config["include"]
         matched = include_patterns.flat_map { |pattern| Dir.glob(pattern, base: config.root) }