saboteur 0.2.0 → 0.3.0

data/lib/saboteur/cli.rb

@@ -3,10 +3,16 @@
 
 require "optparse"
 require_relative "../saboteur"
+require_relative "doctor"
+require_relative "help"
+require_relative "reporter/replay"
 
 module Saboteur
   class CLI
-    USAGE = "Usage: saboteur run [--since REF] [--subject 'Klass#meth'] [--timeout-factor N] [--jobs N] [--seed N]"
+    USAGE = "Usage: saboteur run [SOURCE...] [--test TEST...] [--threshold PERCENT] [--since REF] " \
+            "[--subject 'Klass#meth'] [--timeout-factor N] [--jobs N] [--seed N] [--id DIGEST]
+" \
+            "Run 'saboteur help' for every command, option, config key and exit code."
 
     EXIT_OK = 0
     EXIT_SURVIVORS = 1
@@ -26,8 +32,11 @@ module Saboteur
 
     def call
       case @argv.shift
-      when "run" then run
+      when "run" then help_requested? ? show_help("run") : run
+      when "doctor" then help_requested? ? show_help("doctor") : doctor
       when "version", "--version" then @out.puts(VERSION) || EXIT_OK
+      when "help" then show_help(@argv.shift)
+      when "--help", "-h" then show_help(nil)
       else usage_error
       end
     rescue OptionParser::ParseError => e
@@ -38,8 +47,44 @@ module Saboteur
       @config ||= Config.load(root: Dir.pwd, path: options[:config], overrides: overrides)
     end
 
+    def parser
+      OptionParser.new do |opts|
+        opts.banner = USAGE
+        define_selection_options(opts)
+        define_explicit_file_options(opts)
+        define_execution_options(opts)
+        define_output_options(opts)
+      end
+    end
+
+    # Each command's own option lines, as printed under OPTIONS in `saboteur help`.
+    def option_lines
+      { "run" => parser.summarize, "doctor" => doctor_parser.summarize }
+    end
+
+    def doctor_parser
+      OptionParser.new do |opts|
+        opts.banner = "Usage: saboteur doctor [--config PATH] [--seed N]"
+        opts.on("--config PATH", "Config file (default .saboteur.yml)") { |value| set(:config, value) }
+        opts.on("--seed N", Integer, "Seed for the two baseline runs (default 1)") { |value| set(:seed, value) }
+      end
+    end
+
     private
 
+    def help_requested?
+      @argv.intersect?(%w[--help -h])
+    end
+
+    def show_help(command_name)
+      help = Help.new(option_lines: option_lines)
+      text = command_name ? help.command(command_name) : help.overview
+      return @out.puts(text) || EXIT_OK if text
+
+      @err.puts("unknown command: #{command_name}. Commands: #{help.command_names.join(', ')}")
+      EXIT_USAGE
+    end
+
     def usage_error(message = nil)
       @err.puts(message) if message
       @err.puts(USAGE)
@@ -48,12 +93,16 @@ module Saboteur
 
     def run
       parser.parse!(@argv)
-      raise UsageError, "unexpected arguments: #{@argv.join(' ')}" if @argv.any?
+      use_explicit_sources
 
-      report = Engine.new(config, err: @err).call
+      arguments = { err: @err }
+      arguments[:digest] = options[:id] if options[:id]
+      report = Engine.new(config, **arguments).call
       @err.puts
       Reporter::Terminal.new(@out).call(report)
+      Reporter::Replay.new(@out).call(report) if options[:id]
       write_json(report)
+      write_html(report)
       exit_code(report)
     rescue ConfigError => e
       @err.puts("saboteur: #{e.message}")
@@ -63,6 +112,20 @@ module Saboteur
       EXIT_TOOL
     end
 
+    def doctor
+      doctor_parser.parse!(@argv)
+      raise UsageError, "unexpected arguments: #{@argv.join(' ')}" if @argv.any?
+
+      Doctor.new(config, out: @out).call
+      EXIT_OK
+    rescue ConfigError => e
+      @err.puts("saboteur: #{e.message}")
+      EXIT_USAGE
+    rescue Error => e
+      @err.puts("saboteur: #{e.message}")
+      EXIT_TOOL
+    end
+
     def write_json(report)
       path = config.path("json_report")
       return unless path
@@ -71,33 +134,117 @@ module Saboteur
       @out.puts "\nJSON report: #{path}"
     end
 
+    def write_html(report)
+      path = config.path("html_report")
+      return unless path
+
+      Reporter::Html.new(path).call(report)
+      @out.puts "HTML report: #{path}"
+    end
+
     def exit_code(report)
       executed = report.executed
       return EXIT_TOOL if executed.any? && executed.all? { |result| result.status == :error }
 
-      report.unignored_survivors.any? ? EXIT_SURVIVORS : EXIT_OK
+      return EXIT_TOOL if incomplete_under_threshold?(report)
+
+      failed_quality_gate?(report) ? EXIT_SURVIVORS : EXIT_OK
+    end
+
+    # With --threshold the score decides; without it any unignored survivor does.
+    def failed_quality_gate?(report)
+      threshold = config["threshold"]
+      return report.unignored_survivors.any? if threshold.nil?
+
+      score = report.score
+      return missing_score_reported? if score.nil?
+
+      score * 100 < threshold
+    end
+
+    # A gate that cannot measure a score must not pass: CI would go green
+    # on a run that tested nothing.
+    def missing_score_reported?
+      @err.puts("saboteur: no mutant was tested, so there is no score to compare with --threshold")
+      true
+    end
+
+    # Errors and resource limits are outside the score. With a threshold they
+    # make the run incomplete, so the score alone cannot pass it.
+    def incomplete_under_threshold?(report)
+      return false if config["threshold"].nil?
+
+      count = report.counts.values_at(:error, :resource_limit).sum
+      @err.puts("saboteur: #{count} mutant(s) errored or hit a resource limit, so the score is incomplete") if
+        count.positive?
+      count.positive?
+    end
+
+    # Positional arguments name the files (or directories) to mutate. They
+    # replace the configured include and exclude patterns. Each must exist,
+    # so a typo is an error and not a run over zero subjects.
+    def use_explicit_sources
+      raise UsageError, "--test needs at least one source file" if options[:tests] && @argv.empty?
+      return if @argv.empty?
+
+      options[:include] = @argv.map { |source| source_pattern(source) }
+      options[:exclude] = []
+      options[:tests]&.each { |test| require_existing_test(test) }
+    end
+
+    def source_pattern(source)
+      return "#{glob_escape(source.chomp('/'))}/**/*.rb" if File.directory?(source)
+      raise UsageError, "source is not a Ruby file: #{source}" if File.file?(source) && !source.end_with?(".rb")
+      raise UsageError, "source file not found: #{source}" unless File.file?(source)
+
+      glob_escape(source)
+    end
+
+    # include is a list of glob patterns; a named file must match only itself.
+    def glob_escape(path)
+      path.gsub(/[\\*?\[\]{}]/) { |character| "\\#{character}" }
+    end
+
+    # A test may carry an example selector: spec/a_spec.rb:12 or spec/a_spec.rb[1:2].
+    def require_existing_test(test)
+      path = test.sub(/(:\d+|\[[\d:,]+\])+\z/, "")
+      raise UsageError, "test file not found: #{test}" unless File.exist?(path)
     end
 
     def overrides
-      options.except(:config)
+      options.except(:config, :id)
     end
 
-    def parser
-      OptionParser.new do |opts|
-        opts.banner = USAGE
-        opts.on("--since REF", "Only mutate subjects changed since git REF") { |v| options[:since] = v }
-        opts.on("--subject EXPR", "Klass#meth, Klass.meth or a namespace; repeatable") { |v| add_subject(v) }
-        opts.on("--timeout-factor N", Float, "Multiple of baseline runtime (default 3)") { |v| set(:timeout_factor, v) }
-        opts.on("--jobs N", Integer, "Concurrent workers (default 1)") { |v| set(:jobs, v) }
-        opts.on("--seed N", Integer, "Seed for Ruby's random numbers in every test run (default 1)") do |v|
-          set(:seed, v)
-        end
-        opts.on("--config PATH", "Config file (default .saboteur.yml)") { |v| set(:config, v) }
-        opts.on("--integration NAME", "Test adapter (rspec)") { |v| set(:integration, v) }
-        opts.on("--reeval STRATEGY", Execution::Reevaluator::STRATEGIES, "class_eval or nesting") { |v| set(:reeval, v) }
-        opts.on("--json PATH", "Stryker JSON report path") { |v| set(:json_report, v) }
-        opts.on("--no-cache", "Rebuild the coverage map") { set(:cache, false) }
-        opts.on("--incremental", "Reuse earlier verdicts whose inputs did not change") { set(:incremental, true) }
+    def define_selection_options(opts)
+      opts.on("--id DIGEST", "Replay exactly one current mutant by its full digest") { |v| set(:id, v) }
+      opts.on("--since REF", "Only mutate subjects changed since git REF") { |v| options[:since] = v }
+      opts.on("--subject EXPR", "Klass#meth, Klass.meth or a namespace; repeatable") { |v| add_subject(v) }
+    end
+
+    def define_execution_options(opts)
+      opts.on("--timeout-factor N", Float, "Multiple of baseline runtime (default 3)") { |v| set(:timeout_factor, v) }
+      opts.on("--jobs N", Integer, "Concurrent workers (default 1)") { |v| set(:jobs, v) }
+      opts.on("--seed N", Integer, "Seed for Ruby's random numbers in every test run (default 1)") do |v|
+        set(:seed, v)
+      end
+      opts.on("--config PATH", "Config file (default .saboteur.yml)") { |v| set(:config, v) }
+      opts.on("--integration NAME", "Test adapter (rspec)") { |v| set(:integration, v) }
+      opts.on("--reeval STRATEGY", Execution::Reevaluator::STRATEGIES, "class_eval or nesting") { |v| set(:reeval, v) }
+      opts.on("--no-cache", "Rebuild the coverage map") { set(:cache, false) }
+      opts.on("--incremental", "Reuse earlier verdicts whose inputs did not change") { set(:incremental, true) }
+    end
+
+    def define_output_options(opts)
+      opts.on("--json PATH", "Stryker JSON report path") { |v| set(:json_report, v) }
+      opts.on("--html PATH", "Self-contained HTML report path") { |v| set(:html_report, v) }
+    end
+
+    def define_explicit_file_options(opts)
+      opts.on("--test PATH", "Spec file or directory to run; repeatable. Replaces the configured spec paths") do |v|
+        (options[:tests] ||= []) << v
+      end
+      opts.on("--threshold PERCENT", Float, "Exit 1 when the score is below PERCENT (0-100)") do |v|
+        set(:threshold, v)
       end
     end
 

data/lib/saboteur/config.rb

@@ -21,6 +21,7 @@ module Saboteur
       "coverage_timeout" => 3600.0,
       "max_memory_mb" => 2048,
       "max_processes" => 64,
+      "max_mutants_per_method" => 100,
       "jobs" => 1,
       "ignore_file" => ".saboteur-ignore.yml",
       "sources" => ["builtin"],
@@ -29,17 +30,22 @@ module Saboteur
       "reeval" => "nesting",
       "cache_dir" => ".saboteur",
       "json_report" => ".saboteur/report.json",
+      "html_report" => ".saboteur/report.html",
       "rspec_args" => [],
       "subjects" => [],
       "since" => nil,
       "cache" => true,
       "incremental" => false,
-      "seed" => 1
+      "seed" => 1,
+      "tests" => [],
+      "threshold" => nil
     }.freeze
 
-    INTEGER_KEYS = %w[jobs max_memory_mb max_processes seed].freeze
+    INTEGER_KEYS = %w[jobs max_memory_mb max_processes max_mutants_per_method seed].freeze
     FLOAT_KEYS = %w[timeout_factor timeout_min baseline_timeout coverage_timeout].freeze
-    LIST_KEYS = %w[require include exclude arid sources hooks rspec_args subjects].freeze
+    THRESHOLD_RANGE = (0.0..100.0)
+
+    LIST_KEYS = %w[require include exclude arid sources hooks rspec_args subjects tests].freeze
 
     attr_reader :root
 
@@ -70,6 +76,7 @@ module Saboteur
       @values = DEFAULTS.merge(values)
       coerce!
       validate!
+      validate_report_paths!
     end
 
     def [](key)
@@ -111,19 +118,46 @@ module Saboteur
       INTEGER_KEYS.each { |key| @values[key] = Integer(@values[key]) }
       FLOAT_KEYS.each { |key| @values[key] = Float(@values[key]) }
       LIST_KEYS.each { |key| @values[key] = Array(@values[key]).map(&:to_s) }
+      @values["threshold"] = Float(@values["threshold"]) unless @values["threshold"].nil?
     rescue ArgumentError, TypeError => e
       raise ConfigError, "invalid config value: #{e.message}"
     end
 
+    def validate_mutant_limit!
+      raise ConfigError, "max_mutants_per_method must be > 0" unless self["max_mutants_per_method"].positive?
+    end
+
+    def validate_report_paths!
+      json = path("json_report")
+      return unless json && json == path("html_report")
+
+      raise ConfigError, "json_report and html_report both point to #{json}; give each report its own path"
+    end
+
+    def validate_threshold!
+      threshold = self["threshold"]
+      return if threshold.nil? || THRESHOLD_RANGE.cover?(threshold)
+
+      raise ConfigError, "threshold must be between 0 and 100"
+    end
+
+    def validate_reeval!
+      raise ConfigError, "reeval must be one of: #{Execution::Reevaluator::STRATEGIES.join(', ')}" unless
+        Execution::Reevaluator::STRATEGIES.include?(self["reeval"].to_s)
+    end
+
     def validate!
       raise ConfigError, "jobs must be >= 1" if self["jobs"] < 1
+
+      validate_mutant_limit!
+      validate_threshold!
       raise ConfigError, "seed must be >= 0" if self["seed"].negative?
       raise ConfigError, "max_memory_mb must be > 0" unless self["max_memory_mb"].positive?
       raise ConfigError, "max_processes must be > 0" unless self["max_processes"].positive?
       raise ConfigError, "timeout_factor must be > 0" unless self["timeout_factor"].positive?
       raise ConfigError, "timeout_min must be > 0" unless self["timeout_min"].positive?
-      raise ConfigError, "reeval must be one of: #{Execution::Reevaluator::STRATEGIES.join(', ')}" unless
-        Execution::Reevaluator::STRATEGIES.include?(self["reeval"].to_s)
+
+      validate_reeval!
 
       self["arid"].each { |pattern| Regexp.new(pattern) }
     rescue RegexpError => e

data/lib/saboteur/doctor.rb

@@ -0,0 +1,72 @@
+# SPDX-License-Identifier: MIT
+# frozen_string_literal: true
+
+require "coverage"
+require_relative "baseline_health"
+
+module Saboteur
+  # Reports the current environment and runs baseline checks, never mutations.
+  class Doctor
+    def initialize(config, out: $stdout)
+      @config = config
+      @out = out
+    end
+
+    def call
+      Engine::Boot.refuse_recursion!
+      check_runtime
+      check_coverage
+      check_boot_files
+      hooks = Hooks.new
+      integration = boot_target(hooks)
+      @out.puts "Integration: #{@config['integration']} (loaded)"
+      @out.puts "Rails: #{@config['rails'] ? 'enabled' : 'disabled'}"
+      @out.puts "SimpleCov: #{defined?(::SimpleCov) ? 'present; start suppressed for this process' : 'not loaded'}"
+      check_coverage
+      tests = BaselineHealth.new(@config, integration: integration, hooks: hooks).call
+      @out.puts "Baseline: #{tests.size} tests passed twice in the same order with seed #{@config['seed']}"
+      @out.puts "Healthy in these two runs. This does not prove the suite is free of flaky tests."
+      true
+    end
+
+    private
+
+    def boot_target(hooks)
+      Engine::Boot.new(@config, hooks).load_target
+    rescue LoadError, SyntaxError, SystemExit, StandardError => e
+      raise Error, "target boot failed: #{e.class}: #{e.message}; check require files, hooks and rspec_args"
+    end
+
+    def check_runtime
+      @out.puts "Ruby: #{RUBY_DESCRIPTION}"
+      unless Gem::Version.new(RUBY_VERSION) >= Gem::Version.new("3.3") && Process.respond_to?(:fork)
+        raise Error, "doctor needs Ruby 3.3+ with fork; use supported Ruby on Linux or macOS"
+      end
+      return if @config["integration"] == "rspec"
+
+      raise ConfigError,
+            "doctor supports the rspec integration; set integration: rspec"
+    end
+
+    def check_coverage
+      state = Coverage.state
+      @out.puts "Coverage: #{state}"
+      return if Coverage.supported?(:lines) && state == :suspended
+
+      raise Error, "Coverage must be suspended with line support before boot; use the saboteur executable, " \
+                   "disable SimpleCov and remove other Coverage.start calls"
+    end
+
+    def check_boot_files
+      %w[require hooks].each do |key|
+        @config[key].each do |file|
+          path = File.expand_path(file, @config.root)
+          raise ConfigError, "#{key} file missing: #{file}; fix .saboteur.yml" unless File.file?(path)
+
+          @out.puts "#{key}: #{file} (found)"
+        end
+      end
+      @config.unmatched_patterns.each { |pattern| @out.puts "Warning: #{pattern} matches no file" }
+    end
+  end
+end

data/lib/saboteur/engine/baseline_guard.rb

@@ -0,0 +1,99 @@
+# SPDX-License-Identifier: MIT
+# frozen_string_literal: true
+
+require "json"
+require "fileutils"
+
+module Saboteur
+  class Engine
+    # Two fresh children check the unchanged suite before any cached verdict
+    # or mutation source is used. This catches observed instability, not all
+    # possible flaky behaviour.
+    class BaselineGuard
+      def initialize(config:, integration:, hooks:, supervisor:, diagnostics:)
+        @config = config
+        @integration = integration
+        @hooks = hooks
+        @supervisor = supervisor
+        @diagnostics = diagnostics
+      end
+
+      def call
+        expected = @integration.all_tests.map(&:id)
+        raise Error, "baseline has no tests. Check rspec_args and the suite filters before running saboteur again." if
+          expected.empty?
+
+        @diagnostics.inform("checking the unchanged suite twice (seed #{@config['seed']})")
+        previous = nil
+        2.times do |index|
+          number = index + 1
+          data = check(number)
+          unless data["passed"] && data["failed"].empty?
+            refuse(number,
+                   "the suite failed: #{Array(data['failed']).join(', ')}")
+          end
+          executed = data["executed"]
+          refuse(number, "executed tests do not match the loaded suite") unless
+            executed.sort == expected.sort
+          refuse(number, "test order changed between baseline checks") if previous && executed != previous
+          previous = executed
+        end
+        nil
+      end
+
+      private
+
+      def check(number)
+        Execution::ForkReadiness.prepare_heap
+        result = @supervisor.run(timeout: @config["coverage_timeout"]) { |writer| record(writer, number) }
+        refuse(number, "the suite stopped: #{result.stopped_by} (#{result.detail})") if result.stopped_by
+        refuse(number, "the child did not report a result") if result.payload.empty?
+        refuse(number, "the child exited unsuccessfully") if
+          result.exit_status.is_a?(Process::Status) && !result.exit_status.success?
+        refuse(number, "the suite left child processes behind") if result.leftover_pids.any?
+        parse_result(number, result.payload)
+      end
+
+      def parse_result(number, payload)
+        data = JSON.parse(payload)
+        refuse(number, "the child reported an invalid result") unless data.is_a?(Hash)
+        refuse(number, data["error"]) if data["error"]
+        refuse(number, "the child reported invalid test results") unless valid_result?(data)
+        data
+      rescue JSON::ParserError
+        refuse(number, "the child reported invalid JSON")
+      end
+
+      def valid_result?(data)
+        [true, false].include?(data["passed"]) && data["failed"].is_a?(Array) &&
+          data["executed"].is_a?(Array) && data["executed"].all?(String)
+      end
+
+      def record(writer, number)
+        path = log_path(number)
+        FileUtils.mkdir_p(File.dirname(path))
+        $stdout.reopen(path, "w")
+        $stdout.sync = true
+        $stderr.reopen($stdout)
+        @hooks.run(:after_fork)
+        Kernel.srand(@config["seed"])
+        result = @integration.run_baseline(seed: @config["seed"])
+        writer.write(JSON.generate(passed: result.passed, failed: result.failed_ids, executed: result.executed_ids))
+      rescue Exception => e # rubocop:disable Lint/RescueException
+        writer.write(JSON.generate(error: "#{e.class}: #{e.message}"))
+      ensure
+        writer.close
+      end
+
+      def log_path(number)
+        File.join(@config.path("cache_dir"), "baseline-#{number}.log")
+      end
+
+      def refuse(number, reason)
+        raise Error, "baseline check #{number} failed at seed #{@config['seed']}: #{reason}. " \
+                     "See the log at #{log_path(number)}. Fix the suite or its setup, then run saboteur again " \
+                     "with the same seed. No mutants were started."
+      end
+    end
+  end
+end

data/lib/saboteur/engine/candidate_selection.rb

@@ -0,0 +1,35 @@
+# SPDX-License-Identifier: MIT
+# frozen_string_literal: true
+
+module Saboteur
+  class Engine
+    # Selects a bounded list for each covered subject before workers fork.
+    # Subject skips are not mutants and do not enter the mutation score.
+    class CandidateSelection
+      attr_reader :skipped_subjects
+
+      def initialize(candidates:, map:, limit:, diagnostics:)
+        @candidates = candidates
+        @map = map
+        @limit = limit
+        @diagnostics = diagnostics
+        @skipped_subjects = {}
+      end
+
+      def for(subject, scope: nil)
+        if @map.tests_for(subject).empty?
+          @skipped_subjects[subject] = "no coverage"
+          return []
+        end
+
+        list = @candidates.for(subject)
+        list = scope.restrict(subject, list) if scope
+        omitted = list.size - @limit
+        if omitted.positive?
+          @diagnostics.warn("#{subject.id}: omitted #{omitted} mutant(s) above max_mutants_per_method (#{@limit})")
+        end
+        list.first(@limit)
+      end
+    end
+  end
+end

data/lib/saboteur/engine/coverage_store.rb

@@ -11,7 +11,7 @@ module Saboteur
     # unchanged, otherwise built by running the whole suite once in a
     # supervised child and saved.
     class CoverageStore
-      CACHE_INPUTS = %w[integration require rspec_args rails].freeze
+      CACHE_INPUTS = %w[integration require rspec_args rails tests].freeze
       LOCKFILES = %w[Gemfile.lock gems.locked].freeze
       LOG = "coverage-build.log"
 
@@ -53,6 +53,9 @@ module Saboteur
         end
 
         @diagnostics.inform("building coverage map (runs the whole suite once)")
+        # The suite child forked next inherits this heap; pack it first, to
+        # reduce later copying of shared pages.
+        Execution::ForkReadiness.prepare_heap
         result = @supervisor.run(timeout: @config["coverage_timeout"]) { |writer| record(writer, files) }
         raise Error, "coverage map build stopped: #{result.detail}" if result.stopped_by
         raise Error, "coverage map build failed: #{died(result.exit_status)}" if result.payload.empty?
@@ -86,7 +89,11 @@ module Saboteur
         recorder = Selection::Recorder.new(files)
         recorder.start
         Kernel.srand(@config["seed"])
-        @integration.run_all(recorder)
+        passed = @integration.run_all(recorder)
+        if passed == false
+          raise Error, "the unchanged suite failed while building coverage. Fix the suite, then run again " \
+                       "with seed #{@config['seed']}."
+        end
         build_seconds = Process.clock_gettime(Process::CLOCK_MONOTONIC) - started
         writer.write(recorder.result(build_seconds: build_seconds).to_json)
         writer.close

data/lib/saboteur/engine/distribution.rb

@@ -42,7 +42,8 @@ module Saboteur
           "position" => list.index { |mutation, _| mutation.equal?(result.mutation) },
           "status" => result.status.to_s, "reason" => result.reason, "runtime" => result.runtime,
           "killed_by" => result.killed_by, "covered_by" => result.covered_by,
-          "kill_cause" => result.kill_cause, "reused" => result.reused?
+          "kill_cause" => result.kill_cause, "reused" => result.reused?,
+          "rejected_kill" => result.rejected_kill?
         }
       end
 
@@ -55,7 +56,8 @@ module Saboteur
           result = MutantResult.new(subject: subject, mutation: mutation, status: data["status"].to_sym,
                                     mutated_source: text, reason: data["reason"], runtime: data["runtime"],
                                     killed_by: data["killed_by"], covered_by: data["covered_by"],
-                                    kill_cause: data["kill_cause"], reused: data["reused"])
+                                    kill_cause: data["kill_cause"], reused: data["reused"],
+                                    rejected_kill: data.fetch("rejected_kill", false))
           @cache&.record(result) unless result.reused?
           result
         end