saboteur 0.2.0 → 0.3.0

data/lib/saboteur/help.rb

@@ -0,0 +1,194 @@
+# SPDX-License-Identifier: MIT
+# frozen_string_literal: true
+
+require "json"
+require_relative "config"
+
+module Saboteur
+  # The text printed by `saboteur --help` and `saboteur help <command>`.
+  #
+  # It is a map of everything saboteur can do, for people and for AI agents that
+  # have no docs. The layout is fixed: section headers are upper case on their
+  # own line, command and key rows start with two spaces. Option lines come from
+  # the CLI's own option parser, so a new option shows up here without an edit.
+  class Help
+    Command = Data.define(:name, :summary, :description, :takes_options, :exit_codes, :examples)
+
+    PURPOSE = "mutation testing for Ruby and RSpec"
+    DOCUMENTATION = "https://github.com/vamsipavanmahesh/saboteur"
+
+    EXIT_CODE_MEANINGS = {
+      0 => "Success. Without --threshold: no unignored survivors. With --threshold: the score is at or above " \
+           "it, even if mutants survived.",
+      1 => "Without --threshold: at least one mutant survived and is not on the ignore list. With --threshold " \
+           "(survivors alone no longer fail the run): the score is below it, or no mutant was tested so " \
+           "there is no score.",
+      2 => "Usage or configuration error: fix the command or .saboteur.yml. A replay --id that matches no " \
+           "mutant is only found after the target has booted. A named source or test file that does not exist.",
+      3 => "Tool error: saboteur could not finish, or every executed mutant errored. With --threshold, " \
+           "also any errored or resource-limited mutant."
+    }.freeze
+
+    DOCTOR_EXIT_CODE_MEANINGS = {
+      0 => "Healthy: the environment checks passed and the baseline passed twice in the same order.",
+      2 => "Usage or configuration error: fix the command or .saboteur.yml.",
+      3 => "A check failed: the target did not boot, or the baseline failed or changed between runs."
+    }.freeze
+
+    COMMANDS = [
+      Command.new(
+        name: "run",
+        summary: "Mutate your code and report the mutants your tests do not catch",
+        description: "Builds a per-test coverage map, makes small changes to your methods, runs only the " \
+                     "tests that cover each change, and reports survivors (changes no test noticed). Source files or " \
+                     "directories after run replace include and exclude; --test limits the specs that run; " \
+                     "--threshold PERCENT replaces the survivor check: exit 0 when the score is at or above " \
+                     "PERCENT, exit 1 when it is below.",
+        takes_options: true,
+        exit_codes: EXIT_CODE_MEANINGS,
+        examples: [
+          ["Pull request: only methods changed since the target branch", "saboteur run --since origin/main"],
+          ["Whole project, four workers, repeatable seed", "saboteur run --jobs 4 --seed 42"],
+          ["One class or method", "saboteur run --subject 'Person#adult?'"],
+          ["Replay one mutant by the digest printed next to a survivor", "saboteur run --id 1a2b3c4d5e6f7a8b"],
+          ["Mutate named files against named specs, skipping the whole-suite coverage run",
+           "saboteur run lib/a.rb lib/b.rb --test spec/a_spec.rb --test spec/b_spec.rb"],
+          ["CI gate: exit 1 when the score is below 90 percent", "saboteur run --threshold 90"]
+        ]
+      ),
+      Command.new(
+        name: "doctor",
+        summary: "Check the environment and run the baseline twice, without mutating anything",
+        description: "Boots the target like a run, checks runtime, coverage and boot files, then runs your " \
+                     "suite twice in the same order with one seed and reports whether both runs passed. " \
+                     "It never mutates anything.",
+        takes_options: true,
+        exit_codes: DOCTOR_EXIT_CODE_MEANINGS,
+        examples: [["Check the setup before the first run", "saboteur doctor"],
+                   ["Use another config file and seed", "saboteur doctor --config ci.yml --seed 7"]]
+      ),
+      Command.new(
+        name: "version",
+        summary: "Print the installed version",
+        description: "Prints the version number and nothing else.",
+        takes_options: false,
+        exit_codes: nil,
+        examples: [["Print the version", "saboteur version"]]
+      ),
+      Command.new(
+        name: "help",
+        summary: "Show this map, or details for one command: saboteur help <command>",
+        description: "Without an argument, prints every command, option, config key and exit code. " \
+                     "With a command name, prints that command's usage, options and examples.",
+        takes_options: false,
+        exit_codes: nil,
+        examples: [["Show everything", "saboteur help"], ["Show one command", "saboteur help run"]]
+      )
+    ].freeze
+
+    FILES = [
+      [".saboteur.yml", "Settings. Command line options override it."],
+      [".saboteur-ignore.yml", "Accepted survivors: entries with subject, mutation (digest) and a required reason."],
+      [".saboteur/report.json", "Stryker mutation-testing-report-schema JSON (--json changes the path)."],
+      [".saboteur/report.html", "Self-contained HTML report (--html changes the path)."],
+      [".saboteur/", "Cache: coverage map and incremental verdicts. Safe to delete."]
+    ].freeze
+
+    # option_lines: the option lines of each command's parser, keyed by command name.
+    def initialize(option_lines:, commands: COMMANDS)
+      @option_lines = option_lines
+      @commands = commands
+    end
+
+    def overview
+      [title, usage_section, commands_section, *option_sections, examples_section(all_examples),
+       *exit_code_sections, config_section, files_section, "Documentation: #{DOCUMENTATION}\n"].join("\n")
+    end
+
+    # Details for one command, or nil when there is no such command.
+    def command(name)
+      found = @commands.find { |candidate| candidate.name == name }
+      return unless found
+
+      sections = ["saboteur #{found.name} - #{found.summary}\n", "#{found.description}\n", command_usage(found)]
+      sections << options_section(found) if found.takes_options
+      sections << examples_section(found.examples)
+      sections << exit_codes_section(found.exit_codes) if found.exit_codes
+      sections.join("\n")
+    end
+
+    def command_names
+      @commands.map(&:name)
+    end
+
+    private
+
+    # One section for every command that takes options, so the overview needs no other page.
+    def option_sections
+      @commands.select(&:takes_options).map { |command| options_section(command) }
+    end
+
+    def exit_code_sections
+      @commands.select(&:exit_codes).map { |command| exit_codes_section(command.exit_codes, command.name) }
+    end
+
+    def title
+      "saboteur #{VERSION} - #{PURPOSE}\n"
+    end
+
+    def usage_section
+      "USAGE\n  saboteur <command> [options]\n  saboteur help [command]\n"
+    end
+
+    def command_usage(command)
+      suffix = command.takes_options ? " [options]" : ""
+      "USAGE\n  saboteur #{command.name}#{suffix}\n"
+    end
+
+    def commands_section
+      rows = @commands.map { |command| "  #{command.name.ljust(name_width)}  #{command.summary}" }
+      "COMMANDS\n#{rows.join("\n")}\n"
+    end
+
+    def name_width
+      @commands.map { |command| command.name.length }.max
+    end
+
+    def options_section(command)
+      "OPTIONS (saboteur #{command.name})\n#{@option_lines.fetch(command.name).map(&:rstrip).join("\n")}\n"
+    end
+
+    def all_examples
+      @commands.flat_map(&:examples)
+    end
+
+    def examples_section(examples)
+      "EXAMPLES\n#{examples.map { |note, line| "  # #{note}\n  #{line}" }.join("\n")}\n"
+    end
+
+    def exit_codes_section(codes, command_name = nil)
+      rows = codes.map { |code, meaning| "  #{code}  #{meaning}" }
+      title = command_name ? "EXIT CODES (saboteur #{command_name})" : "EXIT CODES"
+      "#{title}\n#{rows.join("\n")}\n"
+    end
+
+    def config_section
+      rows = Config::DEFAULTS.map { |key, value| "  #{key}: #{describe_default(value)}" }
+      intro = "Every key with its default value, as YAML. Unknown keys are an error."
+      "CONFIG FILE (.saboteur.yml)\n#{intro}\n#{rows.join("\n")}\n"
+    end
+
+    # Valid YAML (JSON flow style), so a copied line keeps the default's meaning.
+    def describe_default(value)
+      JSON.generate(value)
+    end
+
+    def files_section
+      "FILES\n#{FILES.map { |path, meaning| "  #{path.ljust(files_width)}  #{meaning}" }.join("\n")}\n"
+    end
+
+    def files_width
+      FILES.map { |path, _| path.length }.max
+    end
+  end
+end

data/lib/saboteur/integration/base.rb

@@ -61,6 +61,14 @@ module Saboteur
         raise NotImplementedError
       end
 
+      # Checks the unchanged suite in a fresh child. Existing adapters can
+      # use their selected-test runner; adapters may override to retain full
+      # suite ordering and report every executed test.
+      def run_baseline(seed:)
+        Kernel.srand(seed)
+        run(all_tests.map(&:id))
+      end
+
       # Files whose content decides which tests exist (for the cache key).
       def test_files
         []

data/lib/saboteur/integration/rspec.rb

@@ -10,9 +10,9 @@ module Saboteur
       def setup
         require "rspec/core"
         ::RSpec::Core::Runner.disable_autorun!
-        options = ::RSpec::Core::ConfigurationOptions.new(rspec_args)
-        @runner = ::RSpec::Core::Runner.new(options)
+        @runner = ::RSpec::Core::Runner.new(configuration_options)
         @runner.setup($stderr, $stdout)
+        configuration.order = "rand:#{config.fetch('seed', 1)}"
         raise Error, "rspec quit while loading spec files" if world.wants_to_quit
       end
 
@@ -22,6 +22,16 @@ module Saboteur
         args.any? { |arg| !arg.start_with?("-") } ? args : args + ["spec"]
       end
 
+      # RSpec's own parser reads rspec_args and .rspec, so option values such
+      # as `--require path` stay intact. Explicit tests then replace only the
+      # files and directories to run.
+      def configuration_options
+        ::RSpec::Core::ConfigurationOptions.new(rspec_args).tap do |options|
+          explicit = Array(config.fetch("tests", []))
+          options.options[:files_or_directories_to_run] = explicit if explicit.any?
+        end
+      end
+
       def all_tests
         groups = world.ordered_example_groups.flat_map(&:descendants)
         groups.flat_map { |group| world.filtered_examples[group] }
@@ -47,10 +57,21 @@ module Saboteur
       end
 
       def run_all(recorder)
+        configuration.order = "rand:#{config.fetch('seed', 1)}"
         configuration.fail_fast = false
         execute(Listener.new(recorder))
       end
 
+      def run_baseline(seed:)
+        configuration.order = "rand:#{seed}"
+        configuration.fail_fast = false
+        listener = Listener.new
+        passed = execute(listener)
+        RunResult.new(passed: passed && listener.failed.empty?, failed_ids: listener.failed,
+                      executed_ids: listener.executed, runtime: listener.runtime,
+                      failure_classes: listener.failure_classes, failure_messages: listener.failure_messages)
+      end
+
       private
 
       def world

data/lib/saboteur/mutation_source/builtin.rb

@@ -3,6 +3,8 @@
 
 require "prism"
 require_relative "base"
+require_relative "filter_replacement"
+require_relative "literal_replacement"
 
 module Saboteur
   module MutationSource
@@ -46,6 +48,8 @@ module Saboteur
       def initialize(arid: DEFAULT_ARID)
         super()
         @arid = arid.map { |pattern| pattern.is_a?(Regexp) ? pattern : Regexp.new(pattern) }
+        @filter_replacement = FilterReplacement.new(arid: arid)
+        @literal_replacement = LiteralReplacement.new(arid: arid)
       end
 
       def name
@@ -58,7 +62,8 @@ module Saboteur
         @found = []
         coarse(body)
         visit(body)
-        @found
+        @found.concat(@filter_replacement.mutations_for(subject))
+        @found.concat(@literal_replacement.mutations_for(subject))
       ensure
         @subject = nil
       end

data/lib/saboteur/mutation_source/filter_replacement.rb

@@ -0,0 +1,60 @@
+# SPDX-License-Identifier: MIT
+# frozen_string_literal: true
+
+require "prism"
+require_relative "base"
+
+module Saboteur
+  module MutationSource
+    # Swaps the inclusion rule of a filter call with its exclusion rule.
+    # Only message bytes change; arguments, receiver and block stay intact.
+    class FilterReplacement < Base
+      REPLACEMENTS = { select: "reject", reject: "select", select!: "reject!", reject!: "select!" }.freeze
+      SCOPE_NODES = [Prism::DefNode, Prism::ClassNode, Prism::ModuleNode, Prism::SingletonClassNode].freeze
+
+      # Base inherits Object's empty initializer; there is no parent state to set.
+      def initialize(arid: []) # rubocop:disable Lint/MissingSuper
+        @arid = arid.map { |pattern| Regexp.new(pattern) }
+      end
+
+      def name
+        "filter_replacement"
+      end
+
+      def mutations_for(subject)
+        found = []
+        visit(subject.node.body, subject, found)
+        found
+      end
+
+      private
+
+      def visit(node, subject, found)
+        return if node.nil? || SCOPE_NODES.any? { |scope| node.is_a?(scope) }
+
+        if node.is_a?(Prism::StatementsNode)
+          node.body.each { |statement| visit(statement, subject, found) unless arid?(statement) }
+          return
+        end
+
+        replace_filter(node, subject, found) if node.is_a?(Prism::CallNode)
+        node.compact_child_nodes.each { |child| visit(child, subject, found) }
+      end
+
+      def replace_filter(node, subject, found)
+        replacement = REPLACEMENTS[node.name]
+        return unless replacement && node.receiver && node.block
+        return unless node.arguments.nil?
+
+        location = node.message_loc
+        edit = Edit.new(file: subject.file, start_byte: location.start_offset,
+                        end_byte: location.end_offset, replacement: replacement)
+        found << Mutation.new(subject_id: subject.id, operator: name, edits: [edit], source: name)
+      end
+
+      def arid?(node)
+        @arid.any? { |pattern| pattern.match?(node.slice) }
+      end
+    end
+  end
+end

data/lib/saboteur/mutation_source/literal_replacement.rb

@@ -0,0 +1,74 @@
+# SPDX-License-Identifier: MIT
+# frozen_string_literal: true
+
+require "prism"
+require_relative "base"
+
+module Saboteur
+  module MutationSource
+    # Changes literal values without rebuilding the surrounding source.
+    class LiteralReplacement < Base
+      SCOPE_NODES = [Prism::DefNode, Prism::ClassNode, Prism::ModuleNode, Prism::SingletonClassNode].freeze
+      TEXT_NODES = [Prism::InterpolatedStringNode, Prism::SymbolNode, Prism::InterpolatedSymbolNode,
+                    Prism::RegularExpressionNode, Prism::InterpolatedRegularExpressionNode,
+                    Prism::XStringNode, Prism::InterpolatedXStringNode].freeze
+      PARAMETER_NODES = [Prism::ParametersNode, Prism::BlockParametersNode].freeze
+      BOUNDARIES = (SCOPE_NODES + TEXT_NODES + PARAMETER_NODES).freeze
+
+      # Base inherits Object's empty initializer; there is no parent state to set.
+      def initialize(arid: []) # rubocop:disable Lint/MissingSuper
+        @arid = arid.map { |pattern| Regexp.new(pattern) }
+      end
+
+      def name
+        "literal_replacement"
+      end
+
+      def mutations_for(subject)
+        found = []
+        visit(subject.node.body, subject, found)
+        found
+      end
+
+      private
+
+      def visit(node, subject, found)
+        return if node.nil? || boundary?(node)
+
+        if node.is_a?(Prism::StatementsNode)
+          node.body.each { |statement| visit(statement, subject, found) unless arid?(statement) }
+          return
+        end
+
+        case node
+        when Prism::TrueNode then add(subject, found, node, "literal_boolean", "false")
+        when Prism::FalseNode then add(subject, found, node, "literal_boolean", "true")
+        when Prism::StringNode
+          replace_string(node, subject, found)
+        end
+        node.compact_child_nodes.each { |child| visit(child, subject, found) }
+      end
+
+      def boundary?(node)
+        BOUNDARIES.any? { |type| node.is_a?(type) }
+      end
+
+      def replace_string(node, subject, found)
+        return if node.unescaped.empty?
+        return unless node.opening_loc && !node.opening_loc.slice.start_with?("<<")
+
+        add(subject, found, node, "literal_string", '""')
+      end
+
+      def add(subject, found, node, operator, replacement)
+        edit = Edit.new(file: subject.file, start_byte: node.location.start_offset,
+                        end_byte: node.location.end_offset, replacement: replacement)
+        found << Mutation.new(subject_id: subject.id, operator: operator, edits: [edit], source: name)
+      end
+
+      def arid?(node)
+        @arid.any? { |pattern| pattern.match?(node.slice) }
+      end
+    end
+  end
+end

data/lib/saboteur/replay.rb

@@ -0,0 +1,33 @@
+# SPDX-License-Identifier: MIT
+# frozen_string_literal: true
+
+module Saboteur
+  # Resolves one exact digest from current source, before caps or coarse gating.
+  class Replay
+    def initialize(digest:)
+      unless digest.match?(/\A[0-9a-f]{16}\z/)
+        raise ConfigError, "--id needs a full 16 lowercase hexadecimal mutant digest"
+      end
+
+      @digest = digest
+    end
+
+    def select(subjects, candidates, scope: nil)
+      match = nil
+      subjects.each do |subject|
+        list = candidates.for(subject)
+        list = scope.restrict(subject, list) if scope
+        list.each do |mutation, text|
+          digest = mutation.digest(subject_start: subject.start_byte, original_source: subject.file_source)
+          next unless digest == @digest
+          raise ConfigError, "mutant #{@digest} is ambiguous; narrow --subject or include files" if match
+
+          match = [subject, [mutation, text]]
+        end
+      end
+      raise ConfigError, "mutant #{@digest} not found in the selected current source" unless match
+
+      match
+    end
+  end
+end

data/lib/saboteur/reporter/findings.rb

@@ -0,0 +1,123 @@
+# SPDX-License-Identifier: MIT
+# frozen_string_literal: true
+
+module Saboteur
+  module Reporter
+    # Findings for a code-review surface such as the saboteur GitHub Action:
+    # a bounded list of survivors with everything a note on the pull-request
+    # diff needs (file, line, digest, operator, original and replacement).
+    # Survivors accepted on the ignore list are listed apart, with their
+    # reason. The full, unbounded mutant data stays in the report's files
+    # section; survivorsTruncated says how many more findings live only there.
+    module Findings
+      VERSION = 1
+
+      # GitHub shows at most 10 annotations of one kind per step.
+      LIMIT = 10
+
+      # Published ordering, coarse survivors first: a surviving coarse mutant
+      # replaced the whole method body, which hints that no test pins the
+      # method's behavior - though it can also be a re-evaluation limit or an
+      # equivalent mutant, so this is a heuristic, not a guarantee. Ties break
+      # by file, line and digest, so one report always reads the same way.
+      ORDERING = "coarse survivors first, then file, line and digest"
+
+      class << self
+        def build(report)
+          live, accepted = report.survivors.partition { |result| result.ignored.nil? }
+          {
+            version: VERSION,
+            ordering: ORDERING,
+            limit: LIMIT,
+            survivors: listed(report, live),
+            survivorsTruncated: truncated(live),
+            accepted: listed(report, accepted),
+            acceptedTruncated: truncated(accepted)
+          }
+        end
+
+        private
+
+        def listed(report, results)
+          ordered(report, results).first(LIMIT).map { |result| finding(report, result) }
+        end
+
+        def truncated(results)
+          [results.size - LIMIT, 0].max
+        end
+
+        # partition keeps the coarse group ahead; inside a group the key is
+        # total (no two mutants share a digest), so the order is stable.
+        def ordered(report, results)
+          results.partition { |result| result.mutation.coarse }
+                 .map { |group| group.sort_by { |result| tie_break(report, result) } }
+                 .flatten
+        end
+
+        def tie_break(report, result)
+          [path(report, result), first_line(result), result.digest]
+        end
+
+        def first_line(result)
+          result.subject.line_of(result.mutation.edits.map(&:start_byte).min)
+        end
+
+        def finding(report, result)
+          entries = edit_entries(result)
+          entry = {
+            digest: result.digest,
+            path: path(report, result),
+            line: entries.map { |edit| edit[:line] }.min,
+            endLine: entries.map { |edit| edit[:endLine] }.max,
+            subject: result.subject.id,
+            operator: result.mutation.operator,
+            original: joined_texts(result) { |edit| original_text(result, edit) },
+            replacement: joined_texts(result, &:replacement),
+            edits: entries,
+            coarse: result.mutation.coarse
+          }
+          result.ignored ? entry.merge(reason: result.ignored) : entry
+        end
+
+        # Subject#apply applies edits in start-byte order; findings use the
+        # same order, so an unsorted edit list from a custom mutation source
+        # still gets a sane range and one entry per edit.
+        def sorted_edits(result)
+          result.mutation.edits.sort_by(&:start_byte)
+        end
+
+        # Stryker joins a multi-edit mutant's replacements the same way, so
+        # both reports describe one mutant with one text.
+        def joined_texts(result, &pick)
+          sorted_edits(result).map { |edit| safe_text(pick.call(edit)) }.join(" | ")
+        end
+
+        # One entry per edit, so a multi-edit finding can point each joined
+        # text at its own spot on the diff.
+        def edit_entries(result)
+          sorted_edits(result).map do |edit|
+            {
+              line: result.subject.line_of(edit.start_byte),
+              endLine: result.subject.line_of([edit.end_byte - 1, edit.start_byte].max),
+              original: safe_text(original_text(result, edit)),
+              replacement: safe_text(edit.replacement)
+            }
+          end
+        end
+
+        def original_text(result, edit)
+          result.subject.file_source.byteslice(edit.start_byte, edit.end_byte - edit.start_byte)
+        end
+
+        # A byte slice of source may be invalid UTF-8, which JSON cannot hold.
+        def safe_text(text)
+          text.dup.force_encoding(Encoding::UTF_8).scrub
+        end
+
+        def path(report, result)
+          result.subject.file.delete_prefix("#{report.root}/")
+        end
+      end
+    end
+  end
+end