saboteur 0.2.0

data/lib/saboteur/integration/rspec.rb

@@ -0,0 +1,143 @@
+# SPDX-License-Identifier: MIT
+# frozen_string_literal: true
+
+module Saboteur
+  module Integration
+    # Loads spec files once in the parent. In a child, narrows the already
+    # loaded world to the requested example ids and runs it with a private
+    # reporter, so nothing is printed and no status file is written.
+    class RSpec < Base
+      def setup
+        require "rspec/core"
+        ::RSpec::Core::Runner.disable_autorun!
+        options = ::RSpec::Core::ConfigurationOptions.new(rspec_args)
+        @runner = ::RSpec::Core::Runner.new(options)
+        @runner.setup($stderr, $stdout)
+        raise Error, "rspec quit while loading spec files" if world.wants_to_quit
+      end
+
+      # RSpec only adds its default "spec" path when $0 is `rspec`.
+      def rspec_args
+        args = Array(config.fetch("rspec_args", []))
+        args.any? { |arg| !arg.start_with?("-") } ? args : args + ["spec"]
+      end
+
+      def all_tests
+        groups = world.ordered_example_groups.flat_map(&:descendants)
+        groups.flat_map { |group| world.filtered_examples[group] }
+              .map { |example| TestCase.new(id: example.id, file: example.metadata[:rerun_file_path]) }
+      end
+
+      def test_files
+        world.example_groups.map { |group| File.expand_path(group.metadata[:file_path]) }.uniq
+      end
+
+      # Runs the tests in the order given: groups by their earliest example,
+      # examples by position.
+      def run(ids)
+        wanted = Set.new(ids)
+        narrow_to(wanted)
+        order_by(ids)
+        configuration.fail_fast = 1
+        listener = Listener.new
+        passed = execute(listener)
+        RunResult.new(passed: passed && listener.failed.empty?, failed_ids: listener.failed,
+                      executed_ids: listener.executed, runtime: listener.runtime,
+                      failure_classes: listener.failure_classes, failure_messages: listener.failure_messages)
+      end
+
+      def run_all(recorder)
+        configuration.fail_fast = false
+        execute(Listener.new(recorder))
+      end
+
+      private
+
+      def world
+        ::RSpec.world
+      end
+
+      def configuration
+        ::RSpec.configuration
+      end
+
+      def order_by(ids)
+        rank = ids.each_with_index.to_h
+        example_rank = ->(example) { rank.fetch(example.id, Float::INFINITY) }
+        configuration.register_ordering(:global) do |items|
+          items.sort_by do |item|
+            if item.is_a?(Class)
+              item.descendant_filtered_examples.map(&example_rank).min || Float::INFINITY
+            else
+              example_rank.call(item)
+            end
+          end
+        end
+      end
+
+      def narrow_to(wanted)
+        world.filtered_examples.clear
+        world.filtered_examples.default_proc = proc do |hash, group|
+          hash[group] = group.examples.select { |example| wanted.include?(example.id) }
+        end
+        world.example_groups.flat_map(&:descendants).each(&:reset_memoized)
+      end
+
+      def execute(listener)
+        reporter = ::RSpec::Core::Reporter.new(configuration)
+        reporter.register_listener(listener, *Listener::EVENTS)
+        groups = world.ordered_example_groups.select { |group| group.descendant_filtered_examples.any? }
+        count = world.example_count(groups)
+        started = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+        passed = reporter.report(count) do |r|
+          configuration.with_suite_hooks { groups.map { |group| group.run(r) }.all? }
+        end
+        listener.runtime = Process.clock_gettime(Process::CLOCK_MONOTONIC) - started
+        passed && !world.non_example_failure
+      end
+
+      class Listener
+        EVENTS = %i[example_started example_finished example_failed].freeze
+
+        MESSAGE_LIMIT = 200
+
+        attr_reader :executed, :failed, :failure_classes, :failure_messages
+        attr_accessor :runtime
+
+        def initialize(recorder = nil)
+          @recorder = recorder
+          @executed = []
+          @failed = []
+          @failure_classes = {}
+          @failure_messages = {}
+          @runtime = 0.0
+        end
+
+        def example_started(notification)
+          example = notification.example
+          @recorder&.test_started(example.id, File.expand_path(example.metadata[:rerun_file_path]))
+        end
+
+        def example_finished(notification)
+          @executed << notification.example.id
+          @recorder&.test_finished(notification.example.id)
+        end
+
+        def example_failed(notification)
+          @failed << notification.example.id
+          exception = notification.example.execution_result.exception
+          @failure_classes[notification.example.id] = exception&.class&.name
+          @failure_messages[notification.example.id] = first_line(exception&.message)
+        end
+
+        private
+
+        def first_line(message)
+          message.to_s.lines.map(&:strip).find { |line| !line.empty? }.to_s[0, MESSAGE_LIMIT]
+        end
+      end
+    end
+
+    register :rspec, RSpec
+  end
+end

data/lib/saboteur/mutation_source/base.rb

@@ -0,0 +1,26 @@
+# SPDX-License-Identifier: MIT
+# frozen_string_literal: true
+
+# Public extension API: implement Base#mutations_for and yield Mutation
+# objects built from Edit values. Stable across minor versions.
+require_relative "edit"
+require_relative "mutation"
+
+module Saboteur
+  module MutationSource
+    # Extension point. A source yields Mutation objects for a subject.
+    # Built-in operators are one source; external sources (for example a
+    # service that proposes edits) implement the same method and are
+    # registered in config. Every edit must lie inside the subject's byte range.
+    class Base
+      def name
+        self.class.name
+      end
+
+      # subject responds to #id, #file, #start_byte, #end_byte, #source.
+      def mutations_for(_subject)
+        raise NotImplementedError, "#{self.class} must implement #mutations_for"
+      end
+    end
+  end
+end

data/lib/saboteur/mutation_source/builtin.rb

@@ -0,0 +1,232 @@
+# SPDX-License-Identifier: MIT
+# frozen_string_literal: true
+
+require "prism"
+require_relative "base"
+
+module Saboteur
+  module MutationSource
+    # The v1 operator set. Every mutation is a byte-range edit of the original
+    # file text; nothing is unparsed.
+    class Builtin < Base
+      DEFAULT_ARID = [
+        '\A(puts|pp|p|print|warn)\b',
+        '\A(::)?(Rails\.)?logger\b',
+        '\A(::)?(StatsD|Statsd|Datadog|NewRelic)\b',
+        '\.instrument\b'
+      ].freeze
+
+      # A comparison replacement: swap the operator, or put a literal in
+      # place of the whole comparison.
+      Swap = Data.define(:operator)
+      Literal = Data.define(:text)
+
+      # Sufficient relational replacements (Kaminski, Ammann, Offutt 2011):
+      # the others are subsumed by these. For == and != Ruby gets true/false:
+      # <= and >= raise on objects that are not Comparable, which only gives
+      # cheap crash kills.
+      COMPARISON = {
+        :< => [Swap["<="], Swap["!="], Literal["false"]], :<= => [Swap["<"], Swap["=="], Literal["true"]],
+        :> => [Swap[">="], Swap["!="], Literal["false"]], :>= => [Swap[">"], Swap["=="], Literal["true"]],
+        :== => [Literal["false"], Literal["true"]], :!= => [Literal["true"], Literal["false"]]
+      }.freeze
+      ARITHMETIC = { :+ => "-", :- => "+", :* => "/", :/ => "*" }.freeze
+      # Sufficient connector replacements (Just et al. 2012): each operand
+      # alone, then the constant the connector cannot produce from it.
+      ABSORBING = { "&&" => "false", "and" => "false", "||" => "true", "or" => "true" }.freeze
+      NON_NUMERIC = [Prism::StringNode, Prism::InterpolatedStringNode, Prism::XStringNode,
+                     Prism::ArrayNode, Prism::HashNode, Prism::SymbolNode].freeze
+      OR_WRITE = [Prism::LocalVariableOrWriteNode, Prism::InstanceVariableOrWriteNode,
+                  Prism::ClassVariableOrWriteNode, Prism::GlobalVariableOrWriteNode,
+                  Prism::ConstantOrWriteNode, Prism::ConstantPathOrWriteNode,
+                  Prism::CallOrWriteNode, Prism::IndexOrWriteNode].freeze
+      CATCH_ALL = %w[StandardError ::StandardError Exception ::Exception].freeze
+      SCOPE_NODES = [Prism::DefNode, Prism::ClassNode, Prism::ModuleNode, Prism::SingletonClassNode].freeze
+
+      def initialize(arid: DEFAULT_ARID)
+        super()
+        @arid = arid.map { |pattern| pattern.is_a?(Regexp) ? pattern : Regexp.new(pattern) }
+      end
+
+      def name
+        "builtin"
+      end
+
+      def mutations_for(subject)
+        body = subject.node.body
+        @subject = subject
+        @found = []
+        coarse(body)
+        visit(body)
+        @found
+      ensure
+        @subject = nil
+      end
+
+      private
+
+      # --- coarse ---------------------------------------------------------
+
+      def coarse(body)
+        statements = body.is_a?(Prism::BeginNode) ? body.statements : body
+        return if statements.nil?
+
+        unless nothing_to_remove?(statements)
+          # A predicate's nil is just false; true is the value its tests must reject.
+          (@subject.name.end_with?("?") ? %w[true false] : %w[nil]).each do |value|
+            add("body_#{value}", statements, value, coarse: true)
+          end
+        end
+        add("body_raise", statements, "raise", coarse: true) unless rescues_everything?(body)
+      end
+
+      # Nothing but logging: replacing it changes nothing a test can see.
+      def nothing_to_remove?(statements)
+        statements.body.all? { |statement| arid?(statement) }
+      end
+
+      # A raise that the method's own rescue catches is absorbed by design. It
+      # would survive, and a surviving coarse mutant stops every fine mutant
+      # of the method, so it is not generated.
+      def rescues_everything?(body)
+        clause = body.is_a?(Prism::BeginNode) ? body.rescue_clause : nil
+        while clause
+          return true if clause.exceptions.empty? || clause.exceptions.any? { |e| CATCH_ALL.include?(e.slice) }
+
+          clause = clause.subsequent
+        end
+        false
+      end
+
+      # --- fine -----------------------------------------------------------
+
+      def visit(node)
+        return if node.nil?
+        return if SCOPE_NODES.any? { |klass| node.is_a?(klass) }
+
+        if node.is_a?(Prism::StatementsNode)
+          visit_statements(node)
+          return
+        end
+
+        mutate(node)
+        node.compact_child_nodes.each { |child| visit(child) }
+      end
+
+      # Statement removal applies only to sequences of two or more statements.
+      # A single statement is either the whole method body (body_nil covers
+      # it) or an expression slot: a ternary branch, an interpolation, the
+      # body of a modifier if. Replacing those with nil is not removing a
+      # statement and duplicates the condition and coarse operators.
+      def visit_statements(node)
+        statements = node.body
+        statements.each do |statement|
+          next if arid?(statement)
+
+          add("statement_removal", statement, "nil") if statements.size > 1 && removable?(statement)
+          visit(statement)
+        end
+      end
+
+      def mutate(node)
+        case node
+        when Prism::CallNode then mutate_call(node)
+        when Prism::AndNode, Prism::OrNode then mutate_boolean(node)
+        when Prism::IfNode, Prism::UnlessNode then mutate_condition(node.predicate, hang_on: nil)
+        when Prism::WhileNode then mutate_condition(node.predicate, hang_on: "true")
+        when Prism::UntilNode then mutate_condition(node.predicate, hang_on: "false")
+        when Prism::IntegerNode, Prism::FloatNode then mutate_number(node)
+        when *OR_WRITE then add("or_assign", node.operator_loc, "=")
+        end
+      end
+
+      def mutate_call(node)
+        if binary?(node)
+          COMPARISON.fetch(node.name, []).each do |replacement|
+            case replacement
+            when Swap then add("comparison", node.message_loc, replacement.operator)
+            when Literal then add("comparison", node, replacement.text)
+            end
+          end
+          arithmetic = ARITHMETIC[node.name]
+          add("arithmetic", node.message_loc, arithmetic) if arithmetic && numeric_operands?(node)
+        end
+        mutate_negation(node) if node.name == :! && node.receiver
+        add("safe_navigation", node.call_operator_loc, ".") if node.safe_navigation?
+      end
+
+      def binary?(node)
+        node.receiver && node.message_loc && node.call_operator_loc.nil? &&
+          node.arguments&.arguments&.size == 1 && node.block.nil?
+      end
+
+      def numeric_operands?(node)
+        [node.receiver, node.arguments.arguments.first].none? { |operand| NON_NUMERIC.any? { operand.is_a?(_1) } }
+      end
+
+      def mutate_negation(node)
+        add("negation_removal", node, node.receiver.slice)
+      end
+
+      # An operand of && / and binds at least as tightly as the expression it
+      # replaces, so it needs no parentheses (the re-parse check backs this).
+      def mutate_boolean(node)
+        [node.left, node.right].each { |operand| add("boolean_logic", node, operand.slice) }
+        add("boolean_logic", node, ABSORBING.fetch(node.operator_loc.slice))
+      end
+
+      # Replacing a loop condition with the value that never ends it would
+      # only produce timeouts, so that one is skipped.
+      def mutate_condition(predicate, hang_on:)
+        (%w[true false] - [hang_on]).each { |literal| add("condition_#{literal}", predicate, literal) }
+      end
+
+      def mutate_number(node)
+        value = node.value
+        candidates = [value + 1, value - 1, 0]
+        candidates.map! { |n| node.is_a?(Prism::FloatNode) ? n.to_f : n }
+        candidates.uniq.each do |n|
+          next if n == value
+
+          text = n.to_s
+          text = "(#{text})" if n.negative?
+          add("literal_number", node, text)
+        end
+      end
+
+      # --- helpers --------------------------------------------------------
+
+      # Removing nil changes nothing. Removing an if/unless without an else
+      # is the same mutant as forcing its condition (condition_false /
+      # condition_true), so it would only report the same gap twice.
+      def removable?(statement)
+        case statement
+        when Prism::NilNode then false
+        when Prism::IfNode then else_branch?(statement, :subsequent)
+        when Prism::UnlessNode then else_branch?(statement, :else_clause)
+        else true
+        end
+      end
+
+      def else_branch?(node, name)
+        !node.public_send(name).nil?
+      end
+
+      def arid?(node)
+        source = node.slice
+        @arid.any? { |pattern| pattern.match?(source) }
+      end
+
+      # node_or_location: a Prism node or location (both have offsets and slice).
+      def add(operator, node_or_location, replacement, coarse: false)
+        location = node_or_location
+        return if location.slice == replacement
+
+        edit = Edit.new(file: @subject.file, start_byte: location.start_offset,
+                        end_byte: location.end_offset, replacement: replacement)
+        @found << Mutation.new(subject_id: @subject.id, operator: operator, edits: [edit],
+                               coarse: coarse, source: name)
+      end
+    end
+  end
+end

data/lib/saboteur/mutation_source/edit.rb

@@ -0,0 +1,18 @@
+# SPDX-License-Identifier: MIT
+# frozen_string_literal: true
+
+module Saboteur
+  module MutationSource
+    # A replacement of source bytes [start_byte, end_byte) in file with text.
+    # Offsets are byte offsets into the file as read from disk (UTF-8),
+    # matching Prism::Location#start_offset / #end_offset.
+    Edit = Data.define(:file, :start_byte, :end_byte, :replacement) do
+      def initialize(file:, start_byte:, end_byte:, replacement:)
+        raise ArgumentError, "start_byte must be >= 0" if start_byte.negative?
+        raise ArgumentError, "end_byte must be >= start_byte" if end_byte < start_byte
+
+        super
+      end
+    end
+  end
+end

data/lib/saboteur/mutation_source/mutation.rb

@@ -0,0 +1,28 @@
+# SPDX-License-Identifier: MIT
+# frozen_string_literal: true
+
+require "digest"
+
+module Saboteur
+  module MutationSource
+    # One candidate change to one subject. subject_id is "Klass#meth" or "Klass.meth".
+    # coarse: true for whole-body replacements, which run before fine-grained ones.
+    Mutation = Data.define(:subject_id, :operator, :edits, :coarse, :source) do
+      def initialize(subject_id:, operator:, edits:, coarse: false, source: "builtin")
+        raise ArgumentError, "a mutation needs at least one edit" if edits.empty?
+
+        super(subject_id:, operator:, edits: edits.freeze, coarse:, source:)
+      end
+
+      # Stable id for the ignore list. Offsets are made relative to subject_start
+      # so edits elsewhere in the file do not change the id.
+      def digest(subject_start:, original_source:)
+        parts = edits.map do |edit|
+          original = original_source.byteslice(edit.start_byte, edit.end_byte - edit.start_byte)
+          [edit.start_byte - subject_start, original, edit.replacement].join("\0")
+        end
+        Digest::SHA256.hexdigest([subject_id, operator, *parts].join("\x1F"))[0, 16]
+      end
+    end
+  end
+end

data/lib/saboteur/reporter/diff.rb

@@ -0,0 +1,97 @@
+# SPDX-License-Identifier: MIT
+# frozen_string_literal: true
+
+module Saboteur
+  module Reporter
+    # Minimal unified diff between two method texts (LCS over lines). Methods
+    # are short, so the quadratic table is fine.
+    module Diff
+      CONTEXT = 3
+
+      module_function
+
+      def unified(original, mutated, path:, first_line: 1)
+        a = original.lines
+        b = mutated.lines
+        ops = operations(a, b)
+        hunks = hunks(ops)
+        return "" if hunks.empty?
+
+        out = "--- a/#{path}\n+++ b/#{path}\n"
+        hunks.each { |hunk| out << render(hunk, first_line) }
+        out
+      end
+
+      # [[:same|:del|:add, a_index, b_index, text]]
+      def operations(old_lines, new_lines)
+        a = old_lines
+        b = new_lines
+        table = lcs_table(a, b)
+        ops = []
+        i = 0
+        j = 0
+        while i < a.size || j < b.size
+          if i < a.size && j < b.size && a[i] == b[j]
+            ops << [:same, i, j, a[i]]
+            i += 1
+            j += 1
+          elsif i < a.size && (j >= b.size || table[i + 1][j] >= table[i][j + 1])
+            ops << [:del, i, j, a[i]]
+            i += 1
+          else
+            ops << [:add, i, j, b[j]]
+            j += 1
+          end
+        end
+        ops
+      end
+
+      def lcs_table(old_lines, new_lines)
+        a = old_lines
+        b = new_lines
+        table = Array.new(a.size + 1) { Array.new(b.size + 1, 0) }
+        (a.size - 1).downto(0) do |i|
+          (b.size - 1).downto(0) do |j|
+            table[i][j] = a[i] == b[j] ? table[i + 1][j + 1] + 1 : [table[i + 1][j], table[i][j + 1]].max
+          end
+        end
+        table
+      end
+
+      def hunks(ops)
+        changed = ops.each_index.reject { |index| ops[index][0] == :same }
+        return [] if changed.empty?
+
+        ranges = changed.map { |index| [[index - CONTEXT, 0].max, [index + CONTEXT, ops.size - 1].min] }
+        merged = ranges.each_with_object([]) do |(from, to), acc|
+          if acc.any? && from <= acc.last[1] + 1
+            acc.last[1] = [acc.last[1], to].max
+          else
+            acc << [from, to]
+          end
+        end
+        merged.map { |from, to| ops[from..to] }
+      end
+
+      def render(hunk, first_line)
+        a_start = hunk.first[1]
+        b_start = hunk.first[2]
+        a_count = hunk.count { |op| op[0] != :add }
+        b_count = hunk.count { |op| op[0] != :del }
+        out = "@@ -#{range(a_start + first_line, a_count)} +#{range(b_start + first_line, b_count)} @@\n"
+        hunk.each do |op, _, _, text|
+          prefix = { same: " ", del: "-", add: "+" }.fetch(op)
+          out << prefix << text
+          out << "\n" unless text.end_with?("\n")
+        end
+        out
+      end
+
+      def range(start, count)
+        return "#{start - 1},0" if count.zero?
+
+        count == 1 ? start.to_s : "#{start},#{count}"
+      end
+    end
+  end
+end

data/lib/saboteur/reporter/stryker.rb

@@ -0,0 +1,116 @@
+# SPDX-License-Identifier: MIT
+# frozen_string_literal: true
+
+require "fileutils"
+require "json"
+
+module Saboteur
+  module Reporter
+    # JSON in the open mutation-testing-report-schema (schemaVersion 2), as
+    # read by the Stryker dashboard and mutation-testing-elements.
+    class Stryker
+      SCHEMA_VERSION = "2"
+      THRESHOLDS = { high: 80, low: 60 }.freeze
+
+      def initialize(path)
+        @path = path
+      end
+
+      def call(report)
+        FileUtils.mkdir_p(File.dirname(@path))
+        File.write(@path, JSON.generate(self.class.build(report)))
+      end
+
+      # Tests are listed once under testFiles with short ids; mutants refer to
+      # them by id in killedBy/coveredBy instead of repeating full test names.
+      def self.build(report)
+        test_ids = test_index(report.results)
+        files = report.results.group_by { |result| result.subject.file }.to_h do |file, results|
+          [relative(file, report.root), file_entry(results, test_ids)]
+        end
+        {
+          schemaVersion: SCHEMA_VERSION,
+          thresholds: THRESHOLDS,
+          projectRoot: report.root,
+          files: files,
+          testFiles: test_files(test_ids),
+          framework: { name: "saboteur", version: VERSION }
+        }.merge(report.seed ? { config: { seed: report.seed } } : {})
+      end
+
+      def self.test_index(results)
+        names = results.flat_map { |result| result.covered_by + result.killed_by }.uniq
+        names.each_with_index.to_h { |name, index| [name, index.to_s] }
+      end
+
+      # RSpec ids look like "./spec/a_spec.rb[1:2]"; anything else is grouped
+      # under an empty path.
+      def self.test_files(test_ids)
+        test_ids.group_by { |name, _| name[%r{\A\.?/?(.+?\.rb)\[}, 1].to_s }.transform_values do |pairs|
+          { tests: pairs.map { |name, id| { id: id, name: name } } }
+        end
+      end
+
+      def self.file_entry(results, test_ids)
+        source = results.first.subject.file_source.dup.force_encoding(Encoding::UTF_8)
+        { language: "ruby", source: source.scrub,
+          mutants: results.map { |result| mutant(result, source, test_ids) } }
+      end
+
+      def self.mutant(result, source, test_ids)
+        entry = {
+          id: result.digest,
+          mutatorName: result.mutation.operator,
+          replacement: result.mutation.edits.map(&:replacement).join(" | "),
+          location: location(result.mutation.edits.first, source),
+          status: status(result),
+          static: false
+        }
+        entry.merge(optional_fields(result, test_ids))
+      end
+
+      def self.optional_fields(result, test_ids)
+        {
+          statusReason: status_reason(result),
+          killedBy: result.killed_by.map { |name| test_ids.fetch(name) }.then { |ids| ids.empty? ? nil : ids },
+          coveredBy: result.covered_by.map { |name| test_ids.fetch(name) }.then { |ids| ids.empty? ? nil : ids },
+          duration: result.runtime && (result.runtime * 1000).round
+        }.compact
+      end
+
+      def self.status(result)
+        case result.status
+        when :killed then "Killed"
+        when :survived then "Survived"
+        when :timeout then "Timeout"
+        when :error, :resource_limit then "RuntimeError"
+        when :skipped then result.reason == "no coverage" ? "NoCoverage" : "Ignored"
+        end
+      end
+
+      def self.status_reason(result)
+        return "ignored: #{result.ignored}" if result.ignored
+        return "resource limit: #{result.reason}" if result.status == :resource_limit
+        return result.crash_kill? ? "crash: #{result.kill_cause}" : result.kill_cause if result.kill_cause
+
+        result.reason.to_s.lines.first&.strip
+      end
+
+      def self.location(edit, source)
+        { start: position(source, edit.start_byte), end: position(source, edit.end_byte) }
+      end
+
+      # 1-based line and column; column counts characters, not bytes.
+      def self.position(source, byte_offset)
+        before = source.byteslice(0, byte_offset).force_encoding(Encoding::UTF_8).scrub
+        line = before.count("\n") + 1
+        column = before.size - (before.rindex("\n") || -1)
+        { line: line, column: column }
+      end
+
+      def self.relative(file, root)
+        file.delete_prefix("#{root}/")
+      end
+    end
+  end
+end