mutaruby 0.1.0

data/lib/mutaruby/reporters.rb

@@ -0,0 +1,103 @@
+# frozen_string_literal: true
+
+require 'json'
+require 'cgi'
+require_relative 'version'
+
+module Mutaruby
+  module Reporters
+    module_function
+
+    def score_text(report)
+      s = report.score
+      s.nil? ? 'n/a' : format('%.2f%%', s)
+    end
+
+    def human(report, io, verbose: false)
+      survivors = report.outcomes.select { |o| o.status == :survived }
+      if survivors.any?
+        io.puts 'Surviving mutants (not detected by the test command):'
+        survivors.each do |o|
+          m = o.mutation
+          io.puts
+          io.puts "  #{m.location}#{m.method ? " in #{m.method}" : ''}  [##{m.id}]"
+          io.puts "    #{m.label}"
+          m.diff(report.sources[m.file]).each_line { |l| io.puts "    #{l.chomp}" }
+        end
+        io.puts
+      end
+      if verbose
+        report.outcomes.select { |o| o.status == :timeout }.each do |o|
+          io.puts "  timeout: #{o.mutation.location} #{o.mutation.label}"
+        end
+        report.outcomes.select { |o| o.status == :equivalent }.each do |o|
+          io.puts "  equivalent: #{o.mutation.location} #{o.mutation.label} (#{o.ignore.reason})"
+        end
+      end
+      report.stale.each { |e| io.puts "warning: STALE ignore entry (no surviving mutant at this site; remove or re-review it): #{e.label}" }
+      report.warnings.each { |w| io.puts "warning: #{w}" }
+      io.puts 'mutaruby summary'
+      io.puts "  files:       #{report.files.size}"
+      io.puts "  mutants:     #{report.outcomes.size}"
+      io.puts "  killed:      #{report.killed} (#{report.count(:timeout)} by timeout)"
+      io.puts "  survived:    #{report.count(:survived)}"
+      io.puts "  equivalent:  #{report.count(:equivalent)} (ignore file)"
+      io.puts "  score:       #{score_text(report)}  = killed / (mutants - equivalent)"
+      io.puts format('  time:        %.1fs (baseline %.2fs, timeout %ss, jobs %d)', report.seconds,
+                     report.baseline_seconds, report.timeout, report.jobs)
+    end
+
+    def json_data(report)
+      {
+        tool: 'mutaruby', version: VERSION, command: report.command,
+        score: report.score&.round(4), complete: report.complete,
+        counts: { mutants: report.outcomes.size, killed: report.killed, timeout: report.count(:timeout),
+                  survived: report.count(:survived), equivalent: report.count(:equivalent) },
+        files: report.files, baseline_seconds: report.baseline_seconds.round(3), timeout: report.timeout,
+        mutants: report.outcomes.map do |o|
+          o.mutation.to_h.merge(status: o.status.to_s, seconds: o.seconds.round(3), ignore_reason: o.ignore&.reason)
+        end,
+        stale_ignores: report.stale.map(&:to_h).map { |h| h.compact.except(:index) },
+        warnings: report.warnings
+      }
+    end
+
+    def json(report, io)
+      io.puts JSON.pretty_generate(json_data(report))
+    end
+
+    # JUnit-style XML: one testcase per mutant. A survivor is a failure, an equivalent mutant is skipped.
+    def junit(report, io)
+      x = ->(s) { CGI.escapeHTML(s.to_s) }
+      failures = report.count(:survived)
+      io.puts '<?xml version="1.0" encoding="UTF-8"?>'
+      io.puts %(<testsuites name="mutaruby" tests="#{report.outcomes.size}" failures="#{failures}" time="#{report.seconds.round(3)}">)
+      io.puts %(  <testsuite name="mutaruby" tests="#{report.outcomes.size}" failures="#{failures}" errors="0" ) +
+              %(skipped="#{report.count(:equivalent)}" time="#{report.seconds.round(3)}">)
+      io.puts %(    <properties><property name="score" value="#{x.call(score_text(report))}"/></properties>)
+      report.outcomes.each do |o|
+        m = o.mutation
+        name = "##{m.id} #{m.location} #{m.label}"
+        io.print %(    <testcase classname="#{x.call(m.file)}" name="#{x.call(name)}" time="#{o.seconds.round(3)}")
+        case o.status
+        when :survived
+          io.puts '>'
+          io.puts %(      <failure message="#{x.call("survived: #{m.label}")}" type="survived">#{x.call(m.diff(report.sources[m.file]))}</failure>)
+          io.puts '    </testcase>'
+        when :equivalent
+          io.puts '>'
+          io.puts %(      <skipped message="#{x.call("equivalent: #{o.ignore.reason}")}"/>)
+          io.puts '    </testcase>'
+        when :timeout
+          io.puts '>'
+          io.puts '      <system-out>killed by timeout</system-out>'
+          io.puts '    </testcase>'
+        else
+          io.puts '/>'
+        end
+      end
+      io.puts '  </testsuite>'
+      io.puts '</testsuites>'
+    end
+  end
+end

data/lib/mutaruby/runner.rb

@@ -0,0 +1,181 @@
+# frozen_string_literal: true
+
+require 'set'
+require_relative 'generator'
+require_relative 'ignore_file'
+require_relative 'sandbox'
+require_relative 'command'
+require_relative 'selection'
+
+module Mutaruby
+  class BaselineFailed < Error; end
+
+  # Generates the mutants, checks the baseline, runs every mutant and collects the verdicts.
+  class Runner
+    Outcome = Struct.new(:mutation, :status, :seconds, :ignore, keyword_init: true) do
+      # :killed, :timeout (also a kill), :survived, :equivalent (survived, listed in the ignore file)
+      def killed? = %i[killed timeout].include?(status)
+    end
+
+    Report = Struct.new(:root, :command, :files, :outcomes, :sources, :stale, :warnings, :baseline_seconds, :timeout,
+                        :jobs, :seconds, :complete, keyword_init: true) do
+      def count(status) = outcomes.count { |o| o.status == status }
+      def killed = count(:killed) + count(:timeout)
+
+      # killed / (killed + survived); equivalents are left out of both. nil when nothing was scored.
+      def score
+        scored = killed + count(:survived)
+        scored.zero? ? nil : (100.0 * killed / scored)
+      end
+    end
+
+    Options = Struct.new(:root, :command, :patterns, :since, :jobs, :timeout, :timeout_factor, :methods,
+                         :exclude_methods, :lines, :exclude_lines, :shard, :ignore_file, :operators, :copy_excludes,
+                         :progress, :keyword_init => true)
+
+    def initialize(opts, out: $stderr)
+      @o = opts
+      @out = out
+      @root = File.expand_path(opts.root || Dir.pwd)
+      @warnings = []
+    end
+
+    # Mutants of the selected files after every filter, plus what is needed to report them.
+    def plan
+      @plan ||= begin
+        sel = selection
+        files = sel.files
+        sources = files.to_h { |f| [f, File.read(File.join(@root, f), encoding: 'UTF-8')] }
+        all = files.flat_map do |f|
+          g = Generator.new(sources[f], path: File.join(@root, f), rel: f, operators: operator_set)
+          ms = g.mutations
+          @warnings.concat(g.errors)
+          ms
+        end
+        all.each_with_index { |m, i| m.id = i }
+        ignore = load_ignore
+        if ignore
+          errs = ignore.validate(sources)
+          raise Error, "ignore file #{ignore.path}:\n  #{errs.join("\n  ")}" if errs.any?
+        end
+        { files: files, sources: sources, mutations: sel.filter(all), ignore: ignore, complete: sel.complete? && @o.operators.nil? }
+      end
+    end
+
+    def run
+      t0 = monotonic
+      p = plan
+      sandboxes = []
+      sandboxes << Sandbox.new(@root, excludes: @o.copy_excludes || [])
+      cmd = Command.new(@o.command, root: @root)
+      base = cmd.run(sandboxes[0].dir, capture: true)
+      unless base.passed?
+        raise BaselineFailed, "the test command fails on the UNMUTATED code (#{base.timeout? ? 'timed out' : "exit #{base.exitstatus}"}); " \
+                              "every mutant would count as killed, so nothing would be measured.\n" \
+                              "command: #{@o.command}\n--- output (tail) ---\n#{base.output}"
+      end
+      timeout = @o.timeout || ((base.seconds * (@o.timeout_factor || 3.0)) + 5).ceil
+      outcomes = execute(p, cmd, sandboxes, timeout)
+      stale = []
+      if p[:ignore]
+        matched = outcomes.select { |o| o.status == :equivalent }.map(&:ignore).to_set
+        stale = p[:complete] ? p[:ignore].stale(matched, p[:files]) : []
+      end
+      Report.new(root: @root, command: @o.command, files: p[:files], outcomes: outcomes, sources: p[:sources],
+                 stale: stale, warnings: @warnings, baseline_seconds: base.seconds, timeout: timeout,
+                 jobs: jobs, seconds: monotonic - t0, complete: p[:complete])
+    rescue Interrupt, SignalException
+      cmd&.kill_all
+      raise
+    ensure
+      sandboxes&.each(&:cleanup)
+    end
+
+    private
+
+    def jobs
+      [@o.jobs || 1, 1].max
+    end
+
+    def selection
+      Selection.new(root: @root, patterns: @o.patterns || [], since: @o.since, methods: @o.methods || [],
+                    exclude_methods: @o.exclude_methods || [], lines: @o.lines || [],
+                    exclude_lines: @o.exclude_lines || [], shard: @o.shard)
+    end
+
+    def operator_set
+      return nil unless @o.operators
+
+      @o.operators.to_set
+    end
+
+    def load_ignore
+      path = @o.ignore_file
+      default = File.join(@root, '.mutaruby-ignore.yml')
+      path ||= default if File.exist?(default)
+      return nil unless path
+
+      raise Error, "ignore file not found: #{path}" unless File.exist?(path)
+
+      IgnoreFile.load(path)
+    end
+
+    def execute(plan, cmd, sandboxes, timeout)
+      queue = Queue.new
+      plan[:mutations].each { |m| queue << m }
+      n = [jobs, plan[:mutations].size].min
+      (sandboxes.size...n).each { sandboxes << Sandbox.new(@root, excludes: @o.copy_excludes || []) }
+      results = []
+      lock = Mutex.new
+      threads = Array.new(n) do |w|
+        Thread.new do
+          box = sandboxes[w]
+          loop do
+            m = begin
+              queue.pop(true)
+            rescue ThreadError
+              break
+            end
+            original = plan[:sources][m.file]
+            r = box.with_file(m.file, m.text, original) do
+              cmd.run(box.dir, timeout: timeout,
+                               env: { 'MUTARUBY_MUTANT_ID' => m.id.to_s, 'MUTARUBY_FILE' => m.file })
+            end
+            entry = plan[:ignore]&.match(m)
+            status = if r.timeout? then :timeout
+                     elsif !r.passed? then :killed
+                     elsif entry then :equivalent
+                     else :survived
+                     end
+            o = Outcome.new(mutation: m, status: status, seconds: r.seconds, ignore: entry)
+            lock.synchronize do
+              results << o
+              progress(o)
+            end
+          end
+        end
+      end
+      begin
+        threads.each(&:join)
+      rescue Exception # rubocop:disable Lint/RescueException -- stop the workers, then re-raise
+        threads.each(&:kill)
+        raise
+      end
+      @out.puts if @o.progress && results.any?
+      results.sort_by { |o| o.mutation.id }
+    end
+
+    MARK = { killed: '.', timeout: 'T', survived: 'S', equivalent: 'e' }.freeze
+
+    def progress(outcome)
+      return unless @o.progress
+
+      @out.print(MARK[outcome.status])
+      @out.flush
+    end
+
+    def monotonic
+      Process.clock_gettime(Process::CLOCK_MONOTONIC)
+    end
+  end
+end

data/lib/mutaruby/sandbox.rb

@@ -0,0 +1,82 @@
+# frozen_string_literal: true
+
+require 'fileutils'
+require 'find'
+require 'tmpdir'
+
+module Mutaruby
+  # A private copy of the project. Every test-command run happens inside one, so the user's files are never written:
+  # a crash, a timeout or a Ctrl-C in the middle of a run cannot leave a mutant behind in the working tree.
+  #
+  # Each worker owns one sandbox. A mutant is written into it, the test command runs, and the original file content
+  # is written back before the next mutant.
+  class Sandbox
+    DEFAULT_EXCLUDES = %w[.git coverage].freeze
+
+    attr_reader :dir
+
+    def initialize(root, excludes: [], prefix: 'mutaruby-')
+      @root = File.expand_path(root)
+      @excludes = DEFAULT_EXCLUDES + excludes
+      @prefix = prefix
+      @base = Dir.mktmpdir(prefix)
+      @dir = File.join(@base, File.basename(@root))
+      copy_tree
+    end
+
+    def path(rel)
+      File.join(@dir, rel)
+    end
+
+    # Writes `text` to `rel` inside the sandbox, yields, and restores the original bytes afterwards (also on error).
+    def with_file(rel, text, original)
+      target = path(rel)
+      File.binwrite(target, text)
+      yield
+    ensure
+      File.binwrite(target, original) if target
+    end
+
+    def cleanup
+      FileUtils.rm_rf(@base)
+    end
+
+    private
+
+    def excluded?(rel)
+      @excludes.any? do |pat|
+        File.fnmatch?(pat, rel, File::FNM_PATHNAME | File::FNM_DOTMATCH) ||
+          File.fnmatch?(pat, File.basename(rel), File::FNM_DOTMATCH)
+      end
+    end
+
+    # A sandbox directory (this one or another worker's). When the temporary directory lies inside the project (for
+    # example TMPDIR=./tmp), copying it would copy the sandbox into itself.
+    def sandbox?(path)
+      File.basename(path).start_with?(@prefix) && File.directory?(path) &&
+        File.realpath(File.dirname(path)) == File.realpath(File.dirname(@base))
+    end
+
+    def copy_tree
+      FileUtils.mkdir_p(@dir)
+      Find.find(@root) do |src|
+        next if src == @root
+
+        rel = src.delete_prefix("#{@root}/")
+        if excluded?(rel) || sandbox?(src)
+          Find.prune
+          next
+        end
+        dest = File.join(@dir, rel)
+        st = File.lstat(src)
+        if st.symlink?
+          File.symlink(File.readlink(src), dest)
+        elsif st.directory?
+          FileUtils.mkdir_p(dest)
+        elsif st.file?
+          FileUtils.cp(src, dest, preserve: true)
+        end
+      end
+    end
+  end
+end

data/lib/mutaruby/selection.rb

@@ -0,0 +1,107 @@
+# frozen_string_literal: true
+
+require 'open3'
+
+module Mutaruby
+  # Which files to mutate and which of their mutants to run.
+  class Selection
+    DEFAULT_PATTERNS = ['lib/**/*.rb'].freeze
+
+    LineSpec = Struct.new(:file, :from, :to) do
+      def cover?(mutation)
+        (file.nil? || file == mutation.file) && mutation.lineno.between?(from, to)
+      end
+    end
+
+    def self.parse_line_spec(spec)
+      m = spec.match(/\A(?:(.+):)?(\d+)(?:-(\d+))?\z/)
+      raise Error, "bad line spec #{spec.inspect} (expected [FILE:]LINE or [FILE:]FROM-TO)" unless m
+
+      from = m[2].to_i
+      to = (m[3] || m[2]).to_i
+      raise Error, "bad line spec #{spec.inspect} (FROM > TO)" if from > to
+
+      LineSpec.new(m[1]&.sub(%r{\A\./}, ''), from, to)
+    end
+
+    def self.parse_shard(spec)
+      m = spec.to_s.match(%r{\A(\d+)/(\d+)\z})
+      raise Error, "bad --shard #{spec.inspect} (expected I/N, 1 <= I <= N)" unless m && m[2].to_i.positive? && m[1].to_i.between?(1, m[2].to_i)
+
+      [m[1].to_i, m[2].to_i]
+    end
+
+    def initialize(root:, patterns: [], since: nil, methods: [], exclude_methods: [], lines: [], exclude_lines: [],
+                   shard: nil)
+      @root = File.expand_path(root)
+      @patterns = patterns.empty? ? DEFAULT_PATTERNS : patterns
+      @since = since
+      @methods = methods
+      @exclude_methods = exclude_methods
+      @lines = lines.map { |s| s.is_a?(LineSpec) ? s : self.class.parse_line_spec(s) }
+      @exclude_lines = exclude_lines.map { |s| s.is_a?(LineSpec) ? s : self.class.parse_line_spec(s) }
+      @shard = shard
+    end
+
+    # True when every mutant of each selected file is run (so stale ignore entries can be detected).
+    def complete?
+      @methods.empty? && @exclude_methods.empty? && @lines.empty? && @exclude_lines.empty? && @shard.nil?
+    end
+
+    # Sorted paths relative to the root.
+    def files
+      found = @patterns.flat_map do |pat|
+        full = File.expand_path(pat, @root)
+        if File.file?(full)
+          [relative(full)]
+        else
+          Dir.glob(pat, base: @root).select { |f| File.file?(File.join(@root, f)) }
+        end
+      end
+      found = found.map { |f| f.sub(%r{\A\./}, '') }.uniq.sort
+      found &= changed_files if @since
+      found
+    end
+
+    def filter(mutations)
+      kept = mutations.select do |m|
+        (@methods.empty? || @methods.any? { |p| method_match?(p, m.method) }) &&
+          @exclude_methods.none? { |p| method_match?(p, m.method) } &&
+          (@lines.empty? || @lines.any? { |s| s.cover?(m) }) &&
+          @exclude_lines.none? { |s| s.cover?(m) }
+      end
+      return kept unless @shard
+
+      i, n = @shard
+      kept.each_with_index.select { |_, idx| idx % n == i - 1 }.map(&:first)
+    end
+
+    private
+
+    def relative(full)
+      full.delete_prefix("#{@root}/")
+    end
+
+    # A pattern matches the qualified name ("Calc#add", "Calc.build", "Calc#*") or the bare method name ("add").
+    def method_match?(pattern, name)
+      return false if name.nil?
+
+      bare = name.split(/[#.]/).last
+      File.fnmatch?(pattern, name) || File.fnmatch?(pattern, bare)
+    end
+
+    # Files changed relative to a git ref: committed, staged and unstaged edits, plus new untracked files.
+    def changed_files
+      diff = git('diff', '--name-only', '--relative', @since, '--')
+      untracked = git('ls-files', '--others', '--exclude-standard')
+      (diff + untracked).uniq
+    end
+
+    def git(*args)
+      out, err, st = Open3.capture3('git', '-C', @root, *args)
+      raise Error, "git #{args.join(' ')} failed: #{err.strip}" unless st.success?
+
+      out.split("\n").reject(&:empty?)
+    end
+  end
+end

data/lib/mutaruby/splicer.rb

@@ -0,0 +1,27 @@
+# frozen_string_literal: true
+
+module Mutaruby
+  # Raised when a splice would not replace exactly the node's own source. Aborts the whole run: a mutant placed at
+  # the wrong offset tests nothing, and if it "survives" it reports a test gap that does not exist.
+  class SpliceError < StandardError; end
+
+  # Replaces source[begin_pos...end_pos] with a replacement.
+  #
+  # The `parser` gem reports CHARACTER offsets, so the slice is taken by characters. Slicing by bytes would shift every
+  # mutant after the first multi-byte character in the file (an accented letter in a comment is enough) onto the wrong
+  # text. The self-check makes that class of bug (bytes vs characters, a stale buffer, a wrong range) fail loudly:
+  # the slice being replaced must be exactly the node's own source text, or nothing is spliced.
+  module Splicer
+    module_function
+
+    def splice(source, begin_pos, end_pos, expected, replacement)
+      actual = source[begin_pos...end_pos]
+      unless actual == expected
+        raise SpliceError, "splice self-check failed at offsets #{begin_pos}...#{end_pos}: " \
+                           "the slice is #{actual.inspect}, the node's source is #{expected.inspect}"
+      end
+
+      source[0...begin_pos] + replacement + source[end_pos..]
+    end
+  end
+end

data/lib/mutaruby/version.rb

@@ -0,0 +1,5 @@
+# frozen_string_literal: true
+
+module Mutaruby
+  VERSION = '0.1.0'
+end

data/lib/mutaruby.rb

@@ -0,0 +1,19 @@
+# frozen_string_literal: true
+
+# mutaruby: mutation testing for Ruby. See README.md.
+module Mutaruby
+  class Error < StandardError; end
+end
+
+require_relative 'mutaruby/version'
+require_relative 'mutaruby/parsing'
+require_relative 'mutaruby/splicer'
+require_relative 'mutaruby/mutation'
+require_relative 'mutaruby/generator'
+require_relative 'mutaruby/ignore_file'
+require_relative 'mutaruby/sandbox'
+require_relative 'mutaruby/command'
+require_relative 'mutaruby/selection'
+require_relative 'mutaruby/runner'
+require_relative 'mutaruby/reporters'
+require_relative 'mutaruby/cli'

metadata

@@ -0,0 +1,84 @@
+--- !ruby/object:Gem::Specification
+name: mutaruby
+version: !ruby/object:Gem::Version
+  version: 0.1.0
+platform: ruby
+authors:
+- Atlas Authorization
+bindir: exe
+cert_chain: []
+date: 1980-01-02 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: parser
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '3.2'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '4'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '3.2'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '4'
+description: mutaruby changes your code in small ways (mutants) and runs your test
+  command against each change. A mutant your tests do not notice is a gap in your
+  tests. Works with any test command (rspec, minitest, rake), runs every mutant in
+  a fresh process inside a private copy of the project, and reports a mutation score
+  as text, JSON or JUnit XML.
+executables:
+- mutaruby
+extensions: []
+extra_rdoc_files: []
+files:
+- CHANGELOG.md
+- LICENSE
+- README.md
+- exe/mutaruby
+- lib/mutaruby.rb
+- lib/mutaruby/cli.rb
+- lib/mutaruby/command.rb
+- lib/mutaruby/generator.rb
+- lib/mutaruby/ignore_file.rb
+- lib/mutaruby/mutation.rb
+- lib/mutaruby/parsing.rb
+- lib/mutaruby/reporters.rb
+- lib/mutaruby/runner.rb
+- lib/mutaruby/sandbox.rb
+- lib/mutaruby/selection.rb
+- lib/mutaruby/splicer.rb
+- lib/mutaruby/version.rb
+homepage: https://github.com/Atlas-Authorization/mutaruby
+licenses:
+- MIT
+metadata:
+  homepage_uri: https://github.com/Atlas-Authorization/mutaruby
+  source_code_uri: https://github.com/Atlas-Authorization/mutaruby
+  changelog_uri: https://github.com/Atlas-Authorization/mutaruby/blob/main/CHANGELOG.md
+  bug_tracker_uri: https://github.com/Atlas-Authorization/mutaruby/issues
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '3.1'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - ">="
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubygems_version: 4.0.16
+specification_version: 4
+summary: Free, MIT-licensed mutation testing for Ruby
+test_files: []