mutineer 1.4.0 → 1.5.0

data/lib/mutineer/kill_channel.rb

@@ -0,0 +1,284 @@
+# frozen_string_literal: true
+
+require "json"
+
+module Mutineer
+  # The line protocol a `--matrix` mutant child uses to tell its parent how its
+  # tests went. One JSON array per line:
+  #
+  #   ["start"]                         the recorder is armed; once, first
+  #   ["pass", file, name, id]          a test passed against the mutant
+  #   ["kill", file, name, id]          a test failed or errored: it killed it
+  #   ["parallel"]                      the run reached its parallel tests; once
+  #   ["skip"]                          what runs next, a run without --matrix
+  #                                     skips (see below)
+  #   ["unskip"]                        the matching end of a ["skip"]
+  #   ["cleanup"]                       every test ran; suite hooks follow; once
+  #   ["lost"]                          a test the recorder could not describe
+  #   ["end"]                           the suite returned with every test seen;
+  #                                     once, last
+  #
+  # `id` tells two tests apart when `[file, name]` does not, and it is what
+  # identifies the test across mutants (an RSpec example id such as
+  # `./spec/calc_spec.rb[1:2]`; the name can change with the mutant). For
+  # Minitest it equals the name. A test line is written when the test is
+  # recorded, so a child killed at the timeout has already sent every kill it
+  # saw. A skipped test sends nothing.
+  #
+  # Minitest runs its serial test classes first and its `parallelize_me!`
+  # classes after them, and the stop at the first failure of a run without
+  # `--matrix` skips everything after a serial kill. `parallel` marks where the
+  # parallel tests begin, so the parent can tell a kill that the stop would have
+  # followed from one it could not. `cleanup` marks where the tests end and the
+  # suite's own hooks begin (RSpec `after(:suite)`). It decides no verdict
+  # (the `skip` regions do); it only marks a later test line as out of order.
+  #
+  # After the first serial kill, a run without `--matrix` skips every later
+  # test and every later test class or example group, but it still runs the
+  # rest of the code around the failing test: the remainder of its Minitest
+  # class `run` wrapper, the `after(:all)` hooks of its RSpec groups, and the
+  # suite hooks. The recorder writes `skip` as it enters something that run
+  # would skip and `unskip` as that returns, so a child that exits, crashes or
+  # hangs inside a skipped region is one the plain run would have stopped
+  # before (see {Isolation.finish}). The pairs nest.
+  #
+  # A row is complete only with `start` and `end`, in order, and no lost line: a
+  # test that exits the process, a crash, an interrupted run or a recorder that
+  # never armed leaves one out. A stream out of order (a duplicate marker, a
+  # test line before `start` or after `end`) is invalid, and the parent trusts
+  # nothing from it.
+  #
+  # Writing never raises: a recorder that broke the test run would change the
+  # verdict, and the exit status decides the verdict. A lost line leaves the row
+  # incomplete. After one write to a channel fails, nothing more is written to
+  # it, so a later `end` can never make a row with a missing line look complete.
+  #
+  # Stdlib-only, so the app-side daemon can load the files that require it.
+  module KillChannel
+    # Event for a test that passed against the mutant.
+    PASS = "pass"
+
+    # Event for a test that failed or errored against the mutant: it killed it.
+    KILL = "kill"
+
+    # Event that opens a recorded run.
+    START = "start"
+
+    # Event that marks the first parallel test of a run.
+    PARALLEL = "parallel"
+
+    # Event that marks the end of the tests and the start of the suite's cleanup.
+    CLEANUP = "cleanup"
+
+    # Event that opens a region a run without `--matrix` would skip.
+    SKIP = "skip"
+
+    # Event that closes the latest open {SKIP} region.
+    UNSKIP = "unskip"
+
+    # Event for a test the recorder could not describe.
+    LOST = "lost"
+
+    # Event that closes a run whose suite returned with every test seen.
+    FINISH = "end"
+
+    # Serializes writes from tests that record on several threads.
+    LOCK = Mutex.new
+
+    # What one child sent. `killed` and `ran` are sorted, unique
+    # `[file, name, id]` tests (`ran` includes the killers); `lost` counts lines
+    # that could not be read, including a partial last line. `parallel` and
+    # `cleanup` say the marker arrived, and `serial_kill` that a test killed
+    # before the `parallel` marker. `skipping` counts the {SKIP} regions still
+    # open when the stream ended. `invalid` is set by any line out of order.
+    Report = Struct.new(:killed, :ran, :started, :finished, :parallel, :cleanup, :serial_kill, :skipping, :invalid,
+                        :lost, keyword_init: true)
+
+    # Writes one test line.
+    #
+    # @param io [IO] the write end of the channel.
+    # @param event [String] {PASS} or {KILL}.
+    # @param file [String] the file that defines the test.
+    # @param name [String] the test's display name, e.g. `CalculatorTest#test_add`.
+    # @param id [String] what tells the test apart from others with its file and
+    #   name; defaults to the name.
+    # @return [void]
+    def self.write(io, event, file, name, id = name)
+      emit(io, [event, utf8(file), utf8(name), utf8(id)])
+    end
+
+    # Writes the `start` line.
+    #
+    # @param io [IO] the write end of the channel.
+    # @return [void]
+    def self.write_start(io)
+      emit(io, [START])
+    end
+
+    # Writes the `parallel` line.
+    #
+    # @param io [IO] the write end of the channel.
+    # @return [void]
+    def self.write_parallel(io)
+      emit(io, [PARALLEL])
+    end
+
+    # Writes the `skip` line.
+    #
+    # @param io [IO] the write end of the channel.
+    # @return [void]
+    def self.write_skip(io)
+      emit(io, [SKIP])
+    end
+
+    # Writes the `unskip` line.
+    #
+    # @param io [IO] the write end of the channel.
+    # @return [void]
+    def self.write_unskip(io)
+      emit(io, [UNSKIP])
+    end
+
+    # Writes the `cleanup` line.
+    #
+    # @param io [IO] the write end of the channel.
+    # @return [void]
+    def self.write_cleanup(io)
+      emit(io, [CLEANUP])
+    end
+
+    # Writes the `lost` line, for a test the recorder could not describe.
+    #
+    # @param io [IO] the write end of the channel.
+    # @return [void]
+    def self.write_lost(io)
+      emit(io, [LOST])
+    end
+
+    # Writes the `end` line.
+    #
+    # @param io [IO] the write end of the channel.
+    # @return [void]
+    def self.write_end(io)
+      emit(io, [FINISH])
+    end
+
+    # Reads every complete line of `buffer`.
+    #
+    # @param buffer [String] bytes read from the channel.
+    # @return [Report]
+    def self.parse(buffer)
+      report = Report.new(killed: [], ran: [], started: false, finished: false, parallel: false, cleanup: false,
+                          serial_kill: false, skipping: 0, invalid: false, lost: 0)
+      buffer.dup.force_encoding(Encoding::UTF_8).each_line do |line|
+        fields = line.end_with?("\n") ? parse_line(line) : nil
+        fields ? apply(report, fields) : report.lost += 1
+      end
+      report.killed = report.killed.uniq.sort
+      report.ran = report.ran.uniq.sort
+      report
+    end
+
+    # Adds one parsed line to `report`. A line out of order marks the report
+    # invalid; the line still counts, since what it names happened.
+    #
+    # @api private
+    # @param report [Report] the report being built.
+    # @param fields [Array<String>] a valid line's fields.
+    # @return [void]
+    def self.apply(report, fields)
+      case fields[0]
+      when START
+        report.invalid = true if report.started || report.ran.any? || report.parallel || report.cleanup || report.finished
+        report.started = true
+      when PARALLEL
+        report.invalid = true unless open?(report) && !report.parallel && !report.cleanup
+        report.parallel = true
+      when SKIP
+        report.invalid = true unless open?(report) && !report.cleanup
+        report.skipping += 1
+      when UNSKIP
+        report.invalid = true unless open?(report) && !report.cleanup && report.skipping.positive?
+        report.skipping -= 1 if report.skipping.positive?
+      when CLEANUP
+        report.invalid = true unless open?(report) && !report.cleanup
+        report.cleanup = true
+      when LOST then report.lost += 1
+      when FINISH
+        report.invalid = true unless open?(report) && !report.finished
+        report.finished = true
+      else
+        report.invalid = true unless open?(report) && !report.cleanup
+        test = fields[1, 3]
+        report.ran << test
+        return unless fields[0] == KILL
+
+        report.killed << test
+        report.serial_kill = true unless report.parallel
+      end
+    end
+
+    # True between `start` and `end`.
+    #
+    # @api private
+    # @param report [Report] the report being built.
+    # @return [Boolean]
+    def self.open?(report)
+      report.started && !report.finished
+    end
+
+    # Parses one line into its fields, or nil when it is not a valid line.
+    #
+    # @api private
+    # @param line [String] one newline-terminated line.
+    # @return [Array<String>, nil]
+    def self.parse_line(line)
+      fields = JSON.parse(line)
+      return unless fields.is_a?(Array) && fields.all?(String)
+
+      valid =
+        case fields[0]
+        when PASS, KILL then fields.size == 4
+        when START, PARALLEL, SKIP, UNSKIP, CLEANUP, LOST, FINISH then fields.size == 1
+        end
+      fields if valid
+    rescue JSON::ParserError
+      nil
+    end
+
+    # Writes one line. Never raises. After a write to `io` fails, writes
+    # nothing more to it.
+    #
+    # @api private
+    # @param io [IO] the write end of the channel.
+    # @param fields [Array<String>] the line's fields.
+    # @return [void]
+    def self.emit(io, fields)
+      line = "#{JSON.generate(fields)}\n"
+      LOCK.synchronize do
+        next if @broken.equal?(io)
+
+        begin
+          io.write(line)
+        rescue StandardError
+          @broken = io
+        end
+      end
+      nil
+    rescue StandardError
+      nil
+    end
+
+    # A UTF-8 copy of `value`, with invalid bytes replaced, so JSON can encode it.
+    #
+    # @api private
+    # @param value [Object] a file path or test name.
+    # @return [String]
+    def self.utf8(value)
+      str = value.to_s
+      str = str.encode(Encoding::UTF_8, invalid: :replace, undef: :replace) unless str.encoding == Encoding::UTF_8
+      str.scrub
+    end
+    private_class_method :apply, :open?, :parse_line, :emit, :utf8
+  end
+end

data/lib/mutineer/kill_matrix.rb

@@ -0,0 +1,88 @@
+# frozen_string_literal: true
+
+require "set"
+
+module Mutineer
+  # Which tests kill which mutants, read from the {Kills} rows of a `--matrix`
+  # run. Pure: it reads results and runs nothing, and it never touches the score.
+  #
+  # A test is a `[file, name, id]` triple (see {Kills}); file and id identify it,
+  # and every row names a test the same way (see {Runner.share_tests}). The tests it knows are
+  # the ones that ran against at least one mutant of this run, so every answer
+  # is relative to the run's mutants: a test of code outside the run's sources
+  # kills nothing here.
+  #
+  # - A blind test ran in at least one complete row and killed no mutant. A
+  #   test seen only in incomplete rows is never called blind.
+  # - A redundant test killed at least one mutant, and each mutant it killed has
+  #   another killer. Each test is judged alone: two redundant tests can be the
+  #   only killers of one mutant, so delete them one at a time. Finding the
+  #   smallest set of tests that keeps every kill is a set-cover problem, out of
+  #   scope here.
+  #
+  # Kills in incomplete rows count, since they did happen. An incomplete row can
+  # miss kills, so when {#complete?} is false a blind test may have killed one
+  # of those mutants.
+  class KillMatrix
+    # The results that carry a {Kills} row, in run order.
+    #
+    # @return [Array<Mutineer::Result>]
+    attr_reader :rows
+
+    # @param results [Array<Mutineer::Result>] every result of the run.
+    def initialize(results)
+      @rows = results.select(&:kills)
+      @kill_counts = Hash.new(0)
+      @rows.each { |r| r.kills.killed_by.each { |test| @kill_counts[test] += 1 } }
+    end
+
+    # Every test that ran against at least one mutant, sorted by file, name and id.
+    #
+    # @return [Array<Array(String, String, String)>]
+    def tests
+      @tests ||= @rows.flat_map { |r| r.kills.ran + r.kills.killed_by }.uniq.sort
+    end
+
+    # True when every row is complete.
+    #
+    # @return [Boolean]
+    def complete?
+      @rows.all? { |r| r.kills.complete }
+    end
+
+    # The rows that are not complete.
+    #
+    # @return [Array<Mutineer::Result>]
+    def incomplete_rows
+      @rows.reject { |r| r.kills.complete }
+    end
+
+    # How many mutants `test` killed.
+    #
+    # @param test [Array(String, String, String)] a `[file, name, id]` test.
+    # @return [Integer]
+    def kill_count(test)
+      @kill_counts[test]
+    end
+
+    # Tests that ran in a complete row and killed no mutant.
+    #
+    # @return [Array<Array(String, String, String)>] sorted.
+    def blind
+      @blind ||= begin
+        seen = @rows.select { |r| r.kills.complete }.flat_map { |r| r.kills.ran }.to_set
+        tests.select { |test| kill_count(test).zero? && seen.include?(test) }
+      end
+    end
+
+    # Tests that killed a mutant, where every mutant they killed has another killer.
+    #
+    # @return [Array<Array(String, String, String)>] sorted.
+    def redundant
+      @redundant ||= begin
+        sole = @rows.filter_map { |r| r.kills.killed_by.first if r.kills.killed_by.size == 1 }.to_set
+        tests.select { |test| kill_count(test).positive? && !sole.include?(test) }
+      end
+    end
+  end
+end

data/lib/mutineer/minitest_integration/kill_recorder.rb

@@ -0,0 +1,301 @@
+# frozen_string_literal: true
+
+require_relative "outer_reporter"
+require_relative "../kill_channel"
+
+module Mutineer
+  class MinitestIntegration
+    # Sends the outcome of each test to the parent of a `--matrix` mutant run,
+    # through {KillChannel}. It never stops the run: every covering test runs,
+    # so the parent learns exactly which tests kill the mutant. Child-process
+    # only, like MinitestIntegration.
+    #
+    # Only the outer reporter counts (see {OuterReporter}): a test can build and
+    # record on its own reporter, and those results are not the run's.
+    module KillRecorder
+      extend OuterReporter::Client
+
+      # Serializes the counting and the writing of results, which Minitest
+      # records from several threads in a parallel class.
+      LOCK = Mutex.new
+
+      # Prepended on each loaded test class: one test's run. Serial and parallel
+      # runs both call it, in Minitest 5 and 6.
+      module MarkSkippedTest
+        # Runs the test inside a `skip` region once a serial test has killed.
+        #
+        # @return [Minitest::Result]
+        def run
+          KillRecorder.skipping { super }
+        end
+      end
+
+      # Prepended on each loaded test class's singleton under Minitest 6: one
+      # class's run and each test's class-level run, user wrappers included.
+      # A run without `--matrix` skips both after a stop (see
+      # StopAtFirstFailure::SkipAfterStop6).
+      module MarkSkippedClass6
+        # Runs the class inside a `skip` region once a serial test has killed.
+        #
+        # @param reporter [Minitest::CompositeReporter]
+        # @param options [Hash]
+        # @return [Object]
+        def run_suite(reporter, options = {})
+          KillRecorder.class_starting(self, reporter)
+          KillRecorder.skipping { super }
+        end
+
+        # Runs one test's class-level step inside a `skip` region once a
+        # serial test has killed.
+        #
+        # @param klass [Class]
+        # @param method_name [String]
+        # @param reporter [Minitest::CompositeReporter]
+        # @return [Object]
+        def run(klass, method_name, reporter)
+          KillRecorder.skipping { super }
+        end
+      end
+
+      # The Minitest 5 form of {MarkSkippedClass6}.
+      module MarkSkippedClass5
+        # Runs the class inside a `skip` region once a serial test has killed.
+        #
+        # @param reporter [Minitest::CompositeReporter]
+        # @param options [Hash]
+        # @return [Object]
+        def run(reporter, options = {})
+          KillRecorder.class_starting(self, reporter)
+          KillRecorder.skipping { super }
+        end
+
+        # Runs one test's class-level step inside a `skip` region once a
+        # serial test has killed.
+        #
+        # @param klass [Class]
+        # @param method_name [String]
+        # @param reporter [Minitest::CompositeReporter]
+        # @return [Object]
+        def run_one_method(klass, method_name, reporter)
+          KillRecorder.skipping { super }
+        end
+      end
+
+      # Prepended on `Minitest::CompositeReporter`.
+      module Record
+        # Records the result, then sends it to the channel.
+        #
+        # @param result [Minitest::Result] the result of one test.
+        # @return [void]
+        def record(result)
+          super
+          KillRecorder.record(self, result)
+        end
+      end
+
+      class << self
+        # The write end of the channel of the armed run.
+        #
+        # @return [IO, nil]
+        attr_accessor :channel
+
+        # True once the `parallel` line has gone out.
+        #
+        # @return [Boolean, nil]
+        attr_accessor :parallel_marked
+
+        # The loaded test classes of the armed run.
+        #
+        # @return [Array<Class>, nil]
+        attr_accessor :runnables
+
+        # How many results the outer reporter has recorded, skips included.
+        #
+        # @return [Integer, nil]
+        attr_accessor :seen
+
+        # True once a serial test killed: from then on, a run without `--matrix`
+        # skips each later test and class.
+        #
+        # @return [Boolean, nil]
+        attr_accessor :serial_killed
+
+        # Installs the hooks, arms the recorder for this process, and writes the
+        # channel's `start` line. Call it after the test files load, so the
+        # recorder knows the test classes. An unknown Minitest shape arms
+        # nothing and writes nothing, so the parent sees no `start` and keeps
+        # the row incomplete.
+        #
+        # @param channel [IO] the write end of the channel.
+        # @param runnables [Array<Class>] the loaded test classes.
+        # @return [Boolean] false when the Minitest shape is unknown.
+        def arm!(channel, runnables)
+          outer = OuterReporter.hook_for_loaded_minitest
+          return false unless outer
+
+          OuterReporter.prepend_once(::Minitest::CompositeReporter, Record)
+          OuterReporter.prepend_once(::Minitest.singleton_class, outer)
+          class_hook = outer.equal?(OuterReporter::Hook6) ? MarkSkippedClass6 : MarkSkippedClass5
+          runnables.each do |klass|
+            OuterReporter.prepend_once(klass, MarkSkippedTest)
+            OuterReporter.prepend_once(klass.singleton_class, class_hook)
+          end
+          OuterReporter.register(self)
+          self.armed_pid = Process.pid
+          self.armed_reporter = nil
+          self.channel = channel
+          self.parallel_marked = false
+          # A copy: Minitest's list is live, and a class a test defines during
+          # the run joins it without running, which would break the count.
+          self.runnables = runnables.dup
+          self.seen = 0
+          self.serial_killed = false
+          KillChannel.write_start(channel)
+          true
+        end
+
+        # Writes the channel's `end` line once the suite has returned and the
+        # outer reporter has seen every test the loaded classes run. A return
+        # alone proves nothing: Minitest catches an Interrupt in a test and
+        # returns after the tests so far. Does nothing unless the recorder is
+        # armed in this process.
+        #
+        # @return [void]
+        def finish!
+          return unless channel && armed_here?
+
+          KillChannel.write_end(channel) if seen == planned_tests
+        end
+
+        # True when the test class runs its tests in parallel threads
+        # (`parallelize_me!`). Minitest runs these after every serial class, and
+        # the stop at the first failure cannot skip the tests it already queued.
+        # Minitest 6 names the order `run_order`; Minitest 5 names it `test_order`.
+        #
+        # @param klass [Class] a loaded test class.
+        # @return [Boolean]
+        def parallel?(klass)
+          %i[run_order test_order].any? { |order| klass.respond_to?(order) && klass.public_send(order) == :parallel }
+        end
+
+        # Clears the armed state.
+        #
+        # @return [void]
+        def disarm!
+          self.armed_pid = nil
+          self.armed_reporter = nil
+          self.channel = nil
+          self.runnables = nil
+          self.serial_killed = false
+        end
+
+        # Writes the `parallel` line when the first parallel class starts on the
+        # outer reporter. The class object is known here, so an anonymous class
+        # (which Minitest records with no name) counts too. Minitest runs every
+        # serial class first, so a kill before this line came from a serial
+        # test. A test that runs a class on its own reporter is not the run's
+        # parallel phase, so it marks nothing.
+        #
+        # @param klass [Class] the test class starting its run.
+        # @param reporter [Minitest::CompositeReporter] the reporter it runs on.
+        # @return [void]
+        def class_starting(klass, reporter)
+          return unless channel && outer_reporter?(reporter) && parallel?(klass)
+
+          LOCK.synchronize { mark_parallel }
+        end
+
+        # Runs the block between `skip` and `unskip` when a serial test has
+        # killed in this armed process, else just runs it. No `ensure`: a block
+        # that never returns (an exit, a crash) must leave its region open, so
+        # the parent knows the child ended where a run without `--matrix`
+        # would not have been.
+        #
+        # @yieldreturn [Object]
+        # @return [Object] the block's value.
+        def skipping
+          open = channel && serial_killed && armed_here?
+          KillChannel.write_skip(channel) if open
+          value = yield
+          KillChannel.write_unskip(channel) if open
+          value
+        end
+
+        # Sends one result recorded on `reporter`: a pass, or a kill for a
+        # failure or an error. Skips send no test line, though they count as
+        # seen, and results on other reporters count for nothing. The
+        # `parallel` line goes out when the first parallel class starts (see
+        # {.class_starting}), so a kill that precedes it came from a serial test.
+        #
+        # @param reporter [Minitest::CompositeReporter] the recording reporter.
+        # @param result [Minitest::Result] the result of one test.
+        # @return [void]
+        def record(reporter, result)
+          return unless channel && outer_reporter?(reporter)
+
+          LOCK.synchronize do
+            self.seen += 1
+            send_result(result)
+          end
+        end
+
+        private
+
+        # Writes one result's line. A recorder that raised would change the
+        # verdict, so a result it cannot describe (a Minitest without
+        # `source_location` or `klass` on its results) is a `lost` line, which
+        # keeps the row incomplete, as on the RSpec side.
+        #
+        # @param result [Minitest::Result] the result of one test.
+        # @return [void]
+        def send_result(result)
+          return if result.skipped?
+
+          event = result.passed? ? KillChannel::PASS : KillChannel::KILL
+          self.serial_killed = true if event == KillChannel::KILL && !parallel_marked
+          file, line = Array(result.source_location)
+          KillChannel.write(channel, event, file, *name_and_id(result, line))
+        rescue StandardError
+          KillChannel.write_lost(channel)
+        end
+
+        # The test's name and id. `Class#method` is unique within a run, so the
+        # name is also the id. An anonymous class has no name (Minitest records
+        # nil), so its tests would all merge into `#method`; their id adds the
+        # line that defines the method, which no mutant changes. The test's file
+        # is already part of its identity, so the id holds no path.
+        #
+        # @param result [Minitest::Result] the result of one test.
+        # @param line [Integer, nil] the line that defines the test method.
+        # @return [Array(String, String)] name and id.
+        def name_and_id(result, line)
+          return ["#{result.klass}##{result.name}"] * 2 unless result.klass.to_s.empty?
+
+          name = "(anonymous)##{result.name}"
+          [name, "#{name}@#{line}"]
+        end
+
+        # Writes the `parallel` line, once.
+        #
+        # @return [void]
+        def mark_parallel
+          return if parallel_marked
+
+          self.parallel_marked = true
+          KillChannel.write_parallel(channel)
+        end
+
+        # How many tests the loaded classes ran, or nil when it cannot be told.
+        # Asked after the run, because `runnable_methods` shuffles with the
+        # seed that `Minitest.run` sets.
+        #
+        # @return [Integer, nil]
+        def planned_tests
+          runnables.sum { |klass| klass.runnable_methods.size }
+        rescue StandardError
+          nil
+        end
+      end
+    end
+  end
+end