henitai 0.5.1 → 0.6.0

data/lib/henitai/integration/mutant_run_support.rb

@@ -10,8 +10,11 @@ module Henitai
     # The framework-specific test invocation is delegated to #run_tests, which
     # including classes must implement.
     module MutantRunSupport
+      # The previous attempt's report is cleared before the fork, so a retried
+      # child that dies early can never be classified by stale data.
       def spawn_mutant(mutant:, test_files:)
-        log_paths = scenario_log_paths(mutant_log_name(mutant))
+        log_paths = mutant_log_paths(mutant)
+        child_report_store.clear(log_paths[:report_path])
         RspecProcessRunner.new.spawn_mutant(self, mutant:, test_files:, log_paths:)
       end
 
@@ -29,17 +32,22 @@ module Henitai
         }
       end
 
+      # Mutant runs carry a report path and are classified from the child's
+      # report; the baseline suite has none and keeps exit-status semantics.
+      def mutant_log_paths(mutant)
+        log_paths = scenario_log_paths(mutant_log_name(mutant))
+        log_paths.merge(report_path: log_paths[:log_path].sub(/\.log\z/, ".report.json"))
+      end
+
       def build_result(wait_result, log_paths)
         stdout = scenario_log_support.read_log_file(log_paths[:stdout_path])
         stderr = scenario_log_support.read_log_file(log_paths[:stderr_path])
         scenario_log_support.write_combined_log(log_paths[:log_path], stdout, stderr)
+        output = { stdout:, stderr:, log_path: log_paths[:log_path] }
+        return ScenarioExecutionResult.build(wait_result:, **output) unless log_paths.key?(:report_path)
 
-        ScenarioExecutionResult.build(
-          wait_result:,
-          stdout:,
-          stderr:,
-          log_path: log_paths[:log_path]
-        )
+        report = child_report_store.read(log_paths[:report_path])
+        ScenarioExecutionResult.build_for_mutant(wait_result:, report:, **output)
       end
 
       def run_in_child(mutant:, test_files:, log_paths:)
@@ -61,17 +69,69 @@ module Henitai
       private
 
       def run_child_activation_and_tests(mutant:, test_files:, log_paths:)
+        report = ->(data) { child_report_store.write(log_paths[:report_path], data) }
         scenario_log_support.with_coverage_dir(mutant.id) do
           scenario_log_support.capture_child_output(log_paths) do
-            child_debug_log.mutant_meta(mutant)
-            child_debug_log.activation_start(mutant.id)
-            activation_result = Mutant::Activator.activate!(mutant)
-            child_debug_log.activation_check
-            child_debug_log.activation_end(activation_result, test_files:)
-            activation_result == :compile_error ? 2 : run_tests(test_files)
+            run_tests_in_child(mutant, test_files, report)
           end
         end
       end
+
+      # Any failure to put the mutant in place is a harness outcome: the tests
+      # never ran against the mutation, so nothing may count as a kill.
+      def activate_in_child(mutant, test_files, report)
+        return true unless activate_with_debug_log(mutant, test_files) == :compile_error
+
+        report.call("outcome" => "activation_failed", "reason" => "the activation source does not compile")
+        false
+      rescue StandardError, ScriptError => e
+        warn "henitai: mutant activation failed: #{e.class}: #{e.message}"
+        report.call("outcome" => "activation_failed", "reason" => "#{e.class}: #{e.message}")
+        false
+      end
+
+      def activate_with_debug_log(mutant, test_files)
+        log = child_debug_log
+        log.mutant_meta(mutant)
+        log.activation_start(mutant.id)
+        activation_result = Mutant::Activator.activate!(mutant)
+        log.activation_check
+        log.activation_end(activation_result, test_files:)
+        activation_result
+      end
+
+      # RSpec and Minitest let SystemExit through by design. Raised by the
+      # code under test it ends the run midway, so it is reported instead of
+      # becoming the child's exit status.
+      #
+      # The mutant is activated from #run_tests' after-load hook: once the test
+      # files (and the code they require) are loaded, before any test runs.
+      # Activating earlier would load the subject file on its own, without the
+      # superclasses and constants the rest of the application defines.
+      def run_tests_in_child(mutant, test_files, report)
+        activated = nil
+        outcomes = TestFileOutcomes.new
+        exit_code = run_tests(test_files, outcomes) { activated = activate_in_child(mutant, test_files, report) }
+        return finish_without_tests(activated, report) unless activated
+
+        report.call(
+          "outcome" => "tests_finished",
+          "exit_code" => exit_code,
+          "failed_tests" => outcomes.failed_files,
+          "unattributed_failures" => outcomes.unattributed_failures?
+        )
+        exit_code
+      rescue SystemExit => e
+        report.call("outcome" => "system_exit", "status" => e.status)
+        1
+      end
+
+      # A nil activation means the hook never ran: the test files failed to
+      # load, before the mutant was in place.
+      def finish_without_tests(activated, report)
+        report.call("outcome" => "environment_failed") if activated.nil?
+        2
+      end
     end
   end
 end

data/lib/henitai/integration/rspec_child_runner.rb

@@ -8,7 +8,20 @@ module Henitai
     module RspecChildRunner
       private
 
-      def run_rspec_runner(test_files)
+      def run_rspec_runner(test_files, &after_load)
+        log = child_debug_log
+        runner = @preloaded_runner || prepare_rspec_runner(test_files)
+        return 1 unless ready_to_run?(after_load)
+
+        run_rspec_specs(runner, example_groups_to_run(test_files))
+      rescue SystemExit => e
+        log.write("[henitai-debug-child] runner_run_system_exit status=#{e.status.inspect}")
+        raise
+      ensure
+        child_debug_log.write("[henitai-debug-child] runner_run_ensure")
+      end
+
+      def prepare_rspec_runner(test_files)
         log = child_debug_log
         log.write("[henitai-debug-child] build_rspec_runner_start")
         runner = build_rspec_runner
@@ -17,12 +30,34 @@ module Henitai
         configure_rspec_runner(runner)
         log.write("[henitai-debug-child] configure_rspec_runner_return")
         load_rspec_spec_files(test_files)
-        run_rspec_specs(runner)
-      rescue SystemExit => e
-        log.write("[henitai-debug-child] runner_run_system_exit status=#{e.status.inspect}")
-        raise
-      ensure
-        child_debug_log.write("[henitai-debug-child] runner_run_ensure")
+        runner
+      end
+
+      # A fork-server process loaded the union of the run's spec files; each
+      # mutant process runs only the groups defined in its own files.
+      def example_groups_to_run(test_files)
+        groups = ::RSpec.world.ordered_example_groups
+        return groups unless @preloaded_runner
+
+        wanted = test_files.map { |file| File.expand_path(file) }
+        groups.select { |group| wanted.include?(example_group_file(group)) }
+      end
+
+      def example_group_file(group)
+        metadata = group.metadata
+        metadata[:absolute_file_path] || File.expand_path(metadata[:file_path])
+      end
+
+      # The after-load hook (mutant activation) runs only when every spec file
+      # loaded; a falsy hook result also skips the run.
+      def ready_to_run?(after_load)
+        rspec_loaded_cleanly? && (after_load.nil? || after_load.call)
+      end
+
+      # RSpec records a spec file that failed to load and quits before running.
+      def rspec_loaded_cleanly?
+        world = ::RSpec.world
+        !(world.wants_to_quit || world.rspec_is_quitting)
       end
 
       def build_rspec_runner
@@ -51,9 +86,9 @@ module Henitai
         child_debug_log.write("[henitai-debug-child] load_spec_files_return")
       end
 
-      def run_rspec_specs(runner)
+      def run_rspec_specs(runner, example_groups)
         child_debug_log.write("[henitai-debug-child] run_specs_start")
-        result = runner.send(:run_specs, ::RSpec.world.ordered_example_groups)
+        result = runner.send(:run_specs, example_groups)
         child_debug_log.write("[henitai-debug-child] run_specs_return result=#{result.inspect}")
         result
       end

data/lib/henitai/integration/rspec_preload.rb

@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+
+module Henitai
+  module Integration
+    # Lets the fork server (ADR-15) boot the RSpec environment once. The
+    # booted runner stays in the server; every forked mutant process then runs
+    # its own subset of the loaded example groups.
+    module RspecPreload
+      class PreloadError < StandardError; end
+
+      def fork_server_supported?
+        true
+      end
+
+      # Loads RSpec, its configured requires (spec_helper, rails_helper, ...)
+      # and the given spec files, without activating a mutant or running a
+      # test. Coverage startup is suppressed before anything loads, as it is
+      # in every mutant child.
+      #
+      # @raise [PreloadError] when a spec file failed to load
+      def preload_environment(test_files)
+        CoverageRuntimeSuppressors.suppress_simplecov!
+        CoverageRuntimeSuppressors.suppress_coverage!
+        subprocess_env.each { |key, value| ENV[key] = value }
+        require "rspec/core"
+        ::RSpec.configuration.fail_if_no_examples = true
+        runner = prepare_rspec_runner(test_files)
+        raise PreloadError, "the spec files failed to load" unless rspec_loaded_cleanly?
+
+        @preloaded_runner = runner
+        nil
+      end
+    end
+  end
+end

data/lib/henitai/integration/rspec_process_runner.rb

@@ -52,7 +52,10 @@ module Henitai
     end
 
     # Captures the PID and log paths for a spawned mutant child process.
-    ChildHandle = Struct.new(:pid, :log_paths)
+    # +timeout+ is set only when the spawner knows better than the scheduler's
+    # resolver how long this child may run (a fallback child that boots the
+    # suite itself while the fork server is active).
+    ChildHandle = Struct.new(:pid, :log_paths, :timeout)
 
     # Runs RSpec child and suite processes on behalf of the integration.
     class RspecProcessRunner

data/lib/henitai/integration/scenario_log_support.rb

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 
 require "fileutils"
+require_relative "capped_log_sink"
 
 module Henitai
   module Integration
@@ -12,6 +13,9 @@ module Henitai
       MAX_LOG_BYTES_ENV = "HENITAI_MAX_LOG_BYTES"
       DEFAULT_MAX_LOG_BYTES = 5_000_000
       DRAIN_CHUNK_BYTES = 65_536
+      # Bound on the reads of the final drain, so a grandchild that keeps
+      # writing cannot hold the close up.
+      DRAIN_AVAILABLE_READS = 16
 
       def capture_child_output(log_paths)
         output_files = open_child_output(log_paths)
@@ -69,37 +73,46 @@ module Henitai
         file = File.new(path, "wb")
         file.sync = true
         reader, writer = IO.pipe
-        thread = Thread.new { drain_capped(reader, file, cap) }
-        { file:, reader:, writer:, thread: }
-      end
-
-      def drain_capped(reader, file, cap)
-        written = 0
-        truncated = false
+        sink = CappedLogSink.new(file, cap)
+        stop_reader, stop_writer = IO.pipe
+        thread = Thread.new { drain_capped(reader, stop_reader, sink) }
+        { file:, reader:, writer:, stop_reader:, stop_writer:, thread: }
+      end
+
+      # Drains until EOF or until +stop_reader+ becomes readable. Never killed:
+      # the stop signal ends it cooperatively between sink writes.
+      # Uses Kernel#select rather than IO.select: the drain runs beside the
+      # tests under mutation, and specs that stub or count IO.select must not
+      # see the harness's own waits.
+      def drain_capped(reader, stop_reader, sink)
         loop do
-          chunk = reader.readpartial(DRAIN_CHUNK_BYTES)
-          next unless written < cap
-
-          slice = chunk.byteslice(0, cap - written)
-          file.write(slice)
-          written += slice.bytesize
-          next unless written >= cap && !truncated
+          ready, = select([reader, stop_reader])
+          break drain_available(reader, sink) if ready.include?(stop_reader)
 
-          file.write("\n[henitai] output truncated at #{cap} bytes\n")
-          truncated = true
+          sink.write(reader.readpartial(DRAIN_CHUNK_BYTES))
         end
       rescue IOError
         # EOFError (a subclass) on writer close, or a closed pipe: draining done.
         nil
       end
 
+      # Closing the writer lets the drain thread reach EOF. A grandchild may
+      # still hold a copy of the writer, so EOF is not awaited: the stop pipe
+      # makes the thread take what is readable right now and exit.
       def finish_capped_stream(stream)
         return unless stream
 
         stream[:writer].close
+        stream[:stop_writer].close
         stream[:thread].join
-        stream[:reader].close
-        stream[:file].close
+        %i[reader stop_reader file].each { |key| stream[key].close }
+      end
+
+      def drain_available(reader, sink)
+        DRAIN_AVAILABLE_READS.times { sink.write(reader.read_nonblock(DRAIN_CHUNK_BYTES)) }
+      rescue IO::WaitReadable, IOError
+        # Nothing more buffered right now, or the pipe is closed.
+        nil
       end
 
       def redirect_child_output(output_files)

data/lib/henitai/integration.rb

@@ -6,12 +6,15 @@ require_relative "process_wakeup"
 require_relative "integration/child_bootstrap"
 require_relative "integration/rspec_process_runner"
 require_relative "integration/scenario_log_support"
+require_relative "integration/child_report_store"
 require_relative "integration/coverage_suppression"
 require_relative "integration/child_debug_log"
 require_relative "integration/loaded_features"
 require_relative "integration/base"
 require_relative "integration/mutant_run_support"
 require_relative "integration/rspec_child_runner"
+require_relative "rspec_outcomes"
+require_relative "integration/rspec_preload"
 require_relative "integration/rspec_test_selection"
 
 module Henitai
@@ -50,6 +53,7 @@ module Henitai
     class Rspec < Base
       include MutantRunSupport
       include RspecChildRunner
+      include RspecPreload
       include RspecTestSelection
 
       DEFAULT_SUITE_TIMEOUT = 300.0
@@ -98,7 +102,7 @@ module Henitai
         File.open(log_paths[:stdout_path], "w") do |stdout_file|
           File.open(log_paths[:stderr_path], "w") do |stderr_file|
             Process.spawn(
-              subprocess_env,
+              suite_process_env,
               *suite_command(test_files),
               out: stdout_file,
               err: stderr_file,
@@ -110,14 +114,23 @@ module Henitai
 
       private
 
-      def run_tests(test_files)
+      def outcome_recording_hook(outcomes, after_load)
+        lambda do
+          RspecOutcomes.register(outcomes, ::RSpec.__send__(:configuration).reporter)
+          after_load.nil? || after_load.call
+        end
+      end
+
+      # Yields once the spec files are loaded and before they run; a falsy
+      # block result skips the run (see MutantRunSupport#run_tests_in_child).
+      def run_tests(test_files, outcomes, &after_load)
         require "rspec/core"
         ::RSpec.__send__(:configuration).fail_if_no_examples = true
         log = child_debug_log
         log.rspec_trace(test_files:, rspec_options: [], rspec_argv: test_files)
         log.example_count("before_run")
         log.write("[henitai-debug-child] runner_run_start")
-        status = run_rspec_runner(test_files)
+        status = run_rspec_runner(test_files, &outcome_recording_hook(outcomes, after_load))
         log.write("[henitai-debug-child] runner_run_return status=#{status.inspect}")
         log.example_count("after_run")
         log.rspec_exit(status)

data/lib/henitai/minitest_coverage_reporter.rb

@@ -2,6 +2,8 @@
 
 require "minitest"
 require "henitai/per_test_coverage_collector"
+require "henitai/test_file_outcomes"
+require "henitai/minitest_result_file"
 
 module Henitai
   # Minitest reporter that collects per-test line coverage deltas.
@@ -11,26 +13,40 @@ module Henitai
   # to PerTestCoverageCollector so the JSON output format is identical to the
   # RSpec integration.
   class MinitestCoverageReporter < Minitest::Reporter
-    def initialize(io = $stdout, options = {})
+    def initialize(io = $stdout, options = {}, boot_clock: SuiteBootClock.from_env(ENV))
+      super(io, options)
+      @collector = PerTestCoverageCollector.new(boot_clock:)
+      @outcomes = TestFileOutcomes.new
+    end
+
+    def prerecord(klass, name)
       super
-      @collector = PerTestCoverageCollector.new
+      @collector.test_started
     end
 
     def record(result)
       super
-      @collector.record_test(
-        relative_to_pwd(result.source_location.first),
-        duration: result.time
-      )
+      record_coverage(result)
+      MinitestResultFile.record(@outcomes, result)
     end
 
     def report
       super
       @collector.write_report
+      @outcomes.write(TestFileOutcomes.default_report_path)
     end
 
     private
 
+    # A result without a resolvable file is skipped, so its coverage delta is
+    # not attributed to the next recorded test.
+    def record_coverage(result)
+      file = MinitestResultFile.for(result)
+      return @collector.skip_test unless file
+
+      @collector.record_test(relative_to_pwd(file), duration: result.time)
+    end
+
     def relative_to_pwd(path)
       prefix = "#{Dir.pwd}#{File::SEPARATOR}"
       path.start_with?(prefix) ? path.sub(prefix, "") : path

data/lib/henitai/minitest_result_file.rb

@@ -0,0 +1,62 @@
+# frozen_string_literal: true
+
+module Henitai
+  # Finds the test file a Minitest result belongs to.
+  #
+  # `Result#source_location` is where the test method is defined. For tests
+  # inherited from a base class or an included module that is a shared file,
+  # not the file of the test class that ran, so the file that defines the
+  # result's class is used for those. A method defined directly on the class
+  # keeps its own file: the class may have been reopened in another test file.
+  # The method location is also the fallback for classes that cannot be
+  # resolved (anonymous or removed).
+  module MinitestResultFile
+    module_function
+
+    # Records the result's outcome; a failure with no usable file is
+    # recorded as unattributed rather than dropped.
+    def record(outcomes, result)
+      failed = !result.passed? && !result.skipped?
+      file = self.for(result)
+      if file
+        outcomes.record(file, failed:)
+      elsif failed
+        outcomes.record_unattributed_failure
+      end
+    end
+
+    # @return [String, nil] the test file, or nil when none can be found
+    def for(result)
+      return usable(method_file(result)) || usable(class_file(result)) if defined_on_class?(result)
+
+      usable(class_file(result)) || usable(method_file(result))
+    end
+
+    def defined_on_class?(result)
+      klass = Object.const_get(result.klass.to_s)
+      klass.instance_method(result.name).owner == klass
+    rescue NameError
+      false
+    end
+
+    def class_file(result)
+      class_file_of(result.klass.to_s)
+    end
+
+    # The file that defines the named class; nil for anonymous or unknown names.
+    def class_file_of(name)
+      usable(Object.const_source_location(name)&.first)
+    rescue NameError
+      nil
+    end
+
+    def method_file(result)
+      result.source_location&.first
+    end
+
+    # Minitest reports no location for some results and "unknown" for others.
+    def usable(file)
+      file if file.is_a?(String) && !file.empty? && file != "unknown" && file != "(eval)"
+    end
+  end
+end

data/lib/henitai/mutant/activation.rb

@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+module Henitai
+  class Mutant
+    # The Ruby source that puts a mutant in place, and how to evaluate it.
+    #
+    # +mode+ is :toplevel for a real +def+ re-evaluated inside the original
+    # module and class openers (ADR-13), evaluated at the top level with
+    # +file+ and +line+ so +__FILE__+, +__LINE__+ and backtraces stay those of
+    # the original source; +def_offset+ is the index of the +def+ line within
+    # +source+. +mode+ is :class_eval for the +define_method+ fallback,
+    # evaluated inside the target module.
+    #
+    # For :toplevel, +lexical_scope+ holds the opener lines (e.g. "module A",
+    # "class B"), +owner+ the expression the +def+ is defined on ("self" or
+    # the receiver of +def Other.name+), and +singleton+ whether it is a
+    # +defs+. They let a replayed recipe reopen the scope without the AST.
+    Activation = Struct.new(:source, :mode, :file, :line, :def_offset, :lexical_scope, :owner, :singleton,
+                            keyword_init: true) do
+      def self.from_h(hash)
+        new(source: hash.fetch("source"), mode: hash.fetch("mode").to_sym, file: hash["file"],
+            line: hash["line"], def_offset: hash["defOffset"], lexical_scope: hash["lexicalScope"],
+            owner: hash["owner"], singleton: hash["singleton"])
+      end
+
+      def to_h
+        { "source" => source, "mode" => mode.to_s, "file" => file, "line" => line, "defOffset" => def_offset,
+          "lexicalScope" => lexical_scope, "owner" => owner, "singleton" => singleton }
+      end
+    end
+  end
+end