evilution 1.3.0 → 1.4.0

data/lib/evilution/integration/loading/concern_state_cleaner.rb

@@ -13,19 +13,23 @@ class Evilution::Integration::Loading::ConcernStateCleaner
 
   # ObjectSpace hands us every Module in the VM, including ones that answer no
   # message honestly. ActiveSupport::Deprecation::DeprecatedConstantProxy
-  # subclasses Module and undefines all of its instance methods, so a plain
-  # `mod.singleton_class` lands in its method_missing and emits the
-  # deprecation; where the deprecator's behaviour is :raise (grape's
-  # spec_helper sets that), the sweep blows up on a module it merely walked
-  # past. Calling through UnboundMethods bypasses the object's own method
-  # table, so a proxy is never given the chance to intercept -- and this holds
-  # for any method_missing-based wrapper, not just ActiveSupport's.
+  # subclasses Module and undefines all of its instance methods, so asking it
+  # anything lands in its method_missing and emits the deprecation; where the
+  # deprecator's behaviour is :raise (grape's spec_helper sets that), the
+  # sweep blows up on a module it merely walked past. So the module is never
+  # asked: `Concern === mod` is answered by Concern, and the ivars are reached
+  # through UnboundMethods, which bypass the object's own method table -- this
+  # holds for any method_missing-based wrapper, not just ActiveSupport's.
   # EV-v2rc / GH #1581.
-  SINGLETON_CLASS = Object.instance_method(:singleton_class)
+  #
+  # `===` also leaves the module as it found it. Asking for its
+  # singleton_class instead would create one for every module in the VM, each
+  # of them a module the next sweep has to visit and give a singleton class
+  # of its own; a long in-process run doubles its module count per mutation.
   IVAR_DEFINED = Object.instance_method(:instance_variable_defined?)
   IVAR_GET = Object.instance_method(:instance_variable_get)
   REMOVE_IVAR = Object.instance_method(:remove_instance_variable)
-  private_constant :SINGLETON_CLASS, :IVAR_DEFINED, :IVAR_GET, :REMOVE_IVAR
+  private_constant :IVAR_DEFINED, :IVAR_GET, :REMOVE_IVAR
 
   def initialize(subpath_resolver: Evilution::LoadPath::SubpathResolver.new)
     @subpath_resolver = subpath_resolver
@@ -34,11 +38,14 @@ class Evilution::Integration::Loading::ConcernStateCleaner
   def call(file_path)
     return unless defined?(ActiveSupport::Concern)
 
-    absolute = File.expand_path(file_path)
-    subpath = @subpath_resolver.call(file_path)
+    # Anchored on the project, not Dir.pwd: an isolated worker has chdir'd
+    # into a per-mutation sandbox, where a relative path names nothing and no
+    # concern would ever match.
+    absolute = File.expand_path(file_path, Evilution.project_base_dir)
+    subpath = @subpath_resolver.call(absolute)
 
     ObjectSpace.each_object(Module) do |mod|
-      next unless SINGLETON_CLASS.bind_call(mod).ancestors.include?(ActiveSupport::Concern)
+      next unless ActiveSupport::Concern === mod # rubocop:disable Style/CaseEquality
 
       clear_concern_ivars(mod, absolute, subpath)
     end

data/lib/evilution/integration/loading/mutation_applier.rb

@@ -6,6 +6,7 @@ require_relative "constant_pinner"
 require_relative "concern_state_cleaner"
 require_relative "source_evaluator"
 require_relative "redefinition_recovery"
+require_relative "../../diagnostic"
 
 # Composes the load-time pipeline that applies a mutation's new source to the
 # running VM: syntax-validate -> pin top-level constants (beats Zeitwerk) ->
@@ -45,6 +46,21 @@ class Evilution::Integration::Loading::MutationApplier
     failure_result(e, "#{e.class}: #{e.message}")
   end
 
+  # Evaluates the mutation's restore_source, if it has one. Runs after the
+  # tests: a failure here must not change the mutation's result, so it is
+  # reported and swallowed. Concern state is cleared first, as it is before
+  # applying: a concern ignores an `included` block handed to it twice from
+  # the same place, and would keep the mutated one.
+  def restore(mutation)
+    source = mutation.respond_to?(:restore_source) ? mutation.restore_source : nil
+    return unless source
+
+    @concern_state_cleaner.call(mutation.file_path)
+    @source_evaluator.call(source, mutation.file_path)
+  rescue ScriptError, StandardError => e
+    Evilution::Diagnostic.warn("[evilution] could not restore #{mutation.file_path}: #{e.class}: #{e.message}")
+  end
+
   private
 
   def resolve_eval_target(mutation)

data/lib/evilution/integration/loading/test_class_cache.rb

@@ -0,0 +1,46 @@
+# frozen_string_literal: true
+
+require_relative "../loading"
+
+# Remembers which test classes each test file registered when it was loaded,
+# so a process that runs many mutations loads a test file once.
+#
+# Minitest and Test::Unit learn of a test class when it is first defined.
+# Loading its file again reopens the class and registers nothing, so the
+# second mutation run in a process would find no tests. The classes of the
+# first load are handed back instead.
+#
+# The block given to .new returns the classes registered so far; what a load
+# adds to them is taken to be that file's.
+class Evilution::Integration::Loading::TestClassCache
+  def initialize(&registered)
+    @registered = registered
+    @classes = {}
+    @partial = {}
+  end
+
+  # The classes the given files registered. Yields each file that has not
+  # been loaded yet; the block loads it.
+  def fetch(files, &)
+    files.flat_map { |file| @classes[file] ||= load_once(file, &) }.uniq
+  end
+
+  def clear
+    @classes.clear
+    @partial.clear
+  end
+
+  private
+
+  # A load that raises is tried again next time. Classes it defined before
+  # raising are already registered and will not register again, so they are
+  # kept for that next attempt.
+  def load_once(file)
+    before = @registered.call
+    yield file
+    @partial.fetch(file, []) | (@registered.call - before)
+  rescue ScriptError, StandardError
+    @partial[file] = @partial.fetch(file, []) | (@registered.call - before)
+    raise
+  end
+end

data/lib/evilution/integration/minitest/test_ids.rb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+require_relative "../minitest"
+
+# Names a Minitest test the same way in the baseline and in a mutation run:
+# its class and its method, `CalcTest#test_adds`.
+#
+# A test class without a name has only its method to go by, so two such
+# classes with a method of the same name are told apart by neither run.
+module Evilution::Integration::Minitest::TestIds
+  def self.of(result)
+    class_name = result.respond_to?(:class_name) ? result.class_name : result.class.name
+    "#{class_name}##{result.name}"
+  end
+
+  # The ids of the tests that failed, from the summary reporter of the run.
+  # It keeps every result that did not pass, skips included.
+  def self.failed(summary)
+    summary.results.reject(&:skipped?).map { |result| of(result) }
+  end
+end

data/lib/evilution/integration/minitest.rb

@@ -4,9 +4,12 @@ require "stringio"
 require_relative "base"
 require_relative "../diagnostic"
 require_relative "minitest_crash_detector"
+require_relative "known_failures"
 require_relative "loading/test_load_path"
+require_relative "loading/test_class_cache"
 require_relative "../spec_resolver"
 require_relative "../spec_selector"
+require_relative "../baseline"
 
 require_relative "../integration"
 
@@ -47,12 +50,22 @@ class Evilution::Integration::Minitest < Evilution::Integration::Base
     options = ::Minitest.process_args(["--seed", "0"])
     options[:io] = out
     reporter = ::Minitest::CompositeReporter.new
-    reporter << ::Minitest::SummaryReporter.new(out, options)
+    summary = ::Minitest::SummaryReporter.new(out, options)
+    reporter << summary
     initialize_minitest_state(reporter, options)
     reporter.start
     dispatch_minitest_suites(reporter, options)
     reporter.report
-    reporter.passed?
+    Evilution::Baseline::Report.build(
+      passed: reporter.passed?, failures: baseline_failures(summary), failed_ids: TestIds.failed(summary)
+    )
+  end
+
+  # The summary reporter keeps every result that did not pass, skips included.
+  def self.baseline_failures(summary)
+    summary.results.reject(&:skipped?).map do |result|
+      { id: TestIds.of(result), description: "", message: result.failure.message }
+    end
   end
 
   # Mirror Minitest.run's preamble: seed setup + plugin init. Without seeding
@@ -84,6 +97,12 @@ class Evilution::Integration::Minitest < Evilution::Integration::Base
     end
   end
 
+  # Shared by every instance: the canary and the mutation runs build their
+  # own, and a file loaded for one must not be loaded again for the other.
+  def self.test_classes
+    @test_classes ||= Evilution::Integration::Loading::TestClassCache.new { ::Minitest::Runnable.runnables.dup }
+  end
+
   def self.baseline_options
     {
       runner: baseline_runner,
@@ -92,8 +111,10 @@ class Evilution::Integration::Minitest < Evilution::Integration::Base
     }
   end
 
-  def initialize(test_files: nil, hooks: nil, fallback_to_full_suite: false, spec_selector: nil)
+  # known_failures: the ids of the tests the baseline saw failing (see TestIds).
+  def initialize(test_files: nil, hooks: nil, fallback_to_full_suite: false, spec_selector: nil, known_failures: [])
     @test_files = test_files
+    @known_failures = Evilution::Integration::KnownFailures.new(known_failures)
     @minitest_loaded = false
     @spec_selector = spec_selector || Evilution::SpecSelector.new(
       spec_resolver: Evilution::SpecResolver.new(test_dir: "test", test_suffix: "_test.rb", request_dir: "integration")
@@ -132,17 +153,27 @@ class Evilution::Integration::Minitest < Evilution::Integration::Base
   end
 
   def execute_minitest(mutation, files, command)
-    Evilution::Integration::Loading::TestLoadPath.add!(files)
-    files.each { |f| load(File.expand_path(f, Evilution.project_base_dir)) }
+    load_test_classes(files)
 
     detector = reset_crash_detector
     run = run_minitest(build_args(mutation), detector)
 
     return no_tests_ran_result(command) if run[:count].zero?
+    # Before the crash check: a test that was already failing may fail by
+    # raising now, and a crash in it is no more a kill than its failure was.
+    return known_failures_only_result(command) if @known_failures.only?(run[:failed_ids])
 
     build_minitest_result(run[:passed], command, detector)
   end
 
+  # Leaves Minitest with the test classes of these files and no others.
+  def load_test_classes(files)
+    Evilution::Integration::Loading::TestLoadPath.add!(files)
+    paths = files.map { |f| File.expand_path(f, Evilution.project_base_dir) }
+    classes = self.class.test_classes.fetch(paths) { |path| load(path) }
+    ::Minitest::Runnable.runnables.replace(classes)
+  end
+
   # Zero dispatched test methods means the run carries no signal — the result
   # is neither survived nor killed. Most common cause: the project's tests use
   # a framework other than Minitest (e.g. the test-unit gem, whose
@@ -160,6 +191,12 @@ class Evilution::Integration::Minitest < Evilution::Integration::Base
     }
   end
 
+  # Every test that failed was already failing in the baseline. Still a
+  # failed run, but flagged, so it is not taken for a kill.
+  def known_failures_only_result(command)
+    { passed: false, known_failures_only: true, test_command: command }
+  end
+
   def unresolved_result(mutation)
     {
       passed: false,
@@ -190,7 +227,7 @@ class Evilution::Integration::Minitest < Evilution::Integration::Base
     dispatch_minitest_suites(reporter, options)
     reporter.report
 
-    { passed: summary.passed?, count: minitest_method_count }
+    { passed: summary.passed?, count: minitest_method_count, failed_ids: TestIds.failed(summary) }
   end
 
   # Attach evilution's own reporters to the composite AFTER plugin init, and
@@ -293,3 +330,5 @@ class Evilution::Integration::Minitest < Evilution::Integration::Base
     end
   end
 end
+
+require_relative "minitest/test_ids"

data/lib/evilution/integration/rspec/baseline_runner.rb

@@ -1,6 +1,9 @@
 # frozen_string_literal: true
 
+require "stringio"
 require_relative "../rspec"
+require_relative "../../baseline"
+require_relative "example_ids"
 
 class Evilution::Integration::RSpec::BaselineRunner
   def call(spec_file)
@@ -9,10 +12,51 @@ class Evilution::Integration::RSpec::BaselineRunner
     # FrameworkLoader#add_spec_load_path for rationale.
     spec_dir = File.expand_path("spec", Evilution.project_base_dir)
     $LOAD_PATH.unshift(spec_dir) unless $LOAD_PATH.include?(spec_dir)
-    ::RSpec.reset
+    reset_examples
+    output = StringIO.new
     status = ::RSpec::Core::Runner.run(
-      ["--format", "progress", "--no-color", "--order", "defined", spec_file]
+      ["--format", "progress", "--no-color", "--order", "defined", spec_file], output, output
     )
-    status.zero?
+    report(status.zero?, output.string)
+  end
+
+  private
+
+  # Examples are cleared, the configuration is kept: a helper loaded by
+  # --preload is not loaded again by the spec's own require, so an
+  # RSpec.configure block dropped here never comes back, and a spec relying on
+  # it goes red in the baseline alone.
+  def reset_examples
+    ::RSpec.respond_to?(:clear_examples) ? ::RSpec.clear_examples : ::RSpec.reset
+  end
+
+  # What RSpec printed is the only account of a failure outside any example --
+  # a file that does not load, a suite hook that raises.
+  def report(passed, output)
+    return Evilution::Baseline::Report.build(passed: true) if passed
+
+    failures = failed_examples
+    Evilution::Baseline::Report.build(
+      passed: false, failures: failures, error: failures.empty? ? output : nil,
+      failed_ids: Evilution::Integration::RSpec::ExampleIds.failed(::RSpec.world)
+    )
+  end
+
+  def failed_examples
+    world = ::RSpec.world
+    return [] unless world.respond_to?(:all_examples)
+
+    world.all_examples
+         .select { |example| example.execution_result.status == :failed }
+         .map { |example| failure(example) }
+  end
+
+  def failure(example)
+    exception = example.execution_result.exception
+    {
+      id: example.id,
+      description: example.full_description,
+      message: exception ? "#{exception.class}: #{exception.message}" : nil
+    }
   end
 end

data/lib/evilution/integration/rspec/example_ids.rb

@@ -0,0 +1,35 @@
+# frozen_string_literal: true
+
+require_relative "../rspec"
+
+# Names an example the same way wherever it runs.
+#
+# RSpec's own id starts with the spec file as it was given: `./spec/a_spec.rb`
+# from the project root, an absolute path from an isolated worker that has
+# changed directory. The baseline and a mutation run can therefore disagree
+# about the same example, so the path is made absolute and symlinks are
+# resolved before the position within the file is added.
+module Evilution::Integration::RSpec::ExampleIds
+  FAILED = :failed
+
+  def self.of(example)
+    metadata = example.metadata
+    "#{canonical(metadata[:rerun_file_path])}[#{metadata[:scoped_id]}]"
+  end
+
+  # The ids of the examples that failed in the run RSpec.world still holds.
+  def self.failed(world)
+    return [] unless world.respond_to?(:all_examples)
+
+    world.all_examples
+         .select { |example| example.execution_result.status == FAILED }
+         .map { |example| of(example) }
+  end
+
+  def self.canonical(path)
+    File.realpath(path)
+  rescue SystemCallError
+    File.expand_path(path)
+  end
+  private_class_method :canonical
+end

data/lib/evilution/integration/rspec/result_builder.rb

@@ -21,8 +21,13 @@ class Evilution::Integration::RSpec::ResultBuilder
     }
   end
 
-  def from_run(status, command, detector, examples_loaded: nil)
+  # known_failures_only: every example that failed was already failing in the
+  # baseline. Still a failed run, but flagged, so it is not taken for a kill.
+  def from_run(status, command, detector, examples_loaded: nil, known_failures_only: false)
     return { passed: true, test_command: command } if status.zero?
+    # Before the crash check: an example that was already failing may fail by
+    # raising now, and a crash in it is no more a kill than its failure was.
+    return { passed: false, known_failures_only: true, test_command: command } if known_failures_only
 
     if detector.only_crashes?
       classes = detector.unique_crash_classes

data/lib/evilution/integration/rspec/state_guard/anonymous_example_group_examples.rb

@@ -0,0 +1,36 @@
+# frozen_string_literal: true
+
+require_relative "../state_guard"
+require_relative "../../rspec"
+
+# Drops the examples a run registers on RSpec::Core::AnonymousExampleGroup.
+#
+# Example#initialize appends every example to its group's `examples`. Most
+# groups are thrown away with the run, but AnonymousExampleGroup is a constant:
+# whatever lands there stays for the life of the process. A run with a suite
+# hook puts a SuiteHookContext there, and the reporter notifying a non-example
+# exception puts an Example there. Each holds that run's reporter, and the
+# reporter holds every example the run loaded, so one in-process run after
+# another retained them all, about 1 MB per mutation on the memory_check
+# fixture.
+#
+# Forked runs never leak this (the child dies), only the in-process path.
+class Evilution::Integration::RSpec::StateGuard::AnonymousExampleGroupExamples
+  def snapshot
+    examples&.length
+  end
+
+  def release(length)
+    return unless length
+
+    examples&.slice!(length..)
+  end
+
+  private
+
+  def examples
+    return nil unless defined?(::RSpec::Core::AnonymousExampleGroup)
+
+    ::RSpec::Core::AnonymousExampleGroup.examples
+  end
+end

data/lib/evilution/integration/rspec/state_guard.rb

@@ -12,8 +12,10 @@ require_relative "state_guard/world_example_groups"
 require_relative "state_guard/world_sources_by_path"
 require_relative "state_guard/world_filtered_examples"
 require_relative "state_guard/reporter_arrays"
+require_relative "state_guard/anonymous_example_group_examples"
 require_relative "state_guard/example_groups_constants"
 require_relative "state_guard/configuration_state"
+require_relative "../../diagnostic"
 
 class Evilution::Integration::RSpec::StateGuard
   DEFAULT_STRATEGIES = [
@@ -22,6 +24,7 @@ class Evilution::Integration::RSpec::StateGuard
     WorldSourcesByPath.new,
     WorldFilteredExamples.new,
     ReporterArrays.new,
+    AnonymousExampleGroupExamples.new,
     ExampleGroupsConstants.new,
     ConfigurationState.new
   ].freeze
@@ -43,6 +46,6 @@ class Evilution::Integration::RSpec::StateGuard
   def release_one(strategy, captured)
     strategy.release(captured)
   rescue StandardError => e
-    warn "[evilution] state release failed for #{strategy.class.name}: #{e.class}: #{e.message}"
+    Evilution::Diagnostic.warn("[evilution] state release failed for #{strategy.class.name}: #{e.class}: #{e.message}")
   end
 end

data/lib/evilution/integration/rspec.rb

@@ -2,6 +2,7 @@
 
 require "stringio"
 require_relative "base"
+require_relative "known_failures"
 require_relative "../spec_resolver"
 require_relative "../spec_selector"
 require_relative "../related_spec_heuristic"
@@ -24,6 +25,7 @@ class Evilution::Integration::RSpec < Evilution::Integration::Base
     fallback_to_full_suite: false,
     spec_selector: nil,
     example_filter: nil,
+    known_failures: [],
     framework_loader: FrameworkLoader.new,
     test_file_resolver: nil,
     example_filter_applier: nil,
@@ -44,6 +46,7 @@ class Evilution::Integration::RSpec < Evilution::Integration::Base
     @crash_detector_lifecycle = crash_detector_lifecycle
     @result_builder = result_builder
     @state_guard = state_guard
+    @known_failures = Evilution::Integration::KnownFailures.new(known_failures)
     super(hooks: hooks)
   end
 
@@ -71,7 +74,7 @@ class Evilution::Integration::RSpec < Evilution::Integration::Base
     return @result_builder.unresolved_example(mutation) if targets.nil?
 
     result = run_targets(targets)
-    return result unless confirm_survivor?(result, targets, files)
+    return result unless confirm?(result, targets, files)
 
     run_targets(files)
   end
@@ -85,8 +88,12 @@ class Evilution::Integration::RSpec < Evilution::Integration::Base
   # So a survivor is re-run against the whole resolved file, and that run is the
   # one reported. Only survivors pay for it, and only where the subset was
   # actually narrower than the file.
-  def confirm_survivor?(result, targets, files)
-    result[:passed] && targets != files
+  #
+  # The same goes for a subset in which nothing failed but examples that were
+  # failing before any mutation ran: that is no verdict either, and the rest of
+  # the file may hold one.
+  def confirm?(result, targets, files)
+    (result[:passed] || result[:known_failures_only]) && targets != files
   end
 
   def run_targets(targets)
@@ -142,7 +149,8 @@ class Evilution::Integration::RSpec < Evilution::Integration::Base
       claim_rspec_streams(out, err)
       detector = @crash_detector_lifecycle.register
       status = ::RSpec::Core::Runner.run(args, err, out)
-      @result_builder.from_run(status, command, detector, examples_loaded:)
+      flagged = known_failures_only?(status)
+      @result_builder.from_run(status, command, detector, examples_loaded:, known_failures_only: flagged)
     rescue StandardError => e
       { passed: false, error: e.message, test_command: command }
     ensure
@@ -188,12 +196,22 @@ class Evilution::Integration::RSpec < Evilution::Integration::Base
     nil
   end
 
+  # Whether the run failed on nothing but examples the baseline already saw
+  # failing. Decided here, in the process that ran them: the names of every
+  # failing example need not travel back, only the answer.
+  def known_failures_only?(status)
+    return false if status.zero? || @known_failures.empty?
+
+    @known_failures.only?(ExampleIds.failed(::RSpec.world))
+  end
+
   def reset_examples
     ::RSpec.respond_to?(:clear_examples) ? ::RSpec.clear_examples : ::RSpec.reset
   end
 end
 
 require_relative "rspec/framework_loader"
+require_relative "rspec/example_ids"
 require_relative "rspec/test_file_resolver"
 require_relative "rspec/unresolved_spec_warner"
 require_relative "rspec/example_filter_applier"

data/lib/evilution/integration/test_unit/result_builder.rb

@@ -21,6 +21,12 @@ module Evilution::Integration::TestUnit::ResultBuilder
     end
   end
 
+  # Every test that failed was already failing in the baseline. Still a
+  # failed run, but flagged, so it is not taken for a kill.
+  def known_failures_only(command)
+    { passed: false, known_failures_only: true, test_command: command }
+  end
+
   def no_tests_ran(command)
     {
       passed: false,

data/lib/evilution/integration/test_unit/test_ids.rb

@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+require_relative "../test_unit"
+
+# Names a Test::Unit test the same way in the baseline and in a mutation run:
+# the name the framework gives it, `test_adds(CalcTest)`.
+module Evilution::Integration::TestUnit::TestIds
+  # Only failures and errors fail a run; pendings, omissions and notifications
+  # are faults too, and are left out.
+  def self.faults(result)
+    result.failures + result.errors
+  end
+
+  # The ids of the tests that failed or raised in a finished run.
+  def self.failed(result)
+    faults(result).map(&:test_name)
+  end
+end

data/lib/evilution/integration/test_unit.rb

@@ -2,9 +2,12 @@
 
 require_relative "base"
 require_relative "test_unit_crash_detector"
+require_relative "known_failures"
 require_relative "loading/test_load_path"
+require_relative "loading/test_class_cache"
 require_relative "../spec_resolver"
 require_relative "../spec_selector"
+require_relative "../baseline"
 
 require_relative "../integration"
 
@@ -28,6 +31,12 @@ class Evilution::Integration::TestUnit < Evilution::Integration::Base
     Evilution::SpecResolver.new(test_dir: "test", test_suffix: "_test.rb", request_dir: "integration")
   end
 
+  # Shared by every instance: the canary and the mutation runs build their
+  # own, and a file loaded for one must not be loaded again for the other.
+  def self.test_classes
+    @test_classes ||= Evilution::Integration::Loading::TestClassCache.new { SubjectClassRegistry.descendants }
+  end
+
   def self.baseline_options
     {
       runner: baseline_runner,
@@ -40,25 +49,41 @@ class Evilution::Integration::TestUnit < Evilution::Integration::Base
     require_relative "test_unit/framework_loader"
     require_relative "test_unit/subject_class_registry"
     require_relative "test_unit/dispatcher"
+    require_relative "test_unit/test_ids"
     FrameworkLoader.new.call
     files = baseline_test_files(test_file)
     Evilution::Integration::Loading::TestLoadPath.add!(files)
     new_classes = SubjectClassRegistry.newly_loaded do
       files.each { |f| load(File.expand_path(f)) }
     end
-    Dispatcher.call(new_classes, name: "evilution baseline").passed?
+    baseline_report(Dispatcher.call(new_classes, name: "evilution baseline"))
+  end
+
+  def self.baseline_report(result)
+    Evilution::Baseline::Report.build(
+      passed: result.passed?, failures: baseline_failures(result), failed_ids: TestIds.failed(result)
+    )
+  end
+
+  def self.baseline_failures(result)
+    TestIds.faults(result).map do |fault|
+      { id: fault.test_name, description: "", message: fault.message }
+    end
   end
 
   def self.baseline_test_files(test_file)
     File.directory?(test_file) ? Dir.glob(File.join(test_file, "**/*_test.rb")) : [test_file]
   end
 
-  def initialize(test_files: nil, hooks: nil, fallback_to_full_suite: false, spec_selector: nil)
+  # known_failures: the ids of the tests the baseline saw failing (see TestIds).
+  def initialize(test_files: nil, hooks: nil, fallback_to_full_suite: false, spec_selector: nil, known_failures: [])
     require_relative "test_unit/framework_loader"
     require_relative "test_unit/subject_class_registry"
     require_relative "test_unit/dispatcher"
     require_relative "test_unit/test_file_resolver"
     require_relative "test_unit/result_builder"
+    require_relative "test_unit/test_ids"
+    @known_failures = Evilution::Integration::KnownFailures.new(known_failures)
     @framework_loader = FrameworkLoader.new
     @file_resolver = TestFileResolver.new(
       test_files: test_files,
@@ -97,22 +122,25 @@ class Evilution::Integration::TestUnit < Evilution::Integration::Base
 
     detector = reset_crash_detector
     result = Dispatcher.call(new_classes, name: "evilution-mutation")
+    # Before the crash check: a test that was already failing may fail by
+    # raising now, and a crash in it is no more a kill than its failure was.
+    return ResultBuilder.known_failures_only(command) if @known_failures.only?(TestIds.failed(result))
+
     result.faults.each { |fault| detector.record(fault) }
     ResultBuilder.call(passed: result.passed?, command: command, detector: detector)
   end
 
   def load_test_classes(files)
     Evilution::Integration::Loading::TestLoadPath.add!(files)
-    SubjectClassRegistry.newly_loaded do
-      files.each { |f| load(File.expand_path(f, Evilution.project_base_dir)) }
-    end
+    paths = files.map { |f| File.expand_path(f, Evilution.project_base_dir) }
+    self.class.test_classes.fetch(paths) { |path| load(path) }
   end
 
   # Test::Unit has no public registry-clear analogous to
-  # Minitest::Runnable.runnables.clear. SubjectClassRegistry's newly_loaded
-  # block scopes each dispatch to classes loaded in *this* round, so stale
-  # classes from prior mutations sit dormant on ObjectSpace without polluting
-  # the run. #reset_state stays as a contract no-op for parity with Minitest.
+  # Minitest::Runnable.runnables.clear. Each dispatch is handed the classes of
+  # the files resolved for this mutation, so classes of other files sit
+  # dormant on ObjectSpace without polluting the run. #reset_state stays as a
+  # contract no-op for parity with Minitest.
   def reset_state
     # no-op — see comment above
   end