evilution 1.2.0 → 1.4.0

data/lib/evilution/result/coverage_gap_grouper.rb

@@ -1,5 +1,6 @@
 # frozen_string_literal: true
 
+require_relative "../result"
 require_relative "coverage_gap"
 
 class Evilution::Result::CoverageGapGrouper

data/lib/evilution/result/mutation_result.rb

@@ -9,8 +9,11 @@ class Evilution::Result::MutationResult
 
   attr_reader :mutation, :status, :duration, :killing_test, :test_command, :memory, :error, :neutral_reason
 
+  # known_failures_only: the tests failed, but every example that did was
+  # already failing before any mutation ran, so the failure says nothing
+  # about this one.
   def initialize(mutation:, status:, duration: 0.0, killing_test: nil,
-                 test_command: nil, memory: nil, error: nil, neutral_reason: nil)
+                 test_command: nil, memory: nil, error: nil, neutral_reason: nil, known_failures_only: false)
     raise ArgumentError, "invalid status: #{status}" unless STATUSES.include?(status)
 
     @mutation = mutation
@@ -21,6 +24,7 @@ class Evilution::Result::MutationResult
     @memory = memory
     @error = error
     @neutral_reason = neutral_reason
+    @known_failures_only = known_failures_only == true
     freeze
   end
 
@@ -54,6 +58,10 @@ class Evilution::Result::MutationResult
     @error.is_a?(Evilution::Result::ErrorInfo) ? @error.backtrace : nil
   end
 
+  def known_failures_only?
+    @known_failures_only
+  end
+
   def killed?
     status == :killed
   end

data/lib/evilution/result/subject_scorer.rb

@@ -1,5 +1,6 @@
 # frozen_string_literal: true
 
+require_relative "../result"
 require_relative "subject_score"
 
 # Groups mutation results by the subject they belong to and scores each one.

data/lib/evilution/result/summary.rb

@@ -3,13 +3,15 @@
 require_relative "../result"
 require_relative "coverage_gap_grouper"
 require_relative "subject_scorer"
+require_relative "baseline_neutralization"
 
 class Evilution::Result::Summary
   attr_reader :results, :duration, :skipped, :disabled_mutations, :unresolved_target_files,
-              :target_file_count, :infra_retried
+              :target_file_count, :infra_retried, :uncovered_code, :baseline_failures
 
   def initialize(results:, duration: 0.0, truncated: false, skipped: 0, disabled_mutations: [],
-                 unresolved_target_files: [], target_file_count: nil, infra_retried: 0)
+                 unresolved_target_files: [], target_file_count: nil, infra_retried: 0, uncovered_code: [],
+                 baseline_failures: [])
     @results = results
     @duration = duration
     @truncated = truncated
@@ -18,9 +20,27 @@ class Evilution::Result::Summary
     @unresolved_target_files = unresolved_target_files.freeze
     @target_file_count = target_file_count
     @infra_retried = infra_retried
+    @uncovered_code = uncovered_code.freeze
+    @baseline_failures = baseline_failures.freeze
     freeze
   end
 
+  # The kills not counted because only examples already failing in the
+  # baseline failed, gathered under the spec file that was red and paired with
+  # why it was. They are out of the score, so a red spec file would read as
+  # full marks unless the report says they are there.
+  def baseline_neutralizations
+    baseline_neutralized_results.group_by { |result| result.neutral_reason.detail }.map do |spec_file, results|
+      Evilution::Result::BaselineNeutralization.new(
+        spec_file: spec_file, count: results.length, failures: baseline_failures_for(spec_file)
+      )
+    end
+  end
+
+  def baseline_neutralized
+    baseline_neutralized_results.length
+  end
+
   # Neutral results gathered by the reason they were recorded, which is what the
   # report needs to say which kind of neutral a reader is looking at
   # (EV-5pob / GH #1606).
@@ -53,6 +73,12 @@ class Evilution::Result::Summary
     !unresolved_target_files.empty?
   end
 
+  # Targeted lines that hold code outside every subject, so no mutation was
+  # ever generated for them: `{ file:, lines: ["2-4", "9"] }` per file.
+  def uncovered_code?
+    !uncovered_code.empty?
+  end
+
   def truncated?
     @truncated
   end
@@ -167,4 +193,18 @@ class Evilution::Result::Summary
 
     max_rss && (max_rss / 1024.0)
   end
+
+  private
+
+  def baseline_neutralized_results
+    neutral_results.select { |result| result.neutral_reason && result.neutral_reason.kind == :baseline_failure }
+  end
+
+  # No spec file named means the run was given its spec files explicitly, and
+  # any of them being red neutralizes the survivor.
+  def baseline_failures_for(spec_file)
+    return baseline_failures if spec_file.nil?
+
+    baseline_failures.select { |failure| failure.spec_file == spec_file }
+  end
 end

data/lib/evilution/runner/baseline_runner.rb

@@ -12,6 +12,7 @@ require_relative "../integration/test_unit"
 require_relative "../example_filter"
 require_relative "../spec_ast_cache"
 require_relative "../source_ast_cache"
+require_relative "../diagnostic"
 
 unless defined?(Evilution::Runner::INTEGRATIONS)
   Evilution::Runner::INTEGRATIONS = {
@@ -33,15 +34,19 @@ class Evilution::Runner::BaselineRunner
     end
   end
 
-  def build_integration
+  # baseline_result: what the baseline found, when it ran. The tests it saw
+  # failing are handed to the integration, which can then tell a mutation run
+  # that fails on nothing else from one that kills.
+  def build_integration(baseline_result = nil)
     klass = integration_class
-    kwargs = base_integration_kwargs
+    kwargs = base_integration_kwargs(baseline_result)
     kwargs.merge!(rspec_integration_kwargs) if klass == Evilution::Integration::RSpec
     klass.new(**kwargs)
   end
 
-  def base_integration_kwargs
+  def base_integration_kwargs(baseline_result = nil)
     {
+      known_failures: baseline_result ? baseline_result.failed_example_ids : [],
       test_files: config.spec_files.empty? ? nil : config.spec_files,
       hooks: hooks,
       fallback_to_full_suite: config.fallback_to_full_suite?,
@@ -60,18 +65,15 @@ class Evilution::Runner::BaselineRunner
     return nil unless config.baseline? && subjects.any?
 
     log_start
-    baseline = Evilution::Baseline.new(
-      timeout: config.timeout,
-      test_files: config.spec_files.empty? ? nil : config.spec_files,
-      **integration_class.baseline_options
-    )
-    result = baseline.call(subjects)
+    result = build_baseline.call(subjects)
     log_complete(result)
     result
   end
 
+  # The same selector the baseline resolves with, so a surviving mutation is
+  # matched against exactly the spec files the baseline ran.
   def neutralization_resolver
-    integration_class.baseline_options[:spec_resolver] || Evilution::SpecResolver.new
+    config.spec_selector
   end
 
   def neutralization_fallback_dir
@@ -82,6 +84,19 @@ class Evilution::Runner::BaselineRunner
 
   attr_reader :config, :hooks
 
+  # Resolves through config.spec_selector -- the object `run` uses, so
+  # spec_mappings / spec_pattern apply -- and skips unresolved sources unless
+  # the run falls back to the full suite.
+  def build_baseline
+    Evilution::Baseline.new(
+      timeout: config.timeout,
+      test_files: config.spec_files.empty? ? nil : config.spec_files,
+      spec_selector: config.spec_selector,
+      fallback_to_full_suite: config.fallback_to_full_suite?,
+      **integration_class.baseline_options
+    )
+  end
+
   def build_example_filter
     return nil unless config.example_targeting?
 
@@ -109,7 +124,7 @@ class Evilution::Runner::BaselineRunner
 
     Evilution::CoverageExampleFilter.new(map: map, lexical: lexical)
   rescue StandardError => e
-    warn "evilution: coverage targeting unavailable (#{e.class}: #{e.message}); using lexical targeting"
+    Evilution::Diagnostic.warn("evilution: coverage targeting unavailable (#{e.class}: #{e.message}); using lexical targeting")
     lexical
   end
 

data/lib/evilution/runner/canary.rb

@@ -6,6 +6,7 @@ require "securerandom"
 require_relative "../runner"
 require_relative "../mutation"
 require_relative "../subject"
+require_relative "canary_failure_message"
 
 # Runs one guaranteed-unobservable synthetic mutation through the configured
 # integration + isolation at session start. The synthetic spec never references
@@ -34,7 +35,7 @@ class Evilution::Runner::Canary
       test_command: ->(mutation) { build_integration(spec_path).call(mutation) },
       timeout: @config.timeout
     )
-    raise Failed, failure_message(result) unless result.status == :survived
+    raise Failed, Evilution::Runner::CanaryFailureMessage.new(result).to_s unless result.status == :survived
 
     nil
   ensure
@@ -145,60 +146,4 @@ class Evilution::Runner::Canary
   def build_integration(spec_path)
     @integration_class.new(test_files: [spec_path], hooks: @hooks)
   end
-
-  # When the child reported an error, that error IS the diagnosis -- naming it
-  # beats guessing. The speculative list stays only for the cases that carry no
-  # error at all (:killed, :timeout), where guesses are the only help there is.
-  # Diagnosing GH #1581 meant rebuilding this canary by hand to read the field
-  # this message used to drop, and none of the four guesses was the cause.
-  # EV-65nf / GH #1586.
-  def failure_message(result)
-    "#{failure_preamble(result.status)} #{diagnosis(result)} " \
-      "Re-run with --no-canary to bypass this check."
-  end
-
-  def failure_preamble(status)
-    "evilution proof-of-life canary failed: a guaranteed-unobservable synthetic " \
-      "mutation was scored #{status.inspect} instead of :survived. The mutation " \
-      "pipeline is misreporting — every score this run would produce is unreliable."
-  end
-
-  def diagnosis(result)
-    reported = reported_error(result)
-    return speculative_causes if reported.nil?
-
-    "The child reported: #{reported}"
-  end
-
-  def reported_error(result)
-    message = result.error_message
-    return nil if message.nil? || message.empty?
-
-    described = with_class_prefix(message, result.error_class)
-    frame = first_frame(result)
-    frame.nil? ? described : "#{described} (at #{frame})"
-  end
-
-  # MutationApplier already prefixes the class onto the message it packs
-  # ("#{e.class}: #{e.message}"), while other paths pack the bare message and
-  # leave the class in its own field. Prefixing unconditionally would print
-  # "NameError: NameError: ..." for the former.
-  def with_class_prefix(message, klass)
-    return message if klass.nil? || klass.empty? || message.start_with?("#{klass}:")
-
-    "#{klass}: #{message}"
-  end
-
-  def first_frame(result)
-    backtrace = result.error_backtrace
-    return nil if backtrace.nil? || backtrace.empty?
-
-    backtrace.first
-  end
-
-  def speculative_causes
-    "Likely causes: Rails/Zeitwerk autoloading breaking child eval; an env-specific " \
-      "RSpec config (e.g. fail_if_no_examples); a classify_status fallback defect; or " \
-      "an isolation-mode defect."
-  end
 end

data/lib/evilution/runner/canary_failure_message.rb

@@ -0,0 +1,101 @@
+# frozen_string_literal: true
+
+require_relative "../runner"
+
+# Explains why the proof-of-life canary failed, from the result its synthetic
+# mutation came back with. Kept apart from Canary, which only runs the probe:
+# the wording changes for different reasons than the mechanics do.
+class Evilution::Runner::CanaryFailureMessage
+  # Raised by RubyGems when it cannot load a gem at a version that satisfies
+  # every requirement already in play.
+  GEM_ACTIVATION_ERRORS = %w[
+    Gem::ConflictError Gem::LoadError Gem::MissingSpecError Gem::MissingSpecVersionError
+  ].freeze
+
+  def initialize(result)
+    @result = result
+  end
+
+  # When the child reported an error, that error IS the diagnosis -- naming it
+  # beats guessing. The speculative list stays only for the cases that carry no
+  # error at all (:killed, :timeout), where guesses are the only help there is.
+  # Diagnosing GH #1581 meant rebuilding this canary by hand to read the field
+  # this message used to drop, and none of the four guesses was the cause.
+  # EV-65nf / GH #1586.
+  def to_s
+    return gem_activation_message(@result) if gem_activation_failure?(@result)
+
+    "#{failure_preamble(@result.status)} #{diagnosis(@result)} " \
+      "Re-run with --no-canary to bypass this check."
+  end
+
+  private
+
+  # The usual cause is evilution running outside `bundle exec`: RubyGems then
+  # activates the newest installed copy of a shared dependency, which the test
+  # framework may not accept. Every mutation would fail the same way, so
+  # skipping the canary would not help.
+  def gem_activation_message(result)
+    "evilution proof-of-life canary failed: the test framework could not be loaded " \
+      "because of a gem activation conflict. This is an environment problem, not a " \
+      "mutation-pipeline defect, and every mutation would fail the same way. Run " \
+      "evilution under `bundle exec` so the project's Gemfile.lock decides gem versions " \
+      "(for the MCP server: command `bundle`, args [\"exec\", \"evilution\", \"mcp\"]). " \
+      "The child reported: #{reported_error(result)}"
+  end
+
+  # The class may arrive in its own field or only as the message prefix,
+  # depending on which path packed the error.
+  def gem_activation_failure?(result)
+    message = result.error_message.to_s
+
+    GEM_ACTIVATION_ERRORS.any? do |klass|
+      result.error_class == klass || message.start_with?("#{klass}:")
+    end
+  end
+
+  def failure_preamble(status)
+    "evilution proof-of-life canary failed: a guaranteed-unobservable synthetic " \
+      "mutation was scored #{status.inspect} instead of :survived. The mutation " \
+      "pipeline is misreporting — every score this run would produce is unreliable."
+  end
+
+  def diagnosis(result)
+    reported = reported_error(result)
+    return speculative_causes if reported.nil?
+
+    "The child reported: #{reported}"
+  end
+
+  def reported_error(result)
+    message = result.error_message
+    return nil if message.nil? || message.empty?
+
+    described = with_class_prefix(message, result.error_class)
+    frame = first_frame(result)
+    frame.nil? ? described : "#{described} (at #{frame})"
+  end
+
+  # MutationApplier already prefixes the class onto the message it packs
+  # ("#{e.class}: #{e.message}"), while other paths pack the bare message and
+  # leave the class in its own field. Prefixing unconditionally would print
+  # "NameError: NameError: ..." for the former.
+  def with_class_prefix(message, klass)
+    return message if klass.nil? || klass.empty? || message.start_with?("#{klass}:")
+
+    "#{klass}: #{message}"
+  end
+
+  def first_frame(result)
+    backtrace = result.error_backtrace
+    return nil if backtrace.nil? || backtrace.empty?
+
+    backtrace.first
+  end
+
+  def speculative_causes
+    "Likely causes: Rails/Zeitwerk autoloading breaking child eval; an env-specific " \
+      "RSpec config (e.g. fail_if_no_examples); a classify_status fallback defect; or " \
+      "an isolation-mode defect."
+  end
+end

data/lib/evilution/runner/isolation_resolver.rb

@@ -6,6 +6,7 @@ require_relative "../isolation/in_process"
 require_relative "../rails_detector"
 require_relative "../gem_detector"
 require_relative "../integration/loading/test_load_path"
+require_relative "../rails_warmup"
 
 class Evilution::Runner::IsolationResolver
   PRELOAD_CANDIDATES = [
@@ -48,6 +49,7 @@ class Evilution::Runner::IsolationResolver
     prepare_load_path_for_preload(path)
     prepare_integration_for_preload
     require File.expand_path(path)
+    warm_up_rails
   rescue Evilution::ConfigError
     raise
   rescue ScriptError, StandardError => e
@@ -61,6 +63,21 @@ class Evilution::Runner::IsolationResolver
 
   attr_reader :config, :hooks
 
+  # Runs only after a preload (which is what loads Rails). This is primarily an
+  # optimisation for :fork isolation, where children inherit the warmed state.
+  def warm_up_rails
+    return unless config.warmup == :rails
+
+    application = ::Rails.application if defined?(::Rails) && ::Rails.respond_to?(:application)
+    unless application
+      warn_message = "[evilution] warmup: rails requested but Rails is not loaded (or has no application) after preload; skipping"
+      Evilution::Diagnostic.warn(warn_message)
+      return
+    end
+
+    Evilution::RailsWarmup.new(application).call
+  end
+
   # Under :fork, allow preloading — caller resolves whether a path exists (an
   # explicit --preload / preload: value, or an auto-detected rails_helper) and
   # bails early when none does. Under :in_process, only allow preloading when
@@ -131,7 +148,11 @@ class Evilution::Runner::IsolationResolver
   #   have both (a bare `require "support/foo"` must still resolve from spec/).
   # - minitest/test-unit: a test/test_helper.rb doing a non-relative
   #   `require "support/..."` needs test/ on $LOAD_PATH. Route through
-  #   TestLoadPath -- the same policy the per-mutation test load uses  -- so preload and mutation paths agree.
+  #   TestLoadPath, the policy the per-mutation test load uses, with one
+  #   deliberate difference: outside_root_dirs: false. A resolved test file
+  #   outside test/ and spec/ keeps its own directory, but a preload there
+  #   (config/evilution_preload.rb) must not, or config/ lands ahead of the
+  #   gems and config/puma.rb shadows the puma gem (GH #1597).
   def prepare_load_path_for_preload(preload_path)
     if config.integration == :rspec
       prepare_rspec_preload_load_path
@@ -148,7 +169,7 @@ class Evilution::Runner::IsolationResolver
 
   def prepare_test_preload_load_path(preload_path)
     base = detected_rails_root || Evilution.project_base_dir
-    Evilution::Integration::Loading::TestLoadPath.add!([preload_path], base: base)
+    Evilution::Integration::Loading::TestLoadPath.add!([preload_path], base: base, outside_root_dirs: false)
   end
 
   def resolve_spec_dir

data/lib/evilution/runner/mutation_executor/neutralizer/baseline_failed.rb

@@ -4,6 +4,17 @@ require_relative "../neutralizer"
 require_relative "../../../result/mutation_result"
 require_relative "../../../result/neutral_reason"
 
+# Reclassifies a kill as :neutral when the tests failed on nothing but
+# examples that were already failing before any mutation ran.
+#
+# Such an example fails whatever the mutation is, so every mutation it runs
+# against would count as killed and a red spec file would score full marks.
+# The failure says nothing about the mutation: it has no verdict.
+#
+# A survivor is left alone, whatever the baseline did. Its tests passed, the
+# examples that were red in the baseline included, so the gap it reports is
+# real -- and neutralizing it, as used to happen for every survivor a red spec
+# file covered, hid exactly the mutations a reader needs to see.
 class Evilution::Runner::MutationExecutor::Neutralizer::BaselineFailed
   def initialize(config:, spec_resolver:, fallback_dir:)
     @config = config
@@ -12,25 +23,35 @@ class Evilution::Runner::MutationExecutor::Neutralizer::BaselineFailed
   end
 
   def call(result, baseline_result:)
-    return result unless result.survived? && baseline_result && baseline_result.failed?
-
-    if @config.spec_files.any?
-      should_neutralize = true
-      spec_file = nil
-    else
-      spec_file = @spec_resolver.call(result.mutation.file_path) || @fallback_dir
-      should_neutralize = baseline_result.failed_spec_files.include?(spec_file)
-    end
-    return result unless should_neutralize
-
-    neutralize(result, spec_file)
+    return result unless result.killed? && result.known_failures_only?
+
+    neutralize(result, red_spec(result, baseline_result))
   end
 
   private
 
+  # The red spec file to name: one covering the mutated source. A run given
+  # its spec files explicitly has no single one to point at.
+  def red_spec(result, baseline_result)
+    return nil if @config.spec_files.any? || baseline_result.nil?
+
+    failed = baseline_result.failed_spec_files
+    covering_specs(result.mutation.file_path).find { |spec| failed.include?(spec) }
+  end
+
+  # The spec files the baseline ran for this source: every file the resolver
+  # returns (a spec_selector may map one source to several), or the fallback
+  # directory when nothing resolves -- but only if the run falls back to the
+  # full suite; otherwise the baseline never ran it.
+  def covering_specs(file_path)
+    specs = Array(@spec_resolver.call(file_path))
+    return specs unless specs.empty?
+
+    @config.fallback_to_full_suite? ? [@fallback_dir] : []
+  end
+
   # The spec that was already red is the fact a reader needs: it says what to
-  # fix before the run can say anything about these mutations
-  # (EV-5pob / GH #1606).
+  # fix before the run can say anything about these mutations.
   def neutralize(result, spec_file)
     Evilution::Result::MutationResult.new(
       mutation: result.mutation,

data/lib/evilution/runner/mutation_executor/result_cache.rb

@@ -34,6 +34,9 @@ class Evilution::Runner::MutationExecutor::ResultCache
     return unless @backend
     return unless result.killed? || result.timeout?
     return if infra_crash?(result)
+    # Nor is one where only already-failing examples failed: the baseline
+    # that says so belongs to this run, and the cache keeps no trace of it.
+    return if result.known_failures_only?
 
     @backend.store(mutation,
                    status: result.status,

data/lib/evilution/runner/mutation_executor/result_packer.rb

@@ -15,7 +15,8 @@ class Evilution::Runner::MutationExecutor::ResultPacker
       parent_rss_kb: result.parent_rss_kb,
       error_message: result.error_message,
       error_class: result.error_class,
-      error_backtrace: result.error_backtrace
+      error_backtrace: result.error_backtrace,
+      known_failures_only: result.known_failures_only?
     }
   end
 
@@ -35,7 +36,8 @@ class Evilution::Runner::MutationExecutor::ResultPacker
         message: data[:error_message],
         klass: data[:error_class],
         backtrace: data[:error_backtrace]
-      )
+      ),
+      known_failures_only: data[:known_failures_only]
     )
   end
 end

data/lib/evilution/runner/mutation_executor.rb

@@ -30,7 +30,7 @@ class Evilution::Runner::MutationExecutor
   end
 
   def call(mutations, baseline_result = nil)
-    integration = @baseline_runner.build_integration
+    integration = @baseline_runner.build_integration(baseline_result)
     spec_resolver = baseline_failed?(baseline_result) ? @baseline_runner.neutralization_resolver : nil
     notifier = build_notifier
     pipeline = build_pipeline(spec_resolver)

data/lib/evilution/runner/mutation_planner.rb

@@ -7,7 +7,7 @@ require_relative "../ast/pattern/filter"
 require_relative "../equivalent/detector"
 
 class Evilution::Runner::MutationPlanner
-  Plan = Struct.new(:enabled, :equivalent, :skipped_count, :disabled_mutations, keyword_init: true)
+  Plan = Struct.new(:enabled, :equivalent, :skipped_count, :disabled_mutations, :mutated_lines, keyword_init: true)
 
   GenerationResult = Data.define(:mutations, :skipped)
   DisabledFilterResult = Data.define(:enabled, :disabled)
@@ -33,13 +33,22 @@ class Evilution::Runner::MutationPlanner
     sig_filter = filter_sig_blocks(disabled_filter.enabled)
     equivalent_filter = filter_equivalent(sig_filter.enabled)
 
-    build_plan(equivalent_filter, disabled_mutations, total_skipped(generation, disabled_filter, sig_filter))
+    plan = build_plan(equivalent_filter, disabled_mutations, total_skipped(generation, disabled_filter, sig_filter))
+    plan.mutated_lines = mutated_lines(deduped)
+    plan
   end
 
   private
 
   attr_reader :config, :registry
 
+  # The lines mutations were generated for, per file. Taken before any filter:
+  # a line whose mutations were all disabled or judged equivalent was still
+  # reached by an operator.
+  def mutated_lines(mutations)
+    mutations.group_by(&:file_path).transform_values { |group| group.map(&:line).uniq.sort }
+  end
+
   def compute_disabled_mutations(disabled_filter)
     return [] unless config.show_disabled?
 

data/lib/evilution/runner/report_publisher.rb

@@ -5,6 +5,7 @@ require_relative "../reporter/json"
 require_relative "../reporter/cli"
 require_relative "../reporter/html"
 require_relative "../session/store"
+require_relative "../diagnostic"
 
 class Evilution::Runner::ReportPublisher
   def initialize(config)
@@ -46,7 +47,7 @@ class Evilution::Runner::ReportPublisher
   def publish_html(output)
     path = "evilution-report.html"
     File.write(path, output)
-    warn "HTML report written to #{path}"
+    Evilution::Diagnostic.warn("HTML report written to #{path}")
   end
 
   def save_session(summary)
@@ -54,7 +55,7 @@ class Evilution::Runner::ReportPublisher
 
     Evilution::Session::Store.new.save(summary)
   rescue StandardError => e
-    warn "[evilution] failed to save session: #{e.message}" unless config.quiet
+    Evilution::Diagnostic.warn("[evilution] failed to save session: #{e.message}") unless config.quiet
   end
 
   private