evilution 1.3.0 → 1.5.0

data/lib/evilution/result/baseline_neutralization.rb

@@ -0,0 +1,10 @@
+# frozen_string_literal: true
+
+require_relative "../result"
+
+# The kills one red spec file left uncounted -- mutations whose tests failed
+# on nothing but examples that were already failing -- with why it was red.
+#
+# spec_file is nil when no single red file can be named, as in a run given
+# its spec files explicitly; failures then holds all of them.
+Evilution::Result::BaselineNeutralization = Data.define(:spec_file, :count, :failures)

data/lib/evilution/result/mutation_result.rb

@@ -9,8 +9,11 @@ class Evilution::Result::MutationResult
 
   attr_reader :mutation, :status, :duration, :killing_test, :test_command, :memory, :error, :neutral_reason
 
+  # known_failures_only: the tests failed, but every example that did was
+  # already failing before any mutation ran, so the failure says nothing
+  # about this one.
   def initialize(mutation:, status:, duration: 0.0, killing_test: nil,
-                 test_command: nil, memory: nil, error: nil, neutral_reason: nil)
+                 test_command: nil, memory: nil, error: nil, neutral_reason: nil, known_failures_only: false)
     raise ArgumentError, "invalid status: #{status}" unless STATUSES.include?(status)
 
     @mutation = mutation
@@ -21,6 +24,7 @@ class Evilution::Result::MutationResult
     @memory = memory
     @error = error
     @neutral_reason = neutral_reason
+    @known_failures_only = known_failures_only == true
     freeze
   end
 
@@ -54,6 +58,10 @@ class Evilution::Result::MutationResult
     @error.is_a?(Evilution::Result::ErrorInfo) ? @error.backtrace : nil
   end
 
+  def known_failures_only?
+    @known_failures_only
+  end
+
   def killed?
     status == :killed
   end

data/lib/evilution/result/summary.rb

@@ -3,13 +3,15 @@
 require_relative "../result"
 require_relative "coverage_gap_grouper"
 require_relative "subject_scorer"
+require_relative "baseline_neutralization"
 
 class Evilution::Result::Summary
   attr_reader :results, :duration, :skipped, :disabled_mutations, :unresolved_target_files,
-              :target_file_count, :infra_retried
+              :target_file_count, :infra_retried, :uncovered_code, :baseline_failures
 
   def initialize(results:, duration: 0.0, truncated: false, skipped: 0, disabled_mutations: [],
-                 unresolved_target_files: [], target_file_count: nil, infra_retried: 0)
+                 unresolved_target_files: [], target_file_count: nil, infra_retried: 0, uncovered_code: [],
+                 baseline_failures: [])
     @results = results
     @duration = duration
     @truncated = truncated
@@ -18,9 +20,27 @@ class Evilution::Result::Summary
     @unresolved_target_files = unresolved_target_files.freeze
     @target_file_count = target_file_count
     @infra_retried = infra_retried
+    @uncovered_code = uncovered_code.freeze
+    @baseline_failures = baseline_failures.freeze
     freeze
   end
 
+  # The kills not counted because only examples already failing in the
+  # baseline failed, gathered under the spec file that was red and paired with
+  # why it was. They are out of the score, so a red spec file would read as
+  # full marks unless the report says they are there.
+  def baseline_neutralizations
+    baseline_neutralized_results.group_by { |result| result.neutral_reason.detail }.map do |spec_file, results|
+      Evilution::Result::BaselineNeutralization.new(
+        spec_file: spec_file, count: results.length, failures: baseline_failures_for(spec_file)
+      )
+    end
+  end
+
+  def baseline_neutralized
+    baseline_neutralized_results.length
+  end
+
   # Neutral results gathered by the reason they were recorded, which is what the
   # report needs to say which kind of neutral a reader is looking at
   # (EV-5pob / GH #1606).
@@ -53,6 +73,12 @@ class Evilution::Result::Summary
     !unresolved_target_files.empty?
   end
 
+  # Targeted lines that hold code outside every subject, so no mutation was
+  # ever generated for them: `{ file:, lines: ["2-4", "9"] }` per file.
+  def uncovered_code?
+    !uncovered_code.empty?
+  end
+
   def truncated?
     @truncated
   end
@@ -167,4 +193,18 @@ class Evilution::Result::Summary
 
     max_rss && (max_rss / 1024.0)
   end
+
+  private
+
+  def baseline_neutralized_results
+    neutral_results.select { |result| result.neutral_reason && result.neutral_reason.kind == :baseline_failure }
+  end
+
+  # No spec file named means the run was given its spec files explicitly, and
+  # any of them being red neutralizes the survivor.
+  def baseline_failures_for(spec_file)
+    return baseline_failures if spec_file.nil?
+
+    baseline_failures.select { |failure| failure.spec_file == spec_file }
+  end
 end

data/lib/evilution/runner/baseline_runner.rb

@@ -12,6 +12,7 @@ require_relative "../integration/test_unit"
 require_relative "../example_filter"
 require_relative "../spec_ast_cache"
 require_relative "../source_ast_cache"
+require_relative "../diagnostic"
 
 unless defined?(Evilution::Runner::INTEGRATIONS)
   Evilution::Runner::INTEGRATIONS = {
@@ -33,15 +34,19 @@ class Evilution::Runner::BaselineRunner
     end
   end
 
-  def build_integration
+  # baseline_result: what the baseline found, when it ran. The tests it saw
+  # failing are handed to the integration, which can then tell a mutation run
+  # that fails on nothing else from one that kills.
+  def build_integration(baseline_result = nil)
     klass = integration_class
-    kwargs = base_integration_kwargs
+    kwargs = base_integration_kwargs(baseline_result)
     kwargs.merge!(rspec_integration_kwargs) if klass == Evilution::Integration::RSpec
     klass.new(**kwargs)
   end
 
-  def base_integration_kwargs
+  def base_integration_kwargs(baseline_result = nil)
     {
+      known_failures: baseline_result ? baseline_result.failed_example_ids : [],
       test_files: config.spec_files.empty? ? nil : config.spec_files,
       hooks: hooks,
       fallback_to_full_suite: config.fallback_to_full_suite?,
@@ -119,7 +124,7 @@ class Evilution::Runner::BaselineRunner
 
     Evilution::CoverageExampleFilter.new(map: map, lexical: lexical)
   rescue StandardError => e
-    warn "evilution: coverage targeting unavailable (#{e.class}: #{e.message}); using lexical targeting"
+    Evilution::Diagnostic.warn("evilution: coverage targeting unavailable (#{e.class}: #{e.message}); using lexical targeting")
     lexical
   end
 

data/lib/evilution/runner/canary.rb

@@ -6,6 +6,7 @@ require "securerandom"
 require_relative "../runner"
 require_relative "../mutation"
 require_relative "../subject"
+require_relative "canary_failure_message"
 
 # Runs one guaranteed-unobservable synthetic mutation through the configured
 # integration + isolation at session start. The synthetic spec never references
@@ -34,7 +35,7 @@ class Evilution::Runner::Canary
       test_command: ->(mutation) { build_integration(spec_path).call(mutation) },
       timeout: @config.timeout
     )
-    raise Failed, failure_message(result) unless result.status == :survived
+    raise Failed, Evilution::Runner::CanaryFailureMessage.new(result).to_s unless result.status == :survived
 
     nil
   ensure
@@ -145,60 +146,4 @@ class Evilution::Runner::Canary
   def build_integration(spec_path)
     @integration_class.new(test_files: [spec_path], hooks: @hooks)
   end
-
-  # When the child reported an error, that error IS the diagnosis -- naming it
-  # beats guessing. The speculative list stays only for the cases that carry no
-  # error at all (:killed, :timeout), where guesses are the only help there is.
-  # Diagnosing GH #1581 meant rebuilding this canary by hand to read the field
-  # this message used to drop, and none of the four guesses was the cause.
-  # EV-65nf / GH #1586.
-  def failure_message(result)
-    "#{failure_preamble(result.status)} #{diagnosis(result)} " \
-      "Re-run with --no-canary to bypass this check."
-  end
-
-  def failure_preamble(status)
-    "evilution proof-of-life canary failed: a guaranteed-unobservable synthetic " \
-      "mutation was scored #{status.inspect} instead of :survived. The mutation " \
-      "pipeline is misreporting — every score this run would produce is unreliable."
-  end
-
-  def diagnosis(result)
-    reported = reported_error(result)
-    return speculative_causes if reported.nil?
-
-    "The child reported: #{reported}"
-  end
-
-  def reported_error(result)
-    message = result.error_message
-    return nil if message.nil? || message.empty?
-
-    described = with_class_prefix(message, result.error_class)
-    frame = first_frame(result)
-    frame.nil? ? described : "#{described} (at #{frame})"
-  end
-
-  # MutationApplier already prefixes the class onto the message it packs
-  # ("#{e.class}: #{e.message}"), while other paths pack the bare message and
-  # leave the class in its own field. Prefixing unconditionally would print
-  # "NameError: NameError: ..." for the former.
-  def with_class_prefix(message, klass)
-    return message if klass.nil? || klass.empty? || message.start_with?("#{klass}:")
-
-    "#{klass}: #{message}"
-  end
-
-  def first_frame(result)
-    backtrace = result.error_backtrace
-    return nil if backtrace.nil? || backtrace.empty?
-
-    backtrace.first
-  end
-
-  def speculative_causes
-    "Likely causes: Rails/Zeitwerk autoloading breaking child eval; an env-specific " \
-      "RSpec config (e.g. fail_if_no_examples); a classify_status fallback defect; or " \
-      "an isolation-mode defect."
-  end
 end

data/lib/evilution/runner/canary_failure_message.rb

@@ -0,0 +1,101 @@
+# frozen_string_literal: true
+
+require_relative "../runner"
+
+# Explains why the proof-of-life canary failed, from the result its synthetic
+# mutation came back with. Kept apart from Canary, which only runs the probe:
+# the wording changes for different reasons than the mechanics do.
+class Evilution::Runner::CanaryFailureMessage
+  # Raised by RubyGems when it cannot load a gem at a version that satisfies
+  # every requirement already in play.
+  GEM_ACTIVATION_ERRORS = %w[
+    Gem::ConflictError Gem::LoadError Gem::MissingSpecError Gem::MissingSpecVersionError
+  ].freeze
+
+  def initialize(result)
+    @result = result
+  end
+
+  # When the child reported an error, that error IS the diagnosis -- naming it
+  # beats guessing. The speculative list stays only for the cases that carry no
+  # error at all (:killed, :timeout), where guesses are the only help there is.
+  # Diagnosing GH #1581 meant rebuilding this canary by hand to read the field
+  # this message used to drop, and none of the four guesses was the cause.
+  # EV-65nf / GH #1586.
+  def to_s
+    return gem_activation_message(@result) if gem_activation_failure?(@result)
+
+    "#{failure_preamble(@result.status)} #{diagnosis(@result)} " \
+      "Re-run with --no-canary to bypass this check."
+  end
+
+  private
+
+  # The usual cause is evilution running outside `bundle exec`: RubyGems then
+  # activates the newest installed copy of a shared dependency, which the test
+  # framework may not accept. Every mutation would fail the same way, so
+  # skipping the canary would not help.
+  def gem_activation_message(result)
+    "evilution proof-of-life canary failed: the test framework could not be loaded " \
+      "because of a gem activation conflict. This is an environment problem, not a " \
+      "mutation-pipeline defect, and every mutation would fail the same way. Run " \
+      "evilution under `bundle exec` so the project's Gemfile.lock decides gem versions " \
+      "(for the MCP server: command `bundle`, args [\"exec\", \"evilution\", \"mcp\"]). " \
+      "The child reported: #{reported_error(result)}"
+  end
+
+  # The class may arrive in its own field or only as the message prefix,
+  # depending on which path packed the error.
+  def gem_activation_failure?(result)
+    message = result.error_message.to_s
+
+    GEM_ACTIVATION_ERRORS.any? do |klass|
+      result.error_class == klass || message.start_with?("#{klass}:")
+    end
+  end
+
+  def failure_preamble(status)
+    "evilution proof-of-life canary failed: a guaranteed-unobservable synthetic " \
+      "mutation was scored #{status.inspect} instead of :survived. The mutation " \
+      "pipeline is misreporting — every score this run would produce is unreliable."
+  end
+
+  def diagnosis(result)
+    reported = reported_error(result)
+    return speculative_causes if reported.nil?
+
+    "The child reported: #{reported}"
+  end
+
+  def reported_error(result)
+    message = result.error_message
+    return nil if message.nil? || message.empty?
+
+    described = with_class_prefix(message, result.error_class)
+    frame = first_frame(result)
+    frame.nil? ? described : "#{described} (at #{frame})"
+  end
+
+  # MutationApplier already prefixes the class onto the message it packs
+  # ("#{e.class}: #{e.message}"), while other paths pack the bare message and
+  # leave the class in its own field. Prefixing unconditionally would print
+  # "NameError: NameError: ..." for the former.
+  def with_class_prefix(message, klass)
+    return message if klass.nil? || klass.empty? || message.start_with?("#{klass}:")
+
+    "#{klass}: #{message}"
+  end
+
+  def first_frame(result)
+    backtrace = result.error_backtrace
+    return nil if backtrace.nil? || backtrace.empty?
+
+    backtrace.first
+  end
+
+  def speculative_causes
+    "Likely causes: Rails/Zeitwerk autoloading breaking child eval; an env-specific " \
+      "RSpec config (e.g. fail_if_no_examples); a classify_status fallback defect; or " \
+      "an isolation-mode defect."
+  end
+end

data/lib/evilution/runner/mutation_executor/neutralizer/baseline_failed.rb

@@ -4,6 +4,17 @@ require_relative "../neutralizer"
 require_relative "../../../result/mutation_result"
 require_relative "../../../result/neutral_reason"
 
+# Reclassifies a kill as :neutral when the tests failed on nothing but
+# examples that were already failing before any mutation ran.
+#
+# Such an example fails whatever the mutation is, so every mutation it runs
+# against would count as killed and a red spec file would score full marks.
+# The failure says nothing about the mutation: it has no verdict.
+#
+# A survivor is left alone, whatever the baseline did. Its tests passed, the
+# examples that were red in the baseline included, so the gap it reports is
+# real -- and neutralizing it, as used to happen for every survivor a red spec
+# file covered, hid exactly the mutations a reader needs to see.
 class Evilution::Runner::MutationExecutor::Neutralizer::BaselineFailed
   def initialize(config:, spec_resolver:, fallback_dir:)
     @config = config
@@ -12,19 +23,22 @@ class Evilution::Runner::MutationExecutor::Neutralizer::BaselineFailed
   end
 
   def call(result, baseline_result:)
-    return result unless result.survived? && baseline_result && baseline_result.failed?
+    return result unless result.killed? && result.known_failures_only?
 
-    return neutralize(result, nil) if @config.spec_files.any?
-
-    failed = baseline_result.failed_spec_files
-    spec_file = covering_specs(result.mutation.file_path).find { |spec| failed.include?(spec) }
-    return result unless spec_file
-
-    neutralize(result, spec_file)
+    neutralize(result, red_spec(result, baseline_result))
   end
 
   private
 
+  # The red spec file to name: one covering the mutated source. A run given
+  # its spec files explicitly has no single one to point at.
+  def red_spec(result, baseline_result)
+    return nil if @config.spec_files.any? || baseline_result.nil?
+
+    failed = baseline_result.failed_spec_files
+    covering_specs(result.mutation.file_path).find { |spec| failed.include?(spec) }
+  end
+
   # The spec files the baseline ran for this source: every file the resolver
   # returns (a spec_selector may map one source to several), or the fallback
   # directory when nothing resolves -- but only if the run falls back to the

data/lib/evilution/runner/mutation_executor/result_cache.rb

@@ -34,6 +34,9 @@ class Evilution::Runner::MutationExecutor::ResultCache
     return unless @backend
     return unless result.killed? || result.timeout?
     return if infra_crash?(result)
+    # Nor is one where only already-failing examples failed: the baseline
+    # that says so belongs to this run, and the cache keeps no trace of it.
+    return if result.known_failures_only?
 
     @backend.store(mutation,
                    status: result.status,

data/lib/evilution/runner/mutation_executor/result_packer.rb

@@ -15,7 +15,8 @@ class Evilution::Runner::MutationExecutor::ResultPacker
       parent_rss_kb: result.parent_rss_kb,
       error_message: result.error_message,
       error_class: result.error_class,
-      error_backtrace: result.error_backtrace
+      error_backtrace: result.error_backtrace,
+      known_failures_only: result.known_failures_only?
     }
   end
 
@@ -35,7 +36,8 @@ class Evilution::Runner::MutationExecutor::ResultPacker
         message: data[:error_message],
         klass: data[:error_class],
         backtrace: data[:error_backtrace]
-      )
+      ),
+      known_failures_only: data[:known_failures_only]
     )
   end
 end

data/lib/evilution/runner/mutation_executor/strategy/parallel.rb

@@ -13,6 +13,15 @@ class Evilution::Runner::MutationExecutor::Strategy::Parallel
     error_class: "Evilution::Error"
   }.freeze
 
+  # Compact result for an item whose worker sent a frame that was not its own:
+  # the worker was given up, and what it had to say about the item with it.
+  UNREADABLE_COMPACT = {
+    status: :error,
+    duration: 0.0,
+    error_message: "worker's result could not be read; the worker was replaced",
+    error_class: "Evilution::Error"
+  }.freeze
+
   def initialize(cache:, isolator:, packer:, pipeline:, notifier:, pool_factory:, config:, diagnostics: nil)
     @cache = cache
     @isolator = isolator
@@ -113,6 +122,7 @@ class Evilution::Runner::MutationExecutor::Strategy::Parallel
     case result.reason
     when :timeout then timeout_compact
     when :died then DIED_COMPACT
+    when :unreadable then UNREADABLE_COMPACT
     end
   end
 

data/lib/evilution/runner/mutation_executor.rb

@@ -30,7 +30,7 @@ class Evilution::Runner::MutationExecutor
   end
 
   def call(mutations, baseline_result = nil)
-    integration = @baseline_runner.build_integration
+    integration = @baseline_runner.build_integration(baseline_result)
     spec_resolver = baseline_failed?(baseline_result) ? @baseline_runner.neutralization_resolver : nil
     notifier = build_notifier
     pipeline = build_pipeline(spec_resolver)

data/lib/evilution/runner/mutation_planner.rb

@@ -7,7 +7,7 @@ require_relative "../ast/pattern/filter"
 require_relative "../equivalent/detector"
 
 class Evilution::Runner::MutationPlanner
-  Plan = Struct.new(:enabled, :equivalent, :skipped_count, :disabled_mutations, keyword_init: true)
+  Plan = Struct.new(:enabled, :equivalent, :skipped_count, :disabled_mutations, :mutated_lines, keyword_init: true)
 
   GenerationResult = Data.define(:mutations, :skipped)
   DisabledFilterResult = Data.define(:enabled, :disabled)
@@ -33,13 +33,22 @@ class Evilution::Runner::MutationPlanner
     sig_filter = filter_sig_blocks(disabled_filter.enabled)
     equivalent_filter = filter_equivalent(sig_filter.enabled)
 
-    build_plan(equivalent_filter, disabled_mutations, total_skipped(generation, disabled_filter, sig_filter))
+    plan = build_plan(equivalent_filter, disabled_mutations, total_skipped(generation, disabled_filter, sig_filter))
+    plan.mutated_lines = mutated_lines(deduped)
+    plan
   end
 
   private
 
   attr_reader :config, :registry
 
+  # The lines mutations were generated for, per file. Taken before any filter:
+  # a line whose mutations were all disabled or judged equivalent was still
+  # reached by an operator.
+  def mutated_lines(mutations)
+    mutations.group_by(&:file_path).transform_values { |group| group.map(&:line).uniq.sort }
+  end
+
   def compute_disabled_mutations(disabled_filter)
     return [] unless config.show_disabled?
 

data/lib/evilution/runner/report_publisher.rb

@@ -5,6 +5,7 @@ require_relative "../reporter/json"
 require_relative "../reporter/cli"
 require_relative "../reporter/html"
 require_relative "../session/store"
+require_relative "../diagnostic"
 
 class Evilution::Runner::ReportPublisher
   def initialize(config)
@@ -46,7 +47,7 @@ class Evilution::Runner::ReportPublisher
   def publish_html(output)
     path = "evilution-report.html"
     File.write(path, output)
-    warn "HTML report written to #{path}"
+    Evilution::Diagnostic.warn("HTML report written to #{path}")
   end
 
   def save_session(summary)
@@ -54,7 +55,7 @@ class Evilution::Runner::ReportPublisher
 
     Evilution::Session::Store.new.save(summary)
   rescue StandardError => e
-    warn "[evilution] failed to save session: #{e.message}" unless config.quiet
+    Evilution::Diagnostic.warn("[evilution] failed to save session: #{e.message}") unless config.quiet
   end
 
   private

data/lib/evilution/runner/subject_pipeline/target.rb

@@ -0,0 +1,86 @@
+# frozen_string_literal: true
+
+require_relative "../subject_pipeline"
+
+# What `--target` asks for, read once from the text the user gave:
+#
+#   source:lib/**/*.rb   the files of a glob
+#   descendants:Base     a class and everything inheriting from it
+#   User#adult?          a method; `User`, `User#`, `Billing::*` for several
+#
+# NONE stands in when no target was given: it is no kind of target and
+# selects every subject, so the pipeline never has to ask whether there is
+# one.
+class Evilution::Runner::SubjectPipeline::Target
+  PREFIXES = { "source:" => :source_glob, "descendants:" => :descendants }.freeze
+  private_constant :PREFIXES
+
+  def self.parse(text)
+    return NONE if text.nil?
+
+    prefix, kind = PREFIXES.find { |candidate, _| text.start_with?(candidate) }
+    return new(kind: :method, text: text, value: text) if prefix.nil?
+
+    new(kind: kind, text: text, value: text.delete_prefix(prefix))
+  end
+
+  # The part after the prefix: the glob, the base class name, or the method
+  # name as written.
+  attr_reader :value
+
+  def initialize(kind:, text:, value:)
+    @kind = kind
+    @text = text
+    @value = value
+  end
+
+  def source_glob?
+    @kind == :source_glob
+  end
+
+  def descendants?
+    @kind == :descendants
+  end
+
+  def method?
+    @kind == :method
+  end
+
+  # The target as the user wrote it, for messages.
+  def to_s
+    @text.to_s
+  end
+
+  # Only a method target narrows subjects by name.
+  def selects?(subject)
+    !method? || matcher.call(subject.name)
+  end
+
+  NONE = new(kind: :none, text: nil, value: nil)
+
+  private
+
+  def matcher
+    @matcher ||= build_matcher
+  end
+
+  def build_matcher
+    return wildcard_matcher(value.chomp("*")) if value.end_with?("*")
+    return prefix_matcher(value) if value.end_with?("#", ".")
+
+    name_matcher(value)
+  end
+
+  def wildcard_matcher(prefix)
+    ->(name) { name.split(/[#.]/).first.start_with?(prefix) }
+  end
+
+  def prefix_matcher(prefix)
+    ->(name) { name.start_with?(prefix) }
+  end
+
+  # A full method name matches itself; a class name, each of its methods.
+  def name_matcher(target)
+    ->(name) { name == target || name.start_with?("#{target}#") || name.start_with?("#{target}.") }
+  end
+end