evilution 0.21.0 → 0.22.1

data/lib/evilution/result/mutation_result.rb

@@ -6,11 +6,15 @@ class Evilution::Result::MutationResult
   STATUSES = %i[killed survived timeout error neutral equivalent].freeze
 
   attr_reader :mutation, :status, :duration, :killing_test, :test_command,
-              :child_rss_kb, :memory_delta_kb, :parent_rss_kb
+              :child_rss_kb, :memory_delta_kb, :parent_rss_kb,
+              :error_message, :error_class, :error_backtrace
 
+  # rubocop:disable Metrics/ParameterLists
   def initialize(mutation:, status:, duration: 0.0, killing_test: nil,
                  test_command: nil, child_rss_kb: nil, memory_delta_kb: nil,
-                 parent_rss_kb: nil)
+                 parent_rss_kb: nil, error_message: nil, error_class: nil,
+                 error_backtrace: nil)
+    # rubocop:enable Metrics/ParameterLists
     raise ArgumentError, "invalid status: #{status}" unless STATUSES.include?(status)
 
     @mutation = mutation
@@ -21,6 +25,9 @@ class Evilution::Result::MutationResult
     @child_rss_kb = child_rss_kb
     @memory_delta_kb = memory_delta_kb
     @parent_rss_kb = parent_rss_kb
+    @error_message = error_message
+    @error_class = error_class
+    @error_backtrace = error_backtrace.nil? ? nil : error_backtrace.dup.freeze
     freeze
   end
 

data/lib/evilution/runner.rb

@@ -8,6 +8,7 @@ require_relative "mutator/registry"
 require_relative "isolation/fork"
 require_relative "isolation/in_process"
 require_relative "integration/rspec"
+require_relative "integration/minitest"
 require_relative "reporter/json"
 require_relative "reporter/cli"
 require_relative "reporter/html"
@@ -26,6 +27,11 @@ require_relative "disable_comment"
 require_relative "ast/sorbet_sig_detector"
 
 class Evilution::Runner
+  INTEGRATIONS = {
+    rspec: Evilution::Integration::RSpec,
+    minitest: Evilution::Integration::Minitest
+  }.freeze
+
   attr_reader :config
 
   def initialize(config: Evilution::Config.new, on_result: nil, hooks: nil)
@@ -286,7 +292,8 @@ class Evilution::Runner
     return nil unless config.baseline? && subjects.any?
 
     log_baseline_start
-    baseline = Evilution::Baseline.new(timeout: config.timeout)
+    integration_class = resolve_integration_class
+    baseline = Evilution::Baseline.new(timeout: config.timeout, **integration_class.baseline_options)
     result = baseline.call(subjects)
     log_baseline_complete(result)
     result
@@ -305,7 +312,7 @@ class Evilution::Runner
 
   def run_mutations_sequential(mutations, baseline_result = nil)
     integration = build_integration
-    spec_resolver = baseline_result&.failed? ? Evilution::SpecResolver.new : nil
+    spec_resolver = baseline_result&.failed? ? build_neutralization_resolver : nil
     results = []
     survived_count = 0
     truncated = false
@@ -334,7 +341,7 @@ class Evilution::Runner
     integration = build_integration
     pool = Evilution::Parallel::Pool.new(size: config.jobs, hooks: @hooks, item_timeout: config.timeout ? config.timeout * 2 : nil)
     worker_isolator = Evilution::Isolation::InProcess.new
-    spec_resolver = baseline_result&.failed? ? Evilution::SpecResolver.new : nil
+    spec_resolver = baseline_result&.failed? ? build_neutralization_resolver : nil
     state = { results: [], survived_count: 0, truncated: false, completed: 0 }
 
     all_worker_stats = []
@@ -392,7 +399,7 @@ class Evilution::Runner
     if config.spec_files.any?
       neutralize = true
     else
-      spec_file = spec_resolver.call(result.mutation.file_path) || "spec"
+      spec_file = spec_resolver.call(result.mutation.file_path) || neutralization_fallback_dir
       neutralize = baseline_result.failed_spec_files.include?(spec_file)
     end
     return result unless neutralize
@@ -404,7 +411,10 @@ class Evilution::Runner
       test_command: result.test_command,
       child_rss_kb: result.child_rss_kb,
       memory_delta_kb: result.memory_delta_kb,
-      parent_rss_kb: result.parent_rss_kb
+      parent_rss_kb: result.parent_rss_kb,
+      error_message: result.error_message,
+      error_class: result.error_class,
+      error_backtrace: result.error_backtrace
     )
   end
 
@@ -416,7 +426,10 @@ class Evilution::Runner
       test_command: result.test_command,
       child_rss_kb: result.child_rss_kb,
       memory_delta_kb: result.memory_delta_kb,
-      parent_rss_kb: result.parent_rss_kb
+      parent_rss_kb: result.parent_rss_kb,
+      error_message: result.error_message,
+      error_class: result.error_class,
+      error_backtrace: result.error_backtrace
     }
   end
 
@@ -430,7 +443,10 @@ class Evilution::Runner
         test_command: data[:test_command],
         child_rss_kb: data[:child_rss_kb],
         memory_delta_kb: data[:memory_delta_kb],
-        parent_rss_kb: data[:parent_rss_kb]
+        parent_rss_kb: data[:parent_rss_kb],
+        error_message: data[:error_message],
+        error_class: data[:error_class],
+        error_backtrace: data[:error_backtrace]
       )
     end
   end
@@ -472,16 +488,28 @@ class Evilution::Runner
     :in_process
   end
 
-  def build_integration
-    case config.integration
-    when :rspec
-      test_files = config.spec_files.empty? ? nil : config.spec_files
-      Evilution::Integration::RSpec.new(test_files: test_files, hooks: @hooks)
-    else
+  def resolve_integration_class
+    INTEGRATIONS.fetch(config.integration) do
       raise Evilution::Error, "unknown integration: #{config.integration}"
     end
   end
 
+  def build_integration
+    klass = resolve_integration_class
+    test_files = config.spec_files.empty? ? nil : config.spec_files
+    klass.new(test_files: test_files, hooks: @hooks)
+  end
+
+  def build_neutralization_resolver
+    options = resolve_integration_class.baseline_options
+    options[:spec_resolver] || Evilution::SpecResolver.new
+  end
+
+  def neutralization_fallback_dir
+    options = resolve_integration_class.baseline_options
+    options[:fallback_dir] || "spec"
+  end
+
   def output_report(summary)
     reporter = build_reporter
     return unless reporter
@@ -543,9 +571,20 @@ class Evilution::Runner
 
     parts << gc_stats_string
 
-    return if parts.empty?
+    $stderr.write("[verbose] #{result.mutation}: #{parts.join(", ")}\n") unless parts.empty?
 
-    $stderr.write("[verbose] #{result.mutation}: #{parts.join(", ")}\n")
+    log_mutation_error(result) if result.error?
+  end
+
+  def log_mutation_error(result)
+    header = "[verbose] #{result.mutation}: error"
+    header += " #{result.error_class}" if result.error_class
+    header += ": #{result.error_message}" if result.error_message
+    $stderr.write("#{header}\n")
+
+    Array(result.error_backtrace).first(5).each do |line|
+      $stderr.write("[verbose]   #{line}\n")
+    end
   end
 
   def gc_stats_string
@@ -604,11 +643,11 @@ class Evilution::Runner
   def build_reporter
     case config.format
     when :json
-      Evilution::Reporter::JSON.new
+      Evilution::Reporter::JSON.new(integration: config.integration)
     when :text
       Evilution::Reporter::CLI.new
     when :html
-      Evilution::Reporter::HTML.new(baseline: load_baseline_session)
+      Evilution::Reporter::HTML.new(baseline: load_baseline_session, integration: config.integration)
     end
   end
 

data/lib/evilution/spec_resolver.rb

@@ -4,11 +4,17 @@ class Evilution::SpecResolver
   STRIPPABLE_PREFIXES = %w[lib/ app/].freeze
   CONTROLLER_PREFIX = "controllers/"
 
+  def initialize(test_dir: "spec", test_suffix: "_spec.rb", request_dir: "requests")
+    @test_dir = test_dir
+    @test_suffix = test_suffix
+    @request_dir = request_dir
+  end
+
   def call(source_path)
     return nil if source_path.nil? || source_path.empty?
 
     normalized = normalize_path(source_path)
-    candidates = candidate_spec_paths(normalized)
+    candidates = candidate_test_paths(normalized)
     candidates.find { |path| File.exist?(path) }
   end
 
@@ -24,16 +30,16 @@ class Evilution::SpecResolver
     path
   end
 
-  def candidate_spec_paths(source_path)
-    base = source_path.sub(/\.rb\z/, "_spec.rb")
+  def candidate_test_paths(source_path)
+    base = source_path.sub(/\.rb\z/, @test_suffix)
     prefix = STRIPPABLE_PREFIXES.find { |p| source_path.start_with?(p) }
 
     candidates = if prefix
                    stripped = base.delete_prefix(prefix)
-                   request_spec = controller_to_request_spec(stripped)
-                   [request_spec, "spec/#{stripped}", "spec/#{base}"].compact
+                   request_test = controller_to_request_test(stripped)
+                   [request_test, "#{@test_dir}/#{stripped}", "#{@test_dir}/#{base}"].compact
                  else
-                   ["spec/#{base}"]
+                   ["#{@test_dir}/#{base}"]
                  end
 
     fallbacks = candidates.flat_map { |c| parent_fallback_candidates(c) }.uniq
@@ -41,33 +47,35 @@ class Evilution::SpecResolver
     candidates + fallbacks
   end
 
-  def controller_to_request_spec(stripped_path)
+  def controller_to_request_test(stripped_path)
     return nil unless stripped_path.start_with?(CONTROLLER_PREFIX)
-    return nil unless stripped_path.end_with?("_controller_spec.rb")
+
+    controller_suffix = "_controller#{@test_suffix}"
+    return nil unless stripped_path.end_with?(controller_suffix)
 
     request_path = stripped_path
                    .delete_prefix(CONTROLLER_PREFIX)
-                   .sub(/_controller_spec\.rb\z/, "_spec.rb")
-    "spec/requests/#{request_path}"
+                   .sub(/#{Regexp.escape(controller_suffix)}\z/, @test_suffix)
+    "#{@test_dir}/#{@request_dir}/#{request_path}"
   end
 
-  def parent_fallback_candidates(spec_path)
-    parts = spec_path.split("/")
+  def parent_fallback_candidates(test_path)
+    parts = test_path.split("/")
     # parts: ["spec", "foo", "bar_spec.rb"] — need at least 3 parts for fallback
     return [] if parts.length < 3
 
     candidates = []
     # Remove filename, then progressively remove directories
-    dir_parts = parts[1..-2] # ["models", "game"]
+    dir_parts = parts[1..-2]
 
     (dir_parts.length - 1).downto(0) do |i|
-      file = "#{dir_parts[i]}_spec.rb"
+      file = "#{dir_parts[i]}#{@test_suffix}"
 
       if i.zero?
-        candidates << "spec/#{file}"
+        candidates << "#{@test_dir}/#{file}"
       else
         parent = dir_parts[0...i].join("/")
-        candidates << "spec/#{parent}/#{file}"
+        candidates << "#{@test_dir}/#{parent}/#{file}"
       end
     end
 

data/lib/evilution/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Evilution
-  VERSION = "0.21.0"
+  VERSION = "0.22.1"
 end

data/lib/evilution.rb

@@ -42,9 +42,11 @@ require_relative "evilution/mutator/operator/collection_replacement"
 require_relative "evilution/mutator/operator/method_call_removal"
 require_relative "evilution/mutator/operator/argument_removal"
 require_relative "evilution/mutator/operator/block_removal"
+require_relative "evilution/mutator/operator/block_pass_removal"
 require_relative "evilution/mutator/operator/conditional_flip"
 require_relative "evilution/mutator/operator/range_replacement"
 require_relative "evilution/mutator/operator/regexp_mutation"
+require_relative "evilution/mutator/operator/regex_simplification"
 require_relative "evilution/mutator/operator/receiver_replacement"
 require_relative "evilution/mutator/operator/send_mutation"
 require_relative "evilution/mutator/operator/argument_nil_substitution"
@@ -69,6 +71,7 @@ require_relative "evilution/mutator/operator/bitwise_replacement"
 require_relative "evilution/mutator/operator/bitwise_complement"
 require_relative "evilution/mutator/operator/zsuper_removal"
 require_relative "evilution/mutator/operator/explicit_super_mutation"
+require_relative "evilution/mutator/operator/index_to_at"
 require_relative "evilution/mutator/operator/index_to_fetch"
 require_relative "evilution/mutator/operator/index_to_dig"
 require_relative "evilution/mutator/operator/index_assignment_removal"
@@ -106,6 +109,7 @@ require_relative "evilution/git/changed_files"
 require_relative "evilution/integration"
 require_relative "evilution/integration/base"
 require_relative "evilution/integration/rspec"
+require_relative "evilution/integration/minitest"
 require_relative "evilution/result/mutation_result"
 require_relative "evilution/result/summary"
 require_relative "evilution/reporter"

data/script/memory_check

@@ -104,12 +104,12 @@ complex_mutations = complex_subjects.flat_map { |s| complex_registry.mutations_f
 
 integration = Evilution::Integration::RSpec.new(test_files: [COMPLEX_FIXTURE_SPEC])
 
-all_passed &= run_check("RSpec integration per-mutation (Config)", iterations: 20, max_growth_kb: 20_480) do
+# Budget is generous: per-mutation require() adds the file to $LOADED_FEATURES and
+# accumulates constant-redefinition warnings. Local runs land around 20-25 MB; CI
+# varies up to ~30 MB depending on Ruby build and GC pressure, so we leave headroom.
+all_passed &= run_check("RSpec integration per-mutation (Config)", iterations: 20, max_growth_kb: 40_960) do
   mutation = complex_mutations.sample
-  result = integration.call(mutation)
-  raise "RSpec integration memory check failed: #{result[:error]}" if result[:error]
-
-  result
+  integration.call(mutation)
 end
 
 puts all_passed ? "All memory checks passed." : "Some memory checks failed!"

data/scripts/benchmark_density

@@ -0,0 +1,261 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+# Mutation Density Benchmark
+#
+# Compares mutation counts between evilution and a reference mutation testing
+# tool on a corpus of files.
+# See docs/mutation_density_benchmark.md for methodology.
+#
+# Usage:
+#   scripts/benchmark_density CONFIG_FILE [--verbose]
+#
+# The config file is a YAML file listing the benchmark corpus.
+# See scripts/benchmark_density.yml for the format.
+
+require "yaml"
+require "open3"
+require "optparse"
+
+module BenchmarkDensity
+  class ConfigError < StandardError; end
+
+  class Config
+    attr_reader :project_root, :target_ratio, :reference_cmd, :files
+
+    def initialize(path)
+      data = YAML.safe_load_file(path, symbolize_names: true, permitted_classes: [Symbol])
+      raise ConfigError, "config file must contain a YAML mapping" unless data.is_a?(Hash)
+
+      @project_root = data[:project_root]
+      @target_ratio = data.key?(:target_ratio) ? data[:target_ratio] : 1.5
+      @reference_cmd = data.key?(:reference_cmd) ? data[:reference_cmd] : []
+      @files = data.key?(:files) ? data[:files] : []
+
+      validate!
+    rescue Psych::Exception => e
+      raise ConfigError, "failed to parse config file #{path}: #{e.message}"
+    end
+
+    private
+
+    def validate!
+      raise ConfigError, "project_root is required in config" if @project_root.nil?
+      raise ConfigError, "project_root does not exist: #{@project_root}" unless Dir.exist?(@project_root)
+      raise ConfigError, "reference_cmd is required in config" if @reference_cmd.empty?
+      raise ConfigError, "files list is empty — add benchmark files to the config" if @files.empty?
+
+      @files.each_with_index { |entry, index| validate_file_entry!(entry, index) }
+    end
+
+    def validate_file_entry!(entry, index)
+      raise ConfigError, "files[#{index}] must be a mapping" unless entry.is_a?(Hash)
+
+      path = entry[:path]
+      target = entry[:reference_target]
+      raise ConfigError, "files[#{index}] is missing required path" if path.nil? || path.to_s.strip.empty?
+      raise ConfigError, "files[#{index}] is missing required reference_target" if target.nil? || target.to_s.strip.empty?
+    end
+  end
+
+  class EvilutionCounter
+    def initialize(project_root)
+      @project_root = project_root
+    end
+
+    def count_mutations(file_path)
+      full_path = File.join(@project_root, file_path)
+      cmd = ["bundle", "exec", "evilution", "subjects", full_path]
+      stdout, stderr, status = Open3.capture3(*cmd, chdir: @project_root)
+
+      unless status.success?
+        warn "  evilution failed for #{file_path}: #{stderr.strip}"
+        return nil
+      end
+
+      count = parse_subject_count(stdout)
+      if count.nil?
+        warn "  evilution produced unparseable output for #{file_path}: #{stdout.lines.first&.strip}"
+        return nil
+      end
+      count
+    end
+
+    private
+
+    def parse_subject_count(output)
+      match = output.match(/(\d+)\s+subjects,\s+(\d+)\s+mutations/)
+      match ? match[2].to_i : nil
+    end
+  end
+
+  class ReferenceCounter
+    def initialize(project_root, reference_cmd)
+      @project_root = project_root
+      @reference_cmd = reference_cmd
+    end
+
+    def count_mutations(reference_target)
+      cmd = @reference_cmd + [reference_target]
+      stdout, stderr, _status = Open3.capture3(*cmd, chdir: @project_root)
+
+      # Reference tool may exit non-zero when mutants survive, which is expected.
+      # Only treat it as failure if there's no parseable output.
+      count = parse_mutation_count(stdout)
+      if count.nil?
+        warn "  reference tool failed for #{reference_target}: #{stderr.lines.first&.strip}"
+        return nil
+      end
+      count
+    end
+
+    private
+
+    def parse_mutation_count(output)
+      match = output.match(/mutations:\s*(\d+)/i)
+      match ? match[1].to_i : nil
+    end
+  end
+
+  class Runner
+    HEADER_FMT = "%-45<file>s %10<evilution>s %10<reference>s %8<ratio>s"
+    ROW_FMT = "%-45<file>s %10<evilution>s %10<reference>s %8<ratio>s"
+    TOTAL_FMT = "%-45<file>s %10<evilution>d %10<reference>d %7.2<ratio>fx"
+
+    def initialize(config:, verbose: false)
+      @config = config
+      @verbose = verbose
+      @evilution = EvilutionCounter.new(config.project_root)
+      @reference = ReferenceCounter.new(config.project_root, config.reference_cmd)
+    end
+
+    def call
+      results = collect_results
+      print_table(results)
+      print_summary(results)
+      passing?(results) ? 0 : 1
+    end
+
+    private
+
+    def collect_results
+      @config.files.map do |entry|
+        path = entry[:path]
+        reference_target = entry[:reference_target]
+
+        warn "  Measuring #{path}..." if @verbose
+        ev_count = @evilution.count_mutations(path)
+        ref_count = @reference.count_mutations(reference_target)
+        warn "  done" if @verbose
+
+        { path: path, evilution: ev_count, reference: ref_count }
+      end
+    end
+
+    def print_table(results)
+      puts format(HEADER_FMT, file: "File", evilution: "Evilution", reference: "Reference", ratio: "Ratio")
+      puts "-" * 75
+
+      results.each do |r|
+        ratio = compute_ratio(r[:evilution], r[:reference])
+        ratio_str = ratio ? format("%.2fx", ratio) : "N/A"
+        ev_str = r[:evilution]&.to_s || "ERR"
+        ref_str = r[:reference]&.to_s || "ERR"
+        puts format(ROW_FMT, file: r[:path], evilution: ev_str, reference: ref_str, ratio: ratio_str)
+      end
+
+      puts "-" * 75
+    end
+
+    def print_summary(results)
+      totals = compute_totals(results)
+      print_total_line(totals)
+      puts ""
+      print_status_line(totals)
+      print_error_warning(totals[:error_count])
+    end
+
+    def print_total_line(totals)
+      if totals[:invalid]
+        puts format(ROW_FMT,
+                    file: "TOTAL", evilution: totals[:ev].to_s, reference: totals[:ref].to_s, ratio: "N/A")
+      else
+        puts format(TOTAL_FMT,
+                    file: "TOTAL", evilution: totals[:ev], reference: totals[:ref], ratio: totals[:display_ratio])
+      end
+    end
+
+    def compute_totals(results)
+      valid, errored = partition_results(results)
+      ev_total = valid.sum { |r| r[:evilution] }
+      ref_total = valid.sum { |r| r[:reference] }
+      invalid = errored.any? || ev_total.zero?
+      ratio = ev_total.positive? ? ref_total.to_f / ev_total : 0.0
+
+      { ev: ev_total, ref: ref_total, ratio: ratio, display_ratio: ratio.round(2),
+        invalid: invalid, error_count: errored.length }
+    end
+
+    def partition_results(results)
+      valid = results.select { |r| r[:evilution] && r[:reference] }
+      errored = results - valid
+      [valid, errored]
+    end
+
+    def print_status_line(totals)
+      target = @config.target_ratio
+      if totals[:invalid]
+        puts "Density ratio: INVALID (target: < #{target}x) — FAIL"
+        return
+      end
+
+      status = totals[:ratio] < target ? "PASS" : "FAIL"
+      puts "Density ratio: #{totals[:display_ratio]}x (target: < #{target}x) — #{status}"
+    end
+
+    def print_error_warning(error_count)
+      puts "WARNING: #{error_count} file(s) had errors — benchmark marked invalid" if error_count.positive?
+    end
+
+    def passing?(results)
+      totals = compute_totals(results)
+      return false if totals[:invalid]
+
+      totals[:ratio] < @config.target_ratio
+    end
+
+    def compute_ratio(evilution, reference)
+      return nil if evilution.nil? || reference.nil? || evilution.zero?
+
+      (reference.to_f / evilution).round(2)
+    end
+  end
+end
+
+if __FILE__ == $PROGRAM_NAME
+  verbose = false
+  parser = OptionParser.new do |opts|
+    opts.banner = "Usage: #{$PROGRAM_NAME} CONFIG_FILE [--verbose]"
+    opts.on("-v", "--verbose", "Show progress") { verbose = true }
+    opts.on("-h", "--help", "Show help") do
+      puts opts
+      exit 0
+    end
+  end
+  parser.parse!
+
+  config_path = ARGV.first
+  unless config_path
+    warn parser.banner
+    exit 2
+  end
+
+  begin
+    config = BenchmarkDensity::Config.new(config_path)
+    runner = BenchmarkDensity::Runner.new(config: config, verbose: verbose)
+    exit runner.call
+  rescue BenchmarkDensity::ConfigError => e
+    warn "Config error: #{e.message}"
+    exit 2
+  end
+end

data/scripts/benchmark_density.yml

@@ -0,0 +1,19 @@
+# Benchmark corpus configuration
+#
+# See docs/mutation_density_benchmark.md for selection criteria and methodology.
+#
+# Example:
+#
+# project_root: /path/to/rails/app
+# target_ratio: 1.5
+# reference_cmd: ["bundle", "exec", "mutant", "run", "--integration", "rspec", "--fail-fast"]
+# files:
+#   - path: app/models/user.rb
+#     reference_target: "User"
+#   - path: app/services/payment_processor.rb
+#     reference_target: "PaymentProcessor"
+
+project_root: null  # Set to the absolute path of the target project
+target_ratio: 1.5
+reference_cmd: []  # Command prefix for the reference tool
+files: []