evilution 1.2.0 → 1.4.0

data/lib/evilution/runner/subject_pipeline.rb

@@ -2,6 +2,8 @@
 
 require_relative "../runner"
 require_relative "../ast/inheritance_scanner"
+require_relative "../ast/uncovered_code"
+require_relative "../diagnostic"
 require_relative "../git/changed_files"
 
 class Evilution::Runner::SubjectPipeline
@@ -12,6 +14,7 @@ class Evilution::Runner::SubjectPipeline
 
   def call
     subjects = parse_subjects
+    @subjects_by_file = subjects.group_by(&:file_path)
     subjects = filter_by_descendants(subjects) if descendants_target?
     subjects = filter_by_target(subjects) if method_target?
     subjects = filter_by_line_ranges(subjects) if config.line_ranges?
@@ -22,6 +25,28 @@ class Evilution::Runner::SubjectPipeline
     @target_files ||= resolve_target_files
   end
 
+  # Warns about the targeted lines that hold code outside every subject and
+  # that no mutation reached either. mutated_lines: the lines mutations were
+  # generated for, per file -- known only once #call's subjects are planned.
+  #
+  # A line range is checked as given; a whole file only when it has no
+  # subjects at all -- every class with an `include` or a constant would
+  # otherwise warn.
+  def report_uncovered_code(mutated_lines = {})
+    by_file = @subjects_by_file || {}
+    @subjects_by_file = nil
+    @uncovered_code = target_files.filter_map do |file|
+      uncovered_entry(file, by_file.fetch(file, []), mutated_lines.fetch(file, []))
+    end
+    @uncovered_code.each { |entry| warn_uncovered(entry) }
+  end
+
+  # Those lines, as `{ file:, lines: ["2-4", "9"] }` per file, once
+  # #report_uncovered_code has run.
+  def uncovered_code
+    @uncovered_code || []
+  end
+
   private
 
   attr_reader :config, :parser
@@ -30,6 +55,26 @@ class Evilution::Runner::SubjectPipeline
     target_files.flat_map { |file| parser.call(file) }
   end
 
+  def uncovered_entry(file, file_subjects, mutated_lines)
+    range = config.line_ranges[file]
+    return unless range || file_subjects.empty?
+
+    lines = Evilution::AST::UncoveredCode.call(file, file_subjects, lines: range, mutated_lines: mutated_lines)
+    { file: file, lines: lines.map { |run| line_label(run) } } unless lines.empty?
+  end
+
+  # Spelled the way a range is passed on the command line: `2-4`, or `9`.
+  def line_label(run)
+    run.size == 1 ? run.first.to_s : "#{run.first}-#{run.last}"
+  end
+
+  def warn_uncovered(entry)
+    Evilution::Diagnostic.warn(
+      "[evilution] #{entry[:file]}:#{entry[:lines].join(", ")} holds code outside every subject " \
+      "(class-body code such as DSL calls and constants is not mutated); no mutations target those lines."
+    )
+  end
+
   def source_glob_target?
     config.target&.start_with?("source:")
   end
@@ -124,7 +169,7 @@ class Evilution::Runner::SubjectPipeline
   end
 
   def class_matcher(target)
-    ->(s) { s.name.start_with?("#{target}#") || s.name.start_with?("#{target}.") }
+    ->(s) { s.name == target || s.name.start_with?("#{target}#") || s.name.start_with?("#{target}.") }
   end
 
   def filter_by_line_ranges(subjects)

data/lib/evilution/runner.rb

@@ -38,6 +38,7 @@ class Evilution::Runner
     baseline_result = run_baseline(subjects)
 
     plan = mutation_planner.call(subjects)
+    subject_pipeline.report_uncovered_code(plan.mutated_lines)
     release_subject_nodes(subjects)
     clear_operator_caches
     execution = run_mutations(plan.enabled, baseline_result)
@@ -52,7 +53,9 @@ class Evilution::Runner
                                              disabled_mutations: plan.disabled_mutations,
                                              unresolved_target_files: target_spec_audit.call(audited_files),
                                              target_file_count: audited_files.length,
-                                             infra_retried: execution.infra_retried)
+                                             infra_retried: execution.infra_retried,
+                                             uncovered_code: subject_pipeline.uncovered_code,
+                                             baseline_failures: baseline_result ? baseline_result.failures : [])
     output_report(summary)
     save_session(summary)
 

data/lib/evilution/source_ast_cache.rb

@@ -2,7 +2,7 @@
 
 require "digest"
 require "prism"
-require_relative "../evilution"
+require_relative "version"
 
 # Content-hash-keyed LRU of Prism::ParseResult. Different source bytes always
 # yield a different key, so the cache is valid for the lifetime of the process.

data/lib/evilution/spec_ast_cache.rb

@@ -1,7 +1,7 @@
 # frozen_string_literal: true
 
 require "prism"
-require_relative "../evilution"
+require_relative "version"
 
 class Evilution::SpecAstCache
   Block = Struct.new(:kind, :line, :end_line, :body_text)

data/lib/evilution/subject.rb

@@ -1,14 +1,20 @@
 # frozen_string_literal: true
 
+# What mutations are generated for and reported against. A :method subject is
+# a method definition; a :constant subject is a definition assigned to a
+# constant outside any method, such as `Point = Data.define(:x, :y)`. A :scope
+# or :aasm subject is the body of a callable handed to a class-body
+# declaration.
 class Evilution::Subject
-  attr_reader :name, :file_path, :line_number, :source, :node
+  attr_reader :name, :file_path, :line_number, :source, :node, :kind
 
-  def initialize(name:, file_path:, line_number:, source:, node:)
+  def initialize(name:, file_path:, line_number:, source:, node:, kind: :method)
     @name = name
     @file_path = file_path
     @line_number = line_number
     @source = source
     @node = node
+    @kind = kind
   end
 
   def release_node!

data/lib/evilution/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Evilution
-  VERSION = "1.2.0"
+  VERSION = "1.4.0"
 end

data/lib/evilution.rb

@@ -14,6 +14,7 @@ require_relative "evilution/ast/inheritance_scanner"
 require_relative "evilution/ast/pattern"
 require_relative "evilution/ast/sorbet_sig_detector"
 require_relative "evilution/ast/local_reads"
+require_relative "evilution/ast/regexp_pattern"
 require_relative "evilution/ast/pattern/matcher"
 require_relative "evilution/ast/pattern/parser"
 require_relative "evilution/hooks"
@@ -30,6 +31,56 @@ require_relative "evilution/mutator/operator/nil_replacement"
 require_relative "evilution/mutator/operator/boolean_operator_replacement"
 require_relative "evilution/mutator/operator/boolean_operand_promotion"
 require_relative "evilution/mutator/operator/boolean_expression_to_nil"
+require_relative "evilution/mutator/operator/call_to_nil"
+require_relative "evilution/mutator/operator/safe_navigation_removal"
+require_relative "evilution/mutator/operator/attribute_write_to_read"
+require_relative "evilution/mutator/operator/argument_propagation"
+require_relative "evilution/mutator/operator/argument_list_removal"
+require_relative "evilution/mutator/operator/symbol_to_proc_replacement"
+require_relative "evilution/mutator/operator/dynamic_dispatch_resolution"
+require_relative "evilution/mutator/operator/keyword_argument_removal"
+require_relative "evilution/mutator/operator/double_negation_removal"
+require_relative "evilution/mutator/operator/regexp_anchor_to_predicate"
+require_relative "evilution/mutator/operator/reduce_to_sum"
+require_relative "evilution/mutator/operator/array_coercion_to_literal"
+require_relative "evilution/mutator/operator/to_i_to_integer"
+require_relative "evilution/mutator/operator/coercion_emptying"
+require_relative "evilution/mutator/operator/const_get_to_constant_path"
+require_relative "evilution/mutator/operator/proc_to_lambda"
+require_relative "evilution/mutator/operator/dig_to_fetch_chain"
+require_relative "evilution/mutator/operator/inequality_to_negated_identity"
+require_relative "evilution/mutator/operator/binary_operand_promotion"
+require_relative "evilution/mutator/operator/receiver_constructor_swap"
+require_relative "evilution/mutator/operator/block_body_to_nil"
+require_relative "evilution/mutator/operator/block_body_to_raise"
+require_relative "evilution/mutator/operator/block_body_promotion"
+require_relative "evilution/mutator/operator/data_struct_member"
+require_relative "evilution/mutator/operator/pin_operator_removal"
+require_relative "evilution/mutator/operator/rightward_assignment"
+require_relative "evilution/mutator/operator/numbered_parameter_swap"
+require_relative "evilution/mutator/operator/forwarded_argument_drop"
+require_relative "evilution/mutator/operator/pattern_wildcard_widening"
+require_relative "evilution/mutator/operator/no_matching_pattern_else"
+require_relative "evilution/mutator/operator/argument_order_permutation"
+require_relative "evilution/mutator/operator/keyword_value_swap"
+require_relative "evilution/mutator/operator/comparison_operand_swap"
+require_relative "evilution/mutator/operator/integer_division_to_fdiv"
+require_relative "evilution/mutator/operator/off_by_one_boundary"
+require_relative "evilution/mutator/operator/format_specifier_swap"
+require_relative "evilution/mutator/operator/exception_swallow"
+require_relative "evilution/mutator/operator/statement_reorder"
+require_relative "evilution/mutator/operator/alias_removal"
+require_relative "evilution/mutator/operator/regexp_character_type_complement"
+require_relative "evilution/mutator/operator/regexp_anchor_promotion"
+require_relative "evilution/mutator/operator/regexp_alternation_branch_deletion"
+require_relative "evilution/mutator/operator/regexp_quantifier_minimum_swap"
+require_relative "evilution/mutator/operator/regexp_capture_to_passive"
+require_relative "evilution/mutator/operator/regexp_named_group_rename"
+require_relative "evilution/mutator/operator/regexp_option_removal"
+require_relative "evilution/mutator/operator/rescue_handler_promotion"
+require_relative "evilution/mutator/operator/rescue_handler_concatenation"
+require_relative "evilution/mutator/operator/rescue_else_concatenation"
+require_relative "evilution/mutator/operator/return_keyword_removal"
 require_relative "evilution/mutator/operator/arithmetic_replacement"
 require_relative "evilution/mutator/operator/string_literal"
 require_relative "evilution/mutator/operator/array_literal"

data/script/memory_check

@@ -11,6 +11,13 @@ COMPLEX_FIXTURE = File.expand_path("../spec/support/fixtures/memory_check/target
 COMPLEX_FIXTURE_SPEC = File.expand_path("../spec/support/fixtures/memory_check/target_examples.rb", __dir__)
 ITERATIONS = Integer(ENV.fetch("MEMORY_CHECK_ITERATIONS", 50))
 MAX_GROWTH_KB = Integer(ENV.fetch("MEMORY_CHECK_MAX_GROWTH_KB", 10_240))
+# Every check that picks mutations at random draws from its own Random seeded
+# with this, so a run exercises the same mutations every time and a failure can
+# be replayed by passing the printed seed back.
+SEED = Integer(ENV.fetch("MEMORY_CHECK_SEED", 42))
+COMPLEX_POOL_SIZE = 10
+COMPLEX_ITERATIONS = Integer(ENV.fetch("MEMORY_CHECK_COMPLEX_ITERATIONS", 100))
+COMPLEX_MAX_GROWTH_KB = Integer(ENV.fetch("MEMORY_CHECK_COMPLEX_MAX_GROWTH_KB", 10_240))
 
 def setup_workload
   parser = Evilution::AST::Parser.new
@@ -21,25 +28,40 @@ def setup_workload
   mutations
 end
 
-def run_check(name, iterations: ITERATIONS, max_growth_kb: MAX_GROWTH_KB, &)
-  check = Evilution::Memory::LeakCheck.new(iterations: iterations, max_growth_kb: max_growth_kb)
+def run_check(name, iterations: ITERATIONS, max_growth_kb: MAX_GROWTH_KB, picked: nil,
+              warmup_iterations: Evilution::Memory::LeakCheck::WARMUP_ITERATIONS, &)
+  check = Evilution::Memory::LeakCheck.new(iterations: iterations, max_growth_kb: max_growth_kb,
+                                           warmup_iterations: warmup_iterations)
   result = check.run(&)
-  report(name, result)
+  report(name, result, picked)
   result[:passed]
 end
 
-def report(name, result)
+def report(name, result, picked)
   status = result[:passed] ? "PASS" : "FAIL"
   metrics = format_metrics(result)
 
   puts "[#{status}] #{name}"
-  puts "  Growth: #{metrics[:growth]} (max: #{metrics[:max]})"
+  puts "  Sustained growth: #{metrics[:sustained]} (max: #{metrics[:max]})"
+  puts "  Growth: #{metrics[:growth]}"
   puts "  Samples (MB): #{metrics[:samples]}"
+  report_picked(picked) unless result[:passed]
   puts
 end
 
+# What a failed run exercised, so it can be replayed: the seed, and how often
+# each operator came up among the mutations the check picked.
+def report_picked(picked)
+  puts "  Seed: #{SEED} (MEMORY_CHECK_SEED)"
+  return if picked.nil? || picked.empty?
+
+  tally = picked.map(&:operator_name).tally.sort_by { |operator, count| [-count, operator.to_s] }
+  puts "  Operators (#{picked.size} picks): #{tally.map { |operator, count| "#{operator} x#{count}" }.join(", ")}"
+end
+
 def format_metrics(result)
   {
+    sustained: result[:sustained_growth_mb] ? format("%.1f MB", result[:sustained_growth_mb]) : "N/A",
     growth: result[:growth_mb] ? format("%.1f MB", result[:growth_mb]) : "N/A",
     max: format("%.1f MB", result[:max_growth_kb] / 1024.0),
     samples: result[:samples].map { |s| s ? format("%.1f", s / 1024.0) : "N/A" }.join(" -> ")
@@ -53,15 +75,21 @@ stub_test_command = ->(_m) { { passed: false } }
 all_passed = true
 
 # 1. InProcess isolation
-all_passed &= run_check("InProcess isolation") do
-  mutation = mutations.sample
+random = Random.new(SEED)
+picked = []
+all_passed &= run_check("InProcess isolation", picked: picked) do
+  mutation = mutations.sample(random: random)
+  picked << mutation
   isolator = Evilution::Isolation::InProcess.new
   isolator.call(mutation: mutation, test_command: stub_test_command, timeout: 5)
 end
 
 # 2. Fork isolation
-all_passed &= run_check("Fork isolation") do
-  mutation = mutations.sample
+random = Random.new(SEED)
+picked = []
+all_passed &= run_check("Fork isolation", picked: picked) do
+  mutation = mutations.sample(random: random)
+  picked << mutation
   isolator = Evilution::Isolation::Fork.new
   isolator.call(mutation: mutation, test_command: stub_test_command, timeout: 5)
 end
@@ -113,13 +141,31 @@ complex_subjects = complex_parser.call(COMPLEX_FIXTURE)
 complex_mutations = complex_subjects.flat_map { |s| complex_registry.mutations_for(s) }
 
 integration = Evilution::Integration::RSpec.new(test_files: [COMPLEX_FIXTURE_SPEC])
-
-# Budget is generous: per-mutation require() adds the file to $LOADED_FEATURES and
-# accumulates constant-redefinition warnings. Local runs land around 20 MB; CI
-# varies up to ~30 MB depending on Ruby build and GC pressure, so we leave headroom.
-all_passed &= run_check("RSpec integration per-mutation (Config)", iterations: 20, max_growth_kb: 40_960) do
-  mutation = complex_mutations.sample
-  integration.call(mutation)
+complex_pool = complex_mutations.sample(COMPLEX_POOL_SIZE, random: Random.new(SEED))
+complex_turn = complex_pool.cycle
+
+# This check used to grow 20-41 MB over 20 mutations, which was put down to
+# per-mutation require() and redefinition warnings and given a 40 MB budget. It
+# was a leak: every run left its examples reachable from
+# RSpec::Core::AnonymousExampleGroup, about 1 MB per mutation (fixed by
+# StateGuard::AnonymousExampleGroupExamples).
+#
+# The first run of a mutation still grows the process, as the heap reaches the
+# size that mutation needs, and the growth depends on which mutation it is. So
+# the check cycles through a fixed pool: warm-up runs each mutation twice, and
+# every measured run repeats work the warm-up already did. Growth from there
+# on is retained per run.
+#
+# The budget is 10 MB over 100 runs, about 100 KB per mutation: a leak of 200 KB
+# per mutation grows about 18 MB once its largest step is taken out, and fails.
+# Measured on Linux (aarch64) over seeds 1, 7, 42 and 99, sustained growth was
+# -0.9 to 0.6 MB on Ruby 3.4.10 and 0.7 to 2.8 MB on 4.0.7. Ruby 4.0.7 creeps
+# by about 30 KB per mutation in RSS with no Ruby objects retained, which is
+# below what this check can tell apart from allocator noise.
+all_passed &= run_check("RSpec integration per-mutation (Config)",
+                        iterations: COMPLEX_ITERATIONS, max_growth_kb: COMPLEX_MAX_GROWTH_KB,
+                        warmup_iterations: COMPLEX_POOL_SIZE * 2, picked: complex_pool) do
+  integration.call(complex_turn.next)
 end
 
 puts all_passed ? "All memory checks passed." : "Some memory checks failed!"