evilution 1.3.0 → 1.5.0

data/lib/evilution/runner/subject_pipeline.rb

@@ -2,18 +2,25 @@
 
 require_relative "../runner"
 require_relative "../ast/inheritance_scanner"
+require_relative "../ast/uncovered_code"
+require_relative "../diagnostic"
 require_relative "../git/changed_files"
 
 class Evilution::Runner::SubjectPipeline
-  def initialize(config, parser:)
+  autoload :Target, File.expand_path("subject_pipeline/target", __dir__)
+
+  def initialize(config, parser:, registry: Evilution::Mutator::Registry.for_profile(config.profile))
     @config = config
     @parser = parser
+    @registry = registry
+    @target = Target.parse(config.target)
   end
 
   def call
     subjects = parse_subjects
-    subjects = filter_by_descendants(subjects) if descendants_target?
-    subjects = filter_by_target(subjects) if method_target?
+    @subjects_by_file = subjects.group_by(&:file_path)
+    subjects = filter_by_descendants(subjects) if target.descendants?
+    subjects = filter_by_target(subjects) if target.method?
     subjects = filter_by_line_ranges(subjects) if config.line_ranges?
     subjects
   end
@@ -22,28 +29,61 @@ class Evilution::Runner::SubjectPipeline
     @target_files ||= resolve_target_files
   end
 
+  # Warns about the targeted lines that hold code outside every subject and
+  # that no mutation reached either. mutated_lines: the lines mutations were
+  # generated for, per file -- known only once #call's subjects are planned.
+  #
+  # A line range is checked as given; a whole file only when it has no
+  # subjects at all -- every class with an `include` or a DSL call would
+  # otherwise warn.
+  def report_uncovered_code(mutated_lines = {})
+    by_file = @subjects_by_file || {}
+    @subjects_by_file = nil
+    @uncovered_code = target_files.filter_map do |file|
+      uncovered_entry(file, by_file.fetch(file, []), mutated_lines.fetch(file, []))
+    end
+    @uncovered_code.each { |entry| warn_uncovered(entry) }
+  end
+
+  # Those lines, as `{ file:, lines: ["2-4", "9"] }` per file, once
+  # #report_uncovered_code has run.
+  def uncovered_code
+    @uncovered_code || []
+  end
+
   private
 
-  attr_reader :config, :parser
+  attr_reader :config, :parser, :registry, :target
 
+  # The parser finds every subject there is; those of a kind no operator of
+  # this run mutates are left out, so that they neither count as covering
+  # their lines nor show up with nothing generated for them.
   def parse_subjects
-    target_files.flat_map { |file| parser.call(file) }
+    target_files.flat_map { |file| parser.call(file) }.select { |subject| registry.accepts?(subject) }
   end
 
-  def source_glob_target?
-    config.target&.start_with?("source:")
+  def uncovered_entry(file, file_subjects, mutated_lines)
+    range = config.line_ranges[file]
+    return unless range || file_subjects.empty?
+
+    lines = Evilution::AST::UncoveredCode.call(file, file_subjects, lines: range, mutated_lines: mutated_lines)
+    { file: file, lines: lines.map { |run| line_label(run) } } unless lines.empty?
   end
 
-  def descendants_target?
-    config.target&.start_with?("descendants:")
+  # Spelled the way a range is passed on the command line: `2-4`, or `9`.
+  def line_label(run)
+    run.size == 1 ? run.first.to_s : "#{run.first}-#{run.last}"
   end
 
-  def method_target?
-    config.target? && !source_glob_target? && !descendants_target?
+  def warn_uncovered(entry)
+    Evilution::Diagnostic.warn(
+      "[evilution] #{entry[:file]}:#{entry[:lines].join(", ")} holds code outside every subject " \
+      "(class-body code such as DSL calls is not mutated); no mutations target those lines."
+    )
   end
 
   def resolve_source_glob
-    pattern = config.target.delete_prefix("source:")
+    pattern = target.value
     files = Dir.glob(pattern)
     raise Evilution::Error, "no files found matching '#{pattern}'" if files.empty?
 
@@ -51,10 +91,9 @@ class Evilution::Runner::SubjectPipeline
   end
 
   def filter_by_descendants(subjects)
-    base_name = config.target.delete_prefix("descendants:")
     inheritance = Evilution::AST::InheritanceScanner.call(target_files)
-    class_names = resolve_descendant_set(base_name, inheritance)
-    raise Evilution::Error, "no classes found matching '#{config.target}'" if class_names.empty?
+    class_names = resolve_descendant_set(target.value, inheritance)
+    raise Evilution::Error, "no classes found matching '#{target}'" if class_names.empty?
 
     subjects.select { |s| class_names.include?(s.name.split(/[#.]/).first) }
   end
@@ -80,14 +119,14 @@ class Evilution::Runner::SubjectPipeline
   end
 
   def filter_by_target(subjects)
-    matched = subjects.select(&target_matcher)
+    matched = subjects.select { |subject| target.selects?(subject) }
     raise Evilution::Error, build_no_match_error if matched.empty?
 
     matched
   end
 
   def resolve_target_files
-    return resolve_source_glob if source_glob_target?
+    return resolve_source_glob if target.source_glob?
     return config.target_files unless config.target_files.empty?
 
     @used_git_fallback = true
@@ -95,38 +134,13 @@ class Evilution::Runner::SubjectPipeline
   end
 
   def build_no_match_error
-    base = "no subject matched '#{config.target}'"
+    base = "no subject matched '#{target}'"
     return base unless @used_git_fallback
 
     "#{base}; scanned git-changed files only. Pass file paths or " \
       "--target source:<glob> to scan the full codebase."
   end
 
-  def target_matcher
-    target = config.target
-    return wildcard_matcher(target.chomp("*")) if target.end_with?("*")
-    return prefix_matcher(target) if target.end_with?("#", ".")
-    return exact_matcher(target) if target.include?("#") || target.include?(".")
-
-    class_matcher(target)
-  end
-
-  def wildcard_matcher(prefix)
-    ->(s) { s.name.split(/[#.]/).first.start_with?(prefix) }
-  end
-
-  def prefix_matcher(prefix)
-    ->(s) { s.name.start_with?(prefix) }
-  end
-
-  def exact_matcher(target)
-    ->(s) { s.name == target }
-  end
-
-  def class_matcher(target)
-    ->(s) { s.name.start_with?("#{target}#") || s.name.start_with?("#{target}.") }
-  end
-
   def filter_by_line_ranges(subjects)
     subjects.select do |subject|
       range = config.line_ranges[subject.file_path]

data/lib/evilution/runner.rb

@@ -38,6 +38,7 @@ class Evilution::Runner
     baseline_result = run_baseline(subjects)
 
     plan = mutation_planner.call(subjects)
+    subject_pipeline.report_uncovered_code(plan.mutated_lines)
     release_subject_nodes(subjects)
     clear_operator_caches
     execution = run_mutations(plan.enabled, baseline_result)
@@ -52,7 +53,9 @@ class Evilution::Runner
                                              disabled_mutations: plan.disabled_mutations,
                                              unresolved_target_files: target_spec_audit.call(audited_files),
                                              target_file_count: audited_files.length,
-                                             infra_retried: execution.infra_retried)
+                                             infra_retried: execution.infra_retried,
+                                             uncovered_code: subject_pipeline.uncovered_code,
+                                             baseline_failures: baseline_result ? baseline_result.failures : [])
     output_report(summary)
     save_session(summary)
 
@@ -68,7 +71,7 @@ class Evilution::Runner
   attr_reader :parser, :registry, :cache, :on_result, :hooks
 
   def subject_pipeline
-    @subject_pipeline ||= Evilution::Runner::SubjectPipeline.new(config, parser: parser)
+    @subject_pipeline ||= Evilution::Runner::SubjectPipeline.new(config, parser: parser, registry: registry)
   end
 
   def mutation_planner

data/lib/evilution/subject.rb

@@ -1,14 +1,37 @@
 # frozen_string_literal: true
 
+# What mutations are generated for and reported against. A :method subject is
+# a method definition; a :constant subject is a definition assigned to a
+# constant outside any method, such as `Point = Data.define(:x, :y)`. A :scope
+# or :aasm subject is the body of a callable handed to a class-body
+# declaration.
+#
+# A :constant_write subject is any other assignment to a constant, the whole
+# of it: `LIMIT = 10`. Applying a mutation re-evaluates the file, which
+# assigns the constant again, and evaluating the original after the tests
+# puts it back. What read the constant while the file first loaded is
+# not run again -- a class-body call (`validates length: { maximum: LIMIT }`)
+# or a constant computed from it in another file keeps the old value -- so
+# only code that reads the constant when called sees the mutant.
+#
+# A subject written in a concern's `included` or `prepended` block is marked
+# `concern_hook`: the block runs in each class that takes the concern, so a
+# mutation of it has to be run again on the classes that took it already.
 class Evilution::Subject
-  attr_reader :name, :file_path, :line_number, :source, :node
+  attr_reader :name, :file_path, :line_number, :source, :node, :kind
 
-  def initialize(name:, file_path:, line_number:, source:, node:)
+  def initialize(name:, file_path:, line_number:, source:, node:, kind: :method, concern_hook: false)
     @name = name
     @file_path = file_path
     @line_number = line_number
     @source = source
     @node = node
+    @kind = kind
+    @concern_hook = concern_hook
+  end
+
+  def concern_hook?
+    @concern_hook
   end
 
   def release_node!

data/lib/evilution/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Evilution
-  VERSION = "1.3.0"
+  VERSION = "1.5.0"
 end

data/lib/evilution.rb

@@ -14,6 +14,7 @@ require_relative "evilution/ast/inheritance_scanner"
 require_relative "evilution/ast/pattern"
 require_relative "evilution/ast/sorbet_sig_detector"
 require_relative "evilution/ast/local_reads"
+require_relative "evilution/ast/regexp_pattern"
 require_relative "evilution/ast/pattern/matcher"
 require_relative "evilution/ast/pattern/parser"
 require_relative "evilution/hooks"
@@ -26,6 +27,8 @@ require_relative "evilution/mutator/operator/comparison_replacement"
 require_relative "evilution/mutator/operator/boolean_literal_replacement"
 require_relative "evilution/mutator/operator/integer_literal"
 require_relative "evilution/mutator/operator/float_literal"
+require_relative "evilution/mutator/operator/complex_literal"
+require_relative "evilution/mutator/operator/rational_literal"
 require_relative "evilution/mutator/operator/nil_replacement"
 require_relative "evilution/mutator/operator/boolean_operator_replacement"
 require_relative "evilution/mutator/operator/boolean_operand_promotion"
@@ -53,6 +56,51 @@ require_relative "evilution/mutator/operator/receiver_constructor_swap"
 require_relative "evilution/mutator/operator/block_body_to_nil"
 require_relative "evilution/mutator/operator/block_body_to_raise"
 require_relative "evilution/mutator/operator/block_body_promotion"
+require_relative "evilution/mutator/operator/data_struct_member"
+require_relative "evilution/mutator/operator/pin_operator_removal"
+require_relative "evilution/mutator/operator/rightward_assignment"
+require_relative "evilution/mutator/operator/numbered_parameter_swap"
+require_relative "evilution/mutator/operator/forwarded_argument_drop"
+require_relative "evilution/mutator/operator/pattern_wildcard_widening"
+require_relative "evilution/mutator/operator/no_matching_pattern_else"
+require_relative "evilution/mutator/operator/argument_order_permutation"
+require_relative "evilution/mutator/operator/keyword_value_swap"
+require_relative "evilution/mutator/operator/comparison_operand_swap"
+require_relative "evilution/mutator/operator/integer_division_to_fdiv"
+require_relative "evilution/mutator/operator/round_half_mode_swap"
+require_relative "evilution/mutator/operator/off_by_one_boundary"
+require_relative "evilution/mutator/operator/format_specifier_swap"
+require_relative "evilution/mutator/operator/exception_swallow"
+require_relative "evilution/mutator/operator/statement_reorder"
+require_relative "evilution/mutator/operator/alias_removal"
+require_relative "evilution/mutator/operator/regexp_character_type_complement"
+require_relative "evilution/mutator/operator/regexp_anchor_promotion"
+require_relative "evilution/mutator/operator/regexp_alternation_branch_deletion"
+require_relative "evilution/mutator/operator/regexp_quantifier_minimum_swap"
+require_relative "evilution/mutator/operator/regexp_capture_to_passive"
+require_relative "evilution/mutator/operator/regexp_named_group_rename"
+require_relative "evilution/mutator/operator/regexp_option_removal"
+require_relative "evilution/mutator/operator/rescue_handler_promotion"
+require_relative "evilution/mutator/operator/rescue_handler_concatenation"
+require_relative "evilution/mutator/operator/rescue_else_concatenation"
+require_relative "evilution/mutator/operator/return_keyword_removal"
+require_relative "evilution/mutator/operator/memoization_break"
+require_relative "evilution/mutator/operator/short_circuit_to_eager"
+require_relative "evilution/mutator/operator/freeze_removal"
+require_relative "evilution/mutator/operator/rescue_class_widening"
+require_relative "evilution/mutator/operator/visibility_removal"
+require_relative "evilution/mutator/operator/attr_accessor_swap"
+require_relative "evilution/mutator/operator/sleep_to_zero"
+require_relative "evilution/mutator/operator/index_receiver_to_self"
+require_relative "evilution/mutator/operator/index_to_key_predicate"
+require_relative "evilution/mutator/operator/index_range_to_drop"
+require_relative "evilution/mutator/operator/index_write_to_value"
+require_relative "evilution/mutator/operator/constant_read_to_nil"
+require_relative "evilution/mutator/operator/constant_namespace_strip"
+require_relative "evilution/mutator/operator/constant_write_to_nil"
+require_relative "evilution/mutator/operator/variable_read_to_nil"
+require_relative "evilution/mutator/operator/operator_write_receiver_promotion"
+require_relative "evilution/mutator/operator/safe_navigation_insertion"
 require_relative "evilution/mutator/operator/arithmetic_replacement"
 require_relative "evilution/mutator/operator/string_literal"
 require_relative "evilution/mutator/operator/array_literal"
@@ -205,6 +253,48 @@ module Evilution
     @in_isolated_worker = previous
   end
 
+  # Marks the stretch in which the tests run against a mutation whose source has
+  # already loaded. A fork child that dies in it reports the death as the
+  # mutation's doing; one that dies earlier cannot say so.
+  #
+  # The child asks at exit, and by then the block is over in one of two ways.
+  # A stack overflow that exhausts the machine stack before the VM's own makes
+  # Ruby run neither `rescue` nor `ensure`, so the mark is still set. Which of
+  # the two stacks gives out first depends on the stack limit of the process,
+  # not on the Ruby version. Otherwise the exception unwinds through here, the
+  # mark is cleared, and the exception itself is kept to be recognised by.
+  def self.while_running_tests
+    previous = @running_tests
+    @running_tests = true
+    @tests_left_by = nil
+    yield
+  rescue Exception => e # rubocop:disable Lint/RescueException -- recorded and re-raised, never handled
+    @tests_left_by = e
+    raise
+  ensure
+    @running_tests = previous
+  end
+
+  def self.running_tests?
+    @running_tests == true
+  end
+
+  # Whether a process dying on `error` is dying of its test run.
+  def self.died_running_tests?(error)
+    running_tests? || @tests_left_by.equal?(error)
+  end
+
+  # Set by fork isolation in the child it forks for one mutation. Such a
+  # process applies its mutation, runs the tests and ends, so work that only
+  # serves a later mutation in the same process can be left out of it.
+  def self.single_mutation_process!
+    @single_mutation_process = true
+  end
+
+  def self.single_mutation_process?
+    @single_mutation_process == true
+  end
+
   # Base directory for resolving project-relative paths. An isolated worker
   # has chdir'd into a per-mutation sandbox (EV-wqxu / GH #1278), so callers
   # in that context must anchor against PROJECT_ROOT rather than Dir.pwd —

data/script/memory_check

@@ -11,6 +11,13 @@ COMPLEX_FIXTURE = File.expand_path("../spec/support/fixtures/memory_check/target
 COMPLEX_FIXTURE_SPEC = File.expand_path("../spec/support/fixtures/memory_check/target_examples.rb", __dir__)
 ITERATIONS = Integer(ENV.fetch("MEMORY_CHECK_ITERATIONS", 50))
 MAX_GROWTH_KB = Integer(ENV.fetch("MEMORY_CHECK_MAX_GROWTH_KB", 10_240))
+# Every check that picks mutations at random draws from its own Random seeded
+# with this, so a run exercises the same mutations every time and a failure can
+# be replayed by passing the printed seed back.
+SEED = Integer(ENV.fetch("MEMORY_CHECK_SEED", 42))
+COMPLEX_POOL_SIZE = 10
+COMPLEX_ITERATIONS = Integer(ENV.fetch("MEMORY_CHECK_COMPLEX_ITERATIONS", 100))
+COMPLEX_MAX_GROWTH_KB = Integer(ENV.fetch("MEMORY_CHECK_COMPLEX_MAX_GROWTH_KB", 10_240))
 
 def setup_workload
   parser = Evilution::AST::Parser.new
@@ -21,25 +28,40 @@ def setup_workload
   mutations
 end
 
-def run_check(name, iterations: ITERATIONS, max_growth_kb: MAX_GROWTH_KB, &)
-  check = Evilution::Memory::LeakCheck.new(iterations: iterations, max_growth_kb: max_growth_kb)
+def run_check(name, iterations: ITERATIONS, max_growth_kb: MAX_GROWTH_KB, picked: nil,
+              warmup_iterations: Evilution::Memory::LeakCheck::WARMUP_ITERATIONS, &)
+  check = Evilution::Memory::LeakCheck.new(iterations: iterations, max_growth_kb: max_growth_kb,
+                                           warmup_iterations: warmup_iterations)
   result = check.run(&)
-  report(name, result)
+  report(name, result, picked)
   result[:passed]
 end
 
-def report(name, result)
+def report(name, result, picked)
   status = result[:passed] ? "PASS" : "FAIL"
   metrics = format_metrics(result)
 
   puts "[#{status}] #{name}"
-  puts "  Growth: #{metrics[:growth]} (max: #{metrics[:max]})"
+  puts "  Sustained growth: #{metrics[:sustained]} (max: #{metrics[:max]})"
+  puts "  Growth: #{metrics[:growth]}"
   puts "  Samples (MB): #{metrics[:samples]}"
+  report_picked(picked) unless result[:passed]
   puts
 end
 
+# What a failed run exercised, so it can be replayed: the seed, and how often
+# each operator came up among the mutations the check picked.
+def report_picked(picked)
+  puts "  Seed: #{SEED} (MEMORY_CHECK_SEED)"
+  return if picked.nil? || picked.empty?
+
+  tally = picked.map(&:operator_name).tally.sort_by { |operator, count| [-count, operator.to_s] }
+  puts "  Operators (#{picked.size} picks): #{tally.map { |operator, count| "#{operator} x#{count}" }.join(", ")}"
+end
+
 def format_metrics(result)
   {
+    sustained: result[:sustained_growth_mb] ? format("%.1f MB", result[:sustained_growth_mb]) : "N/A",
     growth: result[:growth_mb] ? format("%.1f MB", result[:growth_mb]) : "N/A",
     max: format("%.1f MB", result[:max_growth_kb] / 1024.0),
     samples: result[:samples].map { |s| s ? format("%.1f", s / 1024.0) : "N/A" }.join(" -> ")
@@ -53,15 +75,21 @@ stub_test_command = ->(_m) { { passed: false } }
 all_passed = true
 
 # 1. InProcess isolation
-all_passed &= run_check("InProcess isolation") do
-  mutation = mutations.sample
+random = Random.new(SEED)
+picked = []
+all_passed &= run_check("InProcess isolation", picked: picked) do
+  mutation = mutations.sample(random: random)
+  picked << mutation
   isolator = Evilution::Isolation::InProcess.new
   isolator.call(mutation: mutation, test_command: stub_test_command, timeout: 5)
 end
 
 # 2. Fork isolation
-all_passed &= run_check("Fork isolation") do
-  mutation = mutations.sample
+random = Random.new(SEED)
+picked = []
+all_passed &= run_check("Fork isolation", picked: picked) do
+  mutation = mutations.sample(random: random)
+  picked << mutation
   isolator = Evilution::Isolation::Fork.new
   isolator.call(mutation: mutation, test_command: stub_test_command, timeout: 5)
 end
@@ -113,13 +141,31 @@ complex_subjects = complex_parser.call(COMPLEX_FIXTURE)
 complex_mutations = complex_subjects.flat_map { |s| complex_registry.mutations_for(s) }
 
 integration = Evilution::Integration::RSpec.new(test_files: [COMPLEX_FIXTURE_SPEC])
-
-# Budget is generous: per-mutation require() adds the file to $LOADED_FEATURES and
-# accumulates constant-redefinition warnings. Local runs land around 20 MB; CI
-# varies up to ~30 MB depending on Ruby build and GC pressure, so we leave headroom.
-all_passed &= run_check("RSpec integration per-mutation (Config)", iterations: 20, max_growth_kb: 40_960) do
-  mutation = complex_mutations.sample
-  integration.call(mutation)
+complex_pool = complex_mutations.sample(COMPLEX_POOL_SIZE, random: Random.new(SEED))
+complex_turn = complex_pool.cycle
+
+# This check used to grow 20-41 MB over 20 mutations, which was put down to
+# per-mutation require() and redefinition warnings and given a 40 MB budget. It
+# was a leak: every run left its examples reachable from
+# RSpec::Core::AnonymousExampleGroup, about 1 MB per mutation (fixed by
+# StateGuard::AnonymousExampleGroupExamples).
+#
+# The first run of a mutation still grows the process, as the heap reaches the
+# size that mutation needs, and the growth depends on which mutation it is. So
+# the check cycles through a fixed pool: warm-up runs each mutation twice, and
+# every measured run repeats work the warm-up already did. Growth from there
+# on is retained per run.
+#
+# The budget is 10 MB over 100 runs, about 100 KB per mutation: a leak of 200 KB
+# per mutation grows about 18 MB once its largest step is taken out, and fails.
+# Measured on Linux (aarch64) over seeds 1, 7, 42 and 99, sustained growth was
+# -0.9 to 0.6 MB on Ruby 3.4.10 and 0.7 to 2.8 MB on 4.0.7. Ruby 4.0.7 creeps
+# by about 30 KB per mutation in RSS with no Ruby objects retained, which is
+# below what this check can tell apart from allocator noise.
+all_passed &= run_check("RSpec integration per-mutation (Config)",
+                        iterations: COMPLEX_ITERATIONS, max_growth_kb: COMPLEX_MAX_GROWTH_KB,
+                        warmup_iterations: COMPLEX_POOL_SIZE * 2, picked: complex_pool) do
+  integration.call(complex_turn.next)
 end
 
 puts all_passed ? "All memory checks passed." : "Some memory checks failed!"