evilution 1.4.0 → 1.5.1

data/lib/evilution/integration/loading/superseded_modules.rb

@@ -0,0 +1,103 @@
+# frozen_string_literal: true
+
+require_relative "../loading"
+
+# Empties the anonymous modules an earlier evaluation of a file left behind.
+#
+# A file can install a module it never names:
+#
+#   handler = Module.new { def const_missing(name) = ... }
+#   Object.extend(handler)
+#
+# Evaluating the file again builds a new module and extends with it again, and
+# Ruby has no way to take the old one off. Under in-process isolation the file
+# is evaluated for every mutation and every restore, so the modules pile up on
+# their host, and a `super` in the mutation under test runs the code of every
+# mutation before it. Its tests then fail on a predecessor's bug, and the
+# mutation is reported killed. Fork isolation never sees this: a child holds
+# one mutation.
+#
+# A module cannot be taken off, but its methods can be removed, and method
+# lookup passes through a module that has none. So the modules created while a
+# file is evaluated are remembered, and emptied when the same file is evaluated
+# next.
+#
+# Two limits keep this to what the new evaluation really replaces. Only
+# modules are emptied, not anonymous classes, whose instances may be held
+# elsewhere. And only methods written in the file itself are removed: a module
+# some library built on the file's behalf holds methods written in the
+# library, which evaluating the file again does not define anew.
+class Evilution::Integration::Loading::SupersededModules
+  @registry = {}
+
+  class << self
+    # Created modules by file, for the process: the evaluations of one file
+    # are not all made through the same instance.
+    attr_reader :registry
+  end
+
+  MODULE_NAME = Module.instance_method(:name)
+  INSTANCE_OF = Kernel.instance_method(:instance_of?)
+  private_constant :MODULE_NAME, :INSTANCE_OF
+
+  def initialize(registry: self.class.registry)
+    @registry = registry
+  end
+
+  # Evaluates the block as an evaluation of `file_path`. The modules of the
+  # evaluation before are emptied first, so that the new one starts from what
+  # a fresh process would hold; the modules this one creates are remembered
+  # whether or not it completes.
+  #
+  # A fork child holds one mutation and ends with it, so there is nothing for
+  # it to empty later, and walking the heap of a process just forked is dear:
+  # every page touched has to be mapped first. It evaluates and no more.
+  def track(file_path)
+    return yield if Evilution.single_mutation_process?
+
+    retire(file_path)
+    known = anonymous_modules
+    yield
+  ensure
+    remember(file_path, known) if known
+  end
+
+  private
+
+  # Nothing is kept for a file whose evaluation created none, which is most
+  # files: an entry is only worth carrying when there is a module to empty.
+  def remember(file_path, known)
+    created = anonymous_modules.reject { |mod| known.key?(mod) }.keys
+    @registry[file_path] = created unless created.empty?
+  end
+
+  def retire(file_path)
+    Array(@registry.delete(file_path)).each { |mod| empty(mod, file_path) }
+  end
+
+  def empty(mod, file_path)
+    return if mod.frozen?
+
+    own = mod.instance_methods(false) + mod.private_instance_methods(false)
+    own.each { |name| mod.send(:remove_method, name) if written_in?(mod.instance_method(name), file_path) }
+  end
+
+  def written_in?(method, file_path)
+    location = method.source_location
+    !location.nil? && location.first == file_path
+  end
+
+  # No message is sent to the modules walked. ObjectSpace yields everything
+  # that is a Module, and some of it answers messages in its own way: a class
+  # may define `name`, `hash` or `==` for itself, and ActiveSupport's
+  # DeprecatedConstantProxy, a Module subclass, turns any message into a
+  # deprecation, which a project may have set to raise. Hence the identity
+  # keys and the methods bound from Module and Kernel.
+  def anonymous_modules
+    found = {}.compare_by_identity
+    ObjectSpace.each_object(Module) do |mod|
+      found[mod] = true if INSTANCE_OF.bind_call(mod, Module) && MODULE_NAME.bind_call(mod).nil?
+    end
+    found
+  end
+end

data/lib/evilution/integration/loading/syntax_validator.rb

@@ -1,13 +1,13 @@
 # frozen_string_literal: true
 
-require "prism"
 require_relative "../loading"
+require_relative "../../ast/syntax_check"
 
 class Evilution::Integration::Loading::SyntaxValidator
   ERROR_MESSAGE = "mutated source has syntax errors"
 
   def call(source)
-    return nil if Prism.parse(source).success?
+    return nil if Evilution::AST::SyntaxCheck.valid?(source)
 
     {
       passed: false,

data/lib/evilution/isolation/fork.rb

@@ -1,5 +1,6 @@
 # frozen_string_literal: true
 
+require "English"
 require "fileutils"
 require "tmpdir"
 require_relative "../memory"
@@ -55,8 +56,23 @@ class Evilution::Isolation::Fork
   # group and the TERM/KILL ladder sweeps the whole subtree on timeout (EV-2sh8
   # / GH #1330). The block keeps the marshal-pipe protocol: write a
   # length-prefixed payload, then exit with the pass/fail code.
+  #
+  # The at_exit hook covers a child that never gets that far. The normal path
+  # ends in `exit!`, which skips it, so it runs only when the child is dying.
+  #
+  # Both reports go through method objects taken before the test command runs.
+  # When the mutation under test is one of this file, the child evaluates the
+  # mutated source and the methods here are replaced; a method object keeps the
+  # definition the child started with, so the result is not written by the
+  # mutant it describes.
   def spawn_child(read_io, write_io, sandbox_dir, mutation, test_command)
     @supervisor.spawn(sandbox_dir: sandbox_dir) do
+      write = method(:write_result)
+      report_death = method(:report_death)
+      child_pid = Process.pid
+      # A process the tests fork inherits this hook and the pipe with it; its
+      # death is not the child's to report.
+      at_exit { report_death.call(write_io, $ERROR_INFO, write) if Process.pid == child_pid }
       ENV["TMPDIR"] = sandbox_dir
       # Path-relativizing mutations (e.g. File.join(dir, name) -> name) would
       # otherwise write into the parent's CWD (typically the repo root) and
@@ -67,18 +83,54 @@ class Evilution::Isolation::Fork
       # paths to Evilution::PROJECT_ROOT instead of the sandbox CWD.
       Dir.chdir(sandbox_dir)
       Evilution.in_isolated_worker!
+      Evilution.single_mutation_process!
       read_io.close
       suppress_child_output
       @hooks.fire(:worker_process_start, mutation:) if @hooks
       result = execute_in_child(mutation, test_command)
-      payload = Marshal.dump(result)
-      write_io.write([payload.bytesize].pack("N"))
-      write_io.write(payload)
-      write_io.close
+      write.call(write_io, result)
       exit!(result[:passed] ? 0 : 1)
     end
   end
 
+  def write_result(write_io, result)
+    payload = Marshal.dump(result)
+    write_io.write([payload.bytesize].pack("N"))
+    write_io.write(payload)
+    write_io.close
+  end
+
+  # Some exceptions reach no `rescue` in the child. `execute_in_child` leaves
+  # everything that is not a ScriptError or StandardError alone, and Ruby may
+  # run neither `rescue` nor `ensure` when the stack overflows through
+  # `const_missing`, which a mutation of a process-wide `const_missing` does.
+  # Without a result the parent sees only an exit status.
+  #
+  # A child that exits on purpose or on a signal keeps its status as the
+  # report: the first is its own decision, the second is usually the parent's.
+  def report_death(write_io, error, write)
+    return if error.nil? || write_io.closed?
+    return if error.is_a?(SystemExit) || error.is_a?(SignalException)
+
+    write.call(write_io, death_result(error))
+  end
+
+  # Dying while the tests run, after the mutated source loaded, is what a test
+  # that crashes does, and counts the same. Dying before that says nothing
+  # about the tests.
+  def death_result(error)
+    testing = Evilution.died_running_tests?(error)
+    phase = testing ? "while running the tests" : "before the tests ran"
+    result = {
+      passed: false,
+      error: "child died #{phase}: #{error.class}: #{error.message}",
+      error_class: error.class.name,
+      error_backtrace: Array(error.backtrace).first(5)
+    }
+    result[:test_crashed] = true if testing
+    result
+  end
+
   # The parent owns read_io/write_io (write_io is closed right after spawn so
   # read_io can see EOF), so they are closed here rather than handed to the
   # supervisor. The supervisor reaps the child and removes the sandbox dir; on
@@ -203,28 +255,48 @@ class Evilution::Isolation::Fork
     buf
   end
 
+  # What arrives is not always what the child wrote: every process the child
+  # forks inherits the write end of the pipe. A result that does not load, or
+  # loads to something other than a result, costs this mutation its verdict
+  # and nothing more.
   def decode_payload(data, handle = nil, terminated: false)
     return empty_result(handle, terminated: terminated) if data.nil? || data.empty?
 
-    { timeout: false }.merge(Marshal.load(data))
+    result = Marshal.load(data)
+    return unreadable_result(handle, "expected a Hash, got #{result.class}", terminated: terminated) unless result.is_a?(Hash)
+
+    { timeout: false }.merge(result)
+  rescue TypeError, ArgumentError => e
+    unreadable_result(handle, "#{e.class}: #{e.message.gsub(/\s+/, " ")}", terminated: terminated)
   end
 
   def empty_result(handle, terminated: false)
     { timeout: false, passed: false, error: child_death_reason(handle, terminated: terminated) }
   end
 
+  def unreadable_result(handle, detail, terminated: false)
+    ending = child_ending(handle, terminated: terminated)
+    subject = ending.nil? ? "child's result" : "#{ending}; its result"
+    { timeout: false, passed: false, error: "#{subject} was unreadable (#{detail})" }
+  end
+
   # A child that wrote no payload has usually left its stdout/stderr capture
   # empty too, so its termination status is the only evidence of what happened.
   # Reporting it separates a segfault from an OOM kill from a clean early exit,
   # all of which reach this path identically.
   def child_death_reason(handle, terminated: false)
-    return "child was terminated after the reap deadline without writing a result" if terminated
+    ending = child_ending(handle, terminated: terminated)
+    ending.nil? ? "empty result from child" : "#{ending} without writing a result"
+  end
+
+  def child_ending(handle, terminated: false)
+    return "child was terminated after the reap deadline" if terminated
 
     status = handle.nil? ? nil : handle.status
-    return "empty result from child" unless status.is_a?(Process::Status)
-    return "child exited #{status.exitstatus} without writing a result" unless status.signaled?
+    return nil unless status.is_a?(Process::Status)
+    return "child exited #{status.exitstatus}" unless status.signaled?
 
-    "child died on #{signal_name(status.termsig)} without writing a result"
+    "child died on #{signal_name(status.termsig)}"
   end
 
   def signal_name(termsig)
@@ -239,7 +311,7 @@ class Evilution::Isolation::Fork
 
   def classify_status(result)
     return :timeout if result[:timeout]
-    return :killed if result[:test_crashed]
+    return :killed if result[:test_crashed] || result[:load_crashed]
     return :unresolved if result[:unresolved]
     return :error if result[:error]
     return :survived if result[:passed]

data/lib/evilution/isolation/in_process.rb

@@ -1,5 +1,6 @@
 # frozen_string_literal: true
 
+require "English"
 require "fileutils"
 require "timeout"
 require "tmpdir"
@@ -27,12 +28,66 @@ class Evilution::Isolation::InProcess
 
   @null_out = File.open(File::NULL, "w")
   @null_err = File.open(File::NULL, "w")
+  @died = false
 
   class << self
     attr_reader :null_out, :null_err
+
+    # The mutation this process is testing, remembered so that a process dying
+    # in the middle of it can say which one it was. Ruby runs no `ensure` on
+    # some deaths (a stack overflow through `const_missing`, under the usual
+    # stack limit), so the mutation is still on record when the at_exit hook
+    # asks. The process is recorded with it: a process the tests fork inherits
+    # both the record and the hook, and its death is not this run's.
+    #
+    # The error stream is recorded as well, as it is before the mutation is
+    # tested: `$stderr` is the null stream while it is, and on such a death
+    # nothing puts it back.
+    def watch(mutation)
+      @watched = [Process.pid, mutation, $stderr]
+      return if @death_hook_installed
+
+      @death_hook_installed = true
+      at_exit { report_death($ERROR_INFO) }
+    end
+
+    def unwatch
+      @watched = nil
+      @died = false
+    end
+
+    def watched
+      pid, mutation, = @watched
+      mutation if pid == Process.pid
+    end
+
+    def report_death(error, stream: nil)
+      mutation = watched
+      return if mutation.nil? || error.nil?
+      return if error.is_a?(SystemExit) || error.is_a?(SignalException)
+
+      stream ||= @watched.last
+      stream.puts(
+        "[evilution] the run died while testing a mutation under in-process isolation: " \
+        "#{mutation} (#{error.class}: #{error.message}). " \
+        "In-process isolation cannot contain a mutation that takes the Ruby process down; " \
+        "re-run with --isolation fork."
+      )
+      stream.flush
+      @died = true
+    end
+
+    # Whether this process is ending on a death that was reported. Such a run
+    # measured nothing, which whoever decides the process status has to know:
+    # the hook that reports the death runs before the project's own, and an
+    # exit from there would skip them.
+    def died?
+      @died
+    end
   end
 
   def call(mutation:, test_command:, timeout:)
+    self.class.watch(mutation)
     start_time = Process.clock_gettime(Process::CLOCK_MONOTONIC)
     rss_before = Evilution::Memory.rss_kb
     sandbox_dir = Dir.mktmpdir("evilution-run")
@@ -43,6 +98,7 @@ class Evilution::Isolation::InProcess
 
     build_mutation_result(mutation, result, duration, rss_before, rss_after, delta)
   ensure
+    self.class.unwatch
     FileUtils.rm_rf(sandbox_dir) if sandbox_dir
   end
 
@@ -68,12 +124,31 @@ class Evilution::Isolation::InProcess
   rescue Expired
     { timeout: true }
   rescue ScriptError, StandardError => e
+    failure_result(e, e.message)
+  rescue SignalException, SystemExit
+    raise
+  rescue Exception => e # rubocop:disable Lint/RescueException -- turned into this mutation's result
+    crash_result(e)
+  end
+
+  # An exception outside StandardError that no test framework held back: a
+  # stack overflow that unwound, most likely. Left alone it would end the run.
+  # Leaving the test run that way, after the mutated source loaded, is what a
+  # test that crashes does, and counts the same.
+  def crash_result(error)
+    message = "#{error.class}: #{error.message}"
+    return failure_result(error, message) unless Evilution.died_running_tests?(error)
+
+    failure_result(error, "test run ended by #{message}").merge(test_crashed: true)
+  end
+
+  def failure_result(error, message)
     {
       timeout: false,
       passed: false,
-      error: e.message,
-      error_class: e.class.name,
-      error_backtrace: Array(e.backtrace).first(5)
+      error: message,
+      error_class: error.class.name,
+      error_backtrace: Array(error.backtrace).first(5)
     }
   end
 
@@ -97,7 +172,7 @@ class Evilution::Isolation::InProcess
 
   def classify_status(result)
     return :timeout if result[:timeout]
-    return :killed if result[:test_crashed]
+    return :killed if result[:test_crashed] || result[:load_crashed]
     return :unresolved if result[:unresolved]
     return :error if result[:error]
     return :survived if result[:passed]

data/lib/evilution/mcp/info_tool/status_glossary.rb

@@ -8,7 +8,11 @@ module Evilution::MCP::InfoTool::StatusGlossary
     {
       "status" => "killed",
       "meaning" => "A test failed when the mutation was applied — the test suite caught the mutation. " \
-                   "This is the desired outcome.",
+                   "This is the desired outcome. Also a mutation that made its own file raise while loading, " \
+                   "before any test ran, when the original file still loads (error_message starts with " \
+                   "'load failed:'), and one whose test process died on an exception no handler saw while " \
+                   "the tests ran, as a crashing test would (error_message starts with 'child died' under " \
+                   "fork isolation, 'test run ended by' under in-process isolation).",
       "counted_in_score" => true
     },
     {
@@ -27,7 +31,14 @@ module Evilution::MCP::InfoTool::StatusGlossary
     {
       "status" => "error",
       "meaning" => "Mutation execution raised an unexpected error (syntax error at load time, " \
-                   "boot failure, test-infrastructure crash). The mutation could not be evaluated.",
+                   "boot failure, test-infrastructure crash). The mutation could not be evaluated. " \
+                   "Includes a test process that died on an unhandled exception before the tests ran " \
+                   "('child died ... before the tests ran'; error_class is the exception's own class), and " \
+                   "results that are no verdict on the mutation at all: a fork child's result that " \
+                   "was unreadable, a worker's result that could not be read (the worker was replaced), " \
+                   "a child that died on a signal without writing a result (SIGABRT: the Ruby VM aborted; " \
+                   "SIGSEGV; SIGKILL, e.g. out of memory). Re-run that one mutation; if a VM abort repeats, " \
+                   "the mutant crashes the interpreter.",
       "counted_in_score" => false
     },
     {

data/lib/evilution/mcp/info_tool.rb

@@ -18,7 +18,8 @@ class Evilution::MCP::InfoTool < MCP::Tool
   tool_name "evilution-info"
   description "Discover what evilution sees before running any mutations. " \
               "One tool, five actions: " \
-              "'subjects' lists every mutatable method in the target files with its file, line, and mutation count; " \
+              "'subjects' lists every mutation subject in the target files (methods, and code outside them: " \
+              "constant assignments, scopes, AASM and callback declarations) with its file, line, and mutation count; " \
               "'tests' resolves which spec/test files cover the given sources (so you pick the right --spec before mutating); " \
               "'environment' dumps the effective config (version, ruby, config file, timeout, " \
               "integration, isolation, and every other setting); " \
@@ -36,7 +37,7 @@ class Evilution::MCP::InfoTool < MCP::Tool
         type: "string",
         enum: VALID_ACTIONS,
         description: "Which discovery operation to perform. " \
-                     "'subjects' lists mutatable methods; 'tests' resolves specs for sources; " \
+                     "'subjects' lists mutation subjects; 'tests' resolves specs for sources; " \
                      "'environment' dumps effective config; 'statuses' returns the result-status glossary; " \
                      "'feedback' returns the discussion URL and consent/privacy guidance."
       },

data/lib/evilution/mutation.rb

@@ -8,10 +8,17 @@ class Evilution::Mutation
   Slice = Data.define(:original, :mutated)
   Location = Data.define(:file_path, :line, :column)
 
-  # restore_source: the source that puts back what applying this mutation
-  # changed and re-evaluating another mutation of the file would not -- a
-  # scope declaration (see Mutator::Base#build_restore_source). nil when
-  # nothing needs restoring.
+  # What a mutation holds once its sources are released (see strip_sources!),
+  # and when it was built without a slice. Their fields read as nil, so the
+  # readers below need no nil check of their own.
+  STRIPPED_SOURCES = Sources.new(original: nil, mutated: nil)
+  NO_SLICE = Slice.new(original: nil, mutated: nil)
+  private_constant :STRIPPED_SOURCES, :NO_SLICE
+
+  # restore_source: the original file, evaluated after the mutation's tests
+  # to put back what applying it changed (see
+  # Mutator::Base#build_restore_source). nil when a mutation was built
+  # without one.
   attr_reader :subject, :operator_name, :parse_status, :location, :restore_source
 
   def initialize(subject:, operator_name:, sources:, location:,
@@ -20,7 +27,7 @@ class Evilution::Mutation
     @operator_name = operator_name
     @sources = sources
     @location = location
-    @slice = slice
+    @slice = slice || NO_SLICE
     @parse_status = parse_status
     @eval_source = eval_source
     @restore_source = restore_source
@@ -28,11 +35,11 @@ class Evilution::Mutation
   end
 
   def original_source
-    @sources&.original
+    @sources.original
   end
 
   def mutated_source
-    @sources&.mutated
+    @sources.mutated
   end
 
   # Source to feed to the load-time evaluator. Defaults to mutated_source
@@ -46,11 +53,11 @@ class Evilution::Mutation
   end
 
   def original_slice
-    @slice&.original
+    @slice.original
   end
 
   def mutated_slice
-    @slice&.mutated
+    @slice.mutated
   end
 
   def file_path
@@ -81,7 +88,7 @@ class Evilution::Mutation
 
   def strip_sources!
     diff # ensure diff is cached before clearing sources
-    @sources = nil
+    @sources = STRIPPED_SOURCES
   end
 
   def to_s
@@ -105,7 +112,7 @@ class Evilution::Mutation
   end
 
   def compute_unified_diff
-    return nil if @slice.nil?
+    return nil if @slice == NO_SLICE
 
     original_lines = @slice.original.lines
     mutated_lines = @slice.mutated.lines

data/lib/evilution/mutator/base.rb

@@ -4,6 +4,8 @@ require "prism"
 
 require_relative "../mutator"
 require_relative "primitives"
+require_relative "constant_value_subjects"
+require_relative "receiver_positions"
 require_relative "../integration/loading/body_call_neutralizer"
 require_relative "../ast/heredoc_span"
 
@@ -122,13 +124,14 @@ class Evilution::Mutator::Base < Prism::Visitor
     @body_call_neutralizer.call(surgery.source, file_path: @subject.file_path, **redeclaration(offset))
   end
 
-  # Re-evaluating any mutation of a file redefines all its methods, which is
-  # how an in-process run puts a mutated method back. Class-body calls are
-  # neutralized in that source, so a mutated scope or aasm callable would stay
-  # installed; the original file with just that declaration kept replaces it
-  # again.
+  # The original file, evaluated after a mutation's tests to put back what
+  # the mutation changed. Without it an in-process run would leave the last
+  # mutation of a file applied while the files after it are mutated, failing
+  # their tests for them. Its class-body calls are neutralized like the eval
+  # source's; a scope, aasm or callback subject keeps its own declaration, so
+  # the mutated callable is declared over again as written.
   def build_restore_source(offset)
-    return nil unless redeclared_subject?
+    return self.class.original_restore_source(@subject.file_path, @file_source, @body_call_neutralizer) unless redeclared_subject?
 
     @body_call_neutralizer.call(@file_source, file_path: @subject.file_path, **redeclaration(offset))
   end
@@ -138,8 +141,10 @@ class Evilution::Mutator::Base < Prism::Visitor
   REDECLARED_KINDS = %i[scope aasm callback].freeze
   private_constant :REDECLARED_KINDS
 
+  # So is anything written in a concern's hook block, a method included: the
+  # block has to run again on the classes that took the concern already.
   def redeclared_subject?
-    REDECLARED_KINDS.include?(@subject.kind)
+    REDECLARED_KINDS.include?(@subject.kind) || @subject.concern_hook?
   end
 
   # What the neutralizer needs to keep a redeclared subject's declaration:
@@ -213,7 +218,22 @@ class Evilution::Mutator::Base < Prism::Visitor
     tree
   end
 
+  @restore_cache = {}
+
+  # The restore source of a subject with no declaration of its own to keep is
+  # the same for every mutation of the file: neutralized once and shared.
+  def self.original_restore_source(file_path, file_source, neutralizer)
+    cache = Evilution::Mutator::Base.instance_variable_get(:@restore_cache)
+    entry = cache[file_path]
+    return entry[:source] if entry && entry[:source_hash] == file_source.hash
+
+    source = neutralizer.call(file_source, file_path: file_path).freeze
+    cache[file_path] = { source_hash: file_source.hash, source: source }
+    source
+  end
+
   def self.clear_parse_cache!
     Evilution::Mutator::Base.instance_variable_set(:@parse_cache, {})
+    Evilution::Mutator::Base.instance_variable_set(:@restore_cache, {})
   end
 end

data/lib/evilution/mutator/constant_value_subjects.rb

@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+require_relative "../mutator"
+
+# Extended by the operators that mutate a value wherever it is written, so
+# that they also run on the value of a constant assignment: a literal in
+# `LIMIT = 10` or in `DEFAULTS = { size: 1 }` is as much a literal as one in
+# a method body.
+module Evilution::Mutator::ConstantValueSubjects
+  def subject_kinds
+    super + %i[constant_write]
+  end
+end