evilution 1.2.0 → 1.4.0

data/lib/evilution/baseline.rb

@@ -2,10 +2,11 @@
 
 require_relative "spec_resolver"
 require_relative "process_cleanup"
+require_relative "diagnostic"
 
 class Evilution::Baseline
-  Result = Struct.new(:failed_spec_files, :duration) do
-    def initialize(**)
+  Result = Struct.new(:failed_spec_files, :duration, :failures) do
+    def initialize(failed_spec_files:, duration:, failures: [])
       super
       freeze
     end
@@ -13,44 +14,62 @@ class Evilution::Baseline
     def failed?
       !failed_spec_files.empty?
     end
+
+    # Every example known to have failed before any mutation ran.
+    def failed_example_ids
+      failures.flat_map(&:failed_ids).to_set
+    end
   end
 
+  # spec_selector: the object `run` resolves with (integration layout plus
+  # spec_mappings / spec_pattern), returning every spec covering a source.
+  # Without one, the bare spec_resolver is used. spec_resolver also supplies
+  # the best-guess hint for an unresolved source.
+  #
+  # fallback_to_full_suite: whether an unresolved source runs the whole
+  # fallback_dir. When false its mutations are reported unresolved, so the
+  # baseline skips it rather than running -- and often timing out on -- the
+  # entire test directory.
   def initialize(spec_resolver: Evilution::SpecResolver.new, timeout: 30, runner: nil,
-                 fallback_dir: "spec", test_files: nil)
+                 fallback_dir: "spec", test_files: nil, spec_selector: nil, fallback_to_full_suite: true)
     @spec_resolver = spec_resolver
     @timeout = timeout
     @runner = runner
     @fallback_dir = fallback_dir
     @test_files = test_files
+    @spec_selector = spec_selector
+    @fallback_to_full_suite = fallback_to_full_suite
   end
 
   def call(subjects)
     start_time = Process.clock_gettime(Process::CLOCK_MONOTONIC)
-    spec_files = baseline_spec_files(subjects)
-    failed = Set.new
-
-    spec_files.each do |spec_file|
-      failed.add(spec_file) unless run_spec_file(spec_file)
-    end
+    failures = baseline_spec_files(subjects).filter_map { |spec_file| check_spec_file(spec_file) }
 
     duration = Process.clock_gettime(Process::CLOCK_MONOTONIC) - start_time
-    Result.new(failed_spec_files: failed, duration: duration)
+    Result.new(failed_spec_files: failures.to_set(&:spec_file), duration: duration, failures: failures)
   end
 
   def run_spec_file(spec_file)
+    check_spec_file(spec_file).nil?
+  end
+
+  # nil when the spec file passed, otherwise why it did not. Anything that
+  # stops the baseline from getting an answer counts as failing, and is
+  # recorded as the reason.
+  def check_spec_file(spec_file)
     raise Evilution::Error, "no baseline runner configured" unless @runner
 
     read_io, write_io = IO.pipe
     pid = fork_spec_runner(spec_file, read_io, write_io)
     write_io.close
-    read_result(read_io, pid)
+    read_result(read_io, pid, spec_file)
   rescue Evilution::Error
     raise
-  rescue StandardError
-    false
+  rescue StandardError => e
+    warned(SpecFailure.new(spec_file: spec_file, error: "#{e.class}: #{e.message}"))
   ensure
-    read_io&.close
-    write_io&.close
+    read_io.close if read_io
+    write_io.close if write_io
   end
 
   def fork_spec_runner(spec_file, read_io, write_io)
@@ -60,27 +79,28 @@ class Evilution::Baseline
       $stdout.reopen(File::NULL, "w")
       $stderr.reopen(File::NULL, "w")
 
-      passed = runner.call(spec_file)
-      Marshal.dump({ passed: passed }, write_io)
+      report = child_report(runner, spec_file)
+      Marshal.dump(report, write_io)
       write_io.close
-      exit!(passed ? 0 : 1)
+      exit!(report[:passed] ? 0 : 1)
     end
   end
 
   GRACE_PERIOD = 0.5
 
-  def read_result(read_io, pid)
-    if read_io.wait_readable(@timeout)
-      data = read_io.read
-      Process.wait(pid)
-      return false if data.empty?
-
-      result = Marshal.load(data)
-      result[:passed]
-    else
-      terminate_child(pid)
-      false
-    end
+  def read_result(read_io, pid, spec_file)
+    return timed_out(pid, spec_file) unless read_io.wait_readable(@timeout)
+
+    data = read_io.read
+    _, status = Process.wait2(pid)
+    return warned(SpecFailure.new(spec_file: spec_file, error: unreported_error(status))) if data.empty?
+
+    report = Marshal.load(data)
+    report[:passed] ? nil : warned(SpecFailure.from_report(spec_file, report))
+  end
+
+  def warn_timeout(spec_file)
+    Evilution::Diagnostic.warn("[evilution] Baseline for #{spec_file} timed out after #{@timeout}s; treating it as failing.")
   end
 
   def terminate_child(pid)
@@ -98,11 +118,35 @@ class Evilution::Baseline
 
   private
 
-  # When --spec was provided, run those files only. Auto-discovery is skipped
-  # entirely — the user has declared what covers their subjects and any
-  # mismatch between auto-discovery and their declaration is what produced
-  # the misleading "No matching test found" warning users have reported even
-  # while passing --spec.
+  # Runs in the child. A runner that raises -- a spec helper that cannot load,
+  # a framework that is not there -- is a failure with a reason, not a child
+  # that dies without a word.
+  def child_report(runner, spec_file)
+    Report.from(runner.call(spec_file))
+  rescue StandardError, ScriptError => e
+    Report.build(passed: false, error: "#{e.class}: #{e.message}")
+  end
+
+  def timed_out(pid, spec_file)
+    terminate_child(pid)
+    warn_timeout(spec_file)
+    SpecFailure.new(spec_file: spec_file, error: "timed out after #{@timeout}s")
+  end
+
+  def unreported_error(status)
+    cause = status.signaled? ? "signal #{status.termsig}" : "exit status #{status.exitstatus}"
+    "baseline process ended without reporting (#{cause})"
+  end
+
+  def warned(failure)
+    detail = FailureFormatter.new.call(failure).map { |line| "  #{line}" }
+    Evilution::Diagnostic.warn(
+      ["[evilution] Baseline failed for #{failure.spec_file}; " \
+       "a mutation that fails only its already-failing examples will be reported neutral.", *detail].join("\n")
+    )
+    failure
+  end
+
   def baseline_spec_files(subjects)
     return Array(@test_files).uniq if @test_files && !@test_files.empty?
 
@@ -111,20 +155,35 @@ class Evilution::Baseline
 
   def resolve_unique_spec_files(subjects)
     warned = Set.new
-    subjects.map do |s|
-      resolved = @spec_resolver.call(s.file_path)
-      warn_no_matching_test(s.file_path) if resolved.nil? && warned.add?(s.file_path)
-      resolved || @fallback_dir
+    subjects.flat_map do |s|
+      specs = specs_for(s.file_path)
+      next specs unless specs.empty?
+
+      warn_no_matching_test(s.file_path) if warned.add?(s.file_path)
+      @fallback_to_full_suite ? [@fallback_dir] : []
     end.uniq
   end
 
+  def specs_for(file_path)
+    Array(@spec_selector ? @spec_selector.call(file_path) : @spec_resolver.call(file_path))
+  end
+
   def warn_no_matching_test(file_path)
+    action = @fallback_to_full_suite ? "running full suite" : "marking its mutations unresolved"
+    Evilution::Diagnostic.warn("[evilution] No matching test found for #{file_path}, #{action}. #{no_match_hint(file_path)}")
+  end
+
+  def no_match_hint(file_path)
     suggestion = @spec_resolver.suggest(file_path)
     hint = if suggestion
-             "Pass --spec #{suggestion} (best guess) or the correct test file."
+             "Pass --spec #{suggestion} (best guess) or the correct test file"
            else
-             "Use --spec to specify the test file."
+             "Use --spec to specify the test file"
            end
-    warn "[evilution] No matching test found for #{file_path}, running full suite. #{hint}"
+    @fallback_to_full_suite ? "#{hint}." : "#{hint}, or --fallback-full-suite to run the whole suite."
   end
 end
+
+require_relative "baseline/report"
+require_relative "baseline/spec_failure"
+require_relative "baseline/failure_formatter"

data/lib/evilution/child_output.rb

@@ -1,6 +1,6 @@
 # frozen_string_literal: true
 
-require_relative "../evilution"
+require_relative "version"
 
 module Evilution::ChildOutput
   module_function

data/lib/evilution/cli/command.rb

@@ -1,5 +1,6 @@
 # frozen_string_literal: true
 
+require_relative "../cli"
 require_relative "result"
 
 class Evilution::CLI::Command

data/lib/evilution/cli/commands/tests_list.rb

@@ -5,7 +5,6 @@ require_relative "../command"
 require_relative "../dispatcher"
 require_relative "../printers/tests_list"
 require_relative "../../config"
-require_relative "../../spec_resolver"
 require_relative "../../git/changed_files"
 
 class Evilution::CLI::Commands::TestsList < Evilution::CLI::Command
@@ -25,8 +24,10 @@ class Evilution::CLI::Commands::TestsList < Evilution::CLI::Command
       return 0
     end
 
-    resolver = Evilution::SpecResolver.new
-    entries = source_files.map { |source| { source: source, spec: resolver.call(source) } }
+    # The same selector `run` uses: the integration's test layout (test/*_test.rb
+    # for minitest and test-unit) plus spec_mappings and spec_pattern, so the
+    # two commands cannot disagree about which specs cover a source (GH #1596).
+    entries = source_files.map { |source| { source: source, specs: Array(config.spec_selector.call(source)) } }
     Evilution::CLI::Printers::TestsList.new(mode: :resolved, entries: entries).render(@stdout)
     0
   end

data/lib/evilution/cli/commands.rb

@@ -1,4 +1,6 @@
 # frozen_string_literal: true
 
+require_relative "../cli"
+
 module Evilution::CLI::Commands
 end

data/lib/evilution/cli/dispatcher.rb

@@ -1,5 +1,7 @@
 # frozen_string_literal: true
 
+require_relative "../cli"
+
 module Evilution::CLI::Dispatcher
   @commands = {}
 

data/lib/evilution/cli/exit_guard.rb

@@ -1,5 +1,7 @@
 # frozen_string_literal: true
 
+require_relative "../cli"
+
 # Gives evilution the final word on the process exit status.
 #
 # `--preload` loads the project's own spec helper into the parent process, and

data/lib/evilution/cli/parsed_args.rb

@@ -1,5 +1,7 @@
 # frozen_string_literal: true
 
+require_relative "../cli"
+
 class Evilution::CLI
   ParsedArgs = Struct.new(
     :command, :options, :files, :line_ranges, :stdin_error, :parse_error,

data/lib/evilution/cli/parser/command_extractor.rb

@@ -1,5 +1,7 @@
 # frozen_string_literal: true
 
+require_relative "../parser"
+
 class Evilution::CLI::Parser::CommandExtractor
   SIMPLE_COMMANDS = {
     "version" => :version,

data/lib/evilution/cli/parser/file_args.rb

@@ -1,5 +1,8 @@
 # frozen_string_literal: true
 
+require_relative "../parser"
+require_relative "../../diagnostic"
+
 module Evilution::CLI::Parser::FileArgs
   ParsedPaths = Data.define(:files, :ranges)
 
@@ -22,7 +25,7 @@ module Evilution::CLI::Parser::FileArgs
 
   def expand_spec_dir(dir)
     unless File.directory?(dir)
-      warn("Error: #{dir} is not a directory")
+      Evilution::Diagnostic.warn("Error: #{dir} is not a directory")
       return []
     end
 

data/lib/evilution/cli/parser/options_builder.rb

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 
 require "optparse"
+require_relative "../parser"
 require_relative "../../version"
 require_relative "file_args"
 
@@ -119,6 +120,8 @@ class Evilution::CLI::Parser::OptionsBuilder
                               "(default: auto-detect spec/rails_helper.rb -> spec/spec_helper.rb -> " \
                               "test/test_helper.rb for Rails projects)") { |f| @options[:preload] = f }
     opts.on("--no-preload", "Disable parent-process preload even for Rails projects") { @options[:preload] = false }
+    opts.on("--warmup NAME", "After preload, warm lazily-initialised framework state once " \
+                             "in the parent: none, rails (default: none)") { |w| @options[:warmup] = w }
   end
 
   def add_output_options(opts)
@@ -137,7 +140,7 @@ class Evilution::CLI::Parser::OptionsBuilder
 
   def add_profile_options(opts)
     opts.on("--profile NAME", "Operator profile: default, strict (default: default). " \
-                              "strict adds aggressive truthiness mutators for pre-merge audits.") { |p| @options[:profile] = p }
+                              "strict adds aggressive mutators for pre-merge audits.") { |p| @options[:profile] = p }
     opts.on("--strict", "Shortcut for --profile=strict") { @options[:profile] = "strict" }
   end
 

data/lib/evilution/cli/parser/stdin_reader.rb

@@ -1,5 +1,6 @@
 # frozen_string_literal: true
 
+require_relative "../parser"
 require_relative "file_args"
 
 class Evilution::CLI::Parser::StdinReader

data/lib/evilution/cli/parser.rb

@@ -1,5 +1,6 @@
 # frozen_string_literal: true
 
+require_relative "../cli"
 require_relative "parsed_args"
 
 class Evilution::CLI::Parser

data/lib/evilution/cli/printers/tests_list.rb

@@ -24,16 +24,17 @@ class Evilution::CLI::Printers::TestsList
     io.puts(@specs.length == 1 ? "1 spec file" : "#{@specs.length} spec files")
   end
 
+  # Each entry is `{ source:, specs: [...] }`; a source can resolve to several
+  # spec files (spec_mappings), and each gets its own line.
   def render_resolved(io)
     unique_specs = []
     @entries.each do |entry|
       source = entry[:source]
-      spec = entry[:spec]
-      if spec
+      specs = entry[:specs]
+      io.puts("  #{source}  (no spec found)") if specs.empty?
+      specs.each do |spec|
         unique_specs << spec
         io.puts("  #{spec}  (#{source})")
-      else
-        io.puts("  #{source}  (no spec found)")
       end
     end
 

data/lib/evilution/cli/printers.rb

@@ -1,4 +1,6 @@
 # frozen_string_literal: true
 
+require_relative "../cli"
+
 module Evilution::CLI::Printers
 end

data/lib/evilution/cli/result.rb

@@ -1,5 +1,7 @@
 # frozen_string_literal: true
 
+require_relative "../cli"
+
 class Evilution::CLI
   Result = Struct.new(:exit_code, :error, :error_rendered, keyword_init: true) do
     def initialize(exit_code:, error: nil, error_rendered: false)

data/lib/evilution/cli.rb

@@ -1,6 +1,11 @@
 # frozen_string_literal: true
 
 require_relative "version"
+
+# Declared before the children are required: each child requires this file
+# and nests under Evilution::CLI, so the namespace must exist first.
+class Evilution::CLI; end # rubocop:disable Lint/EmptyClass
+
 require_relative "cli/parser"
 require_relative "cli/parsed_args"
 require_relative "cli/printers/subjects"
@@ -19,6 +24,7 @@ require_relative "cli/commands/session_gc"
 require_relative "cli/commands/compare"
 require_relative "cli/commands/run"
 require_relative "cli/exit_guard"
+require_relative "diagnostic"
 
 # @api private
 class Evilution::CLI
@@ -37,14 +43,14 @@ class Evilution::CLI
     return run_subcommand_error(@parse_error) if @command == :parse_error
 
     result = Dispatcher.lookup(@command).new(@parsed, stdout: $stdout, stderr: $stderr).call
-    warn("Error: #{result.error.message}") if result.error && !result.error_rendered
+    Evilution::Diagnostic.warn("Error: #{result.error.message}") if result.error && !result.error_rendered
     result.exit_code
   end
 
   private
 
   def run_subcommand_error(message)
-    warn("Error: #{message}")
+    Evilution::Diagnostic.warn("Error: #{message}")
     2
   end
 end

data/lib/evilution/config/builders.rb

@@ -1,4 +1,6 @@
 # frozen_string_literal: true
 
+require_relative "../config"
+
 module Evilution::Config::Builders
 end

data/lib/evilution/config/env_loader.rb

@@ -1,5 +1,7 @@
 # frozen_string_literal: true
 
+require_relative "../config"
+
 module Evilution::Config::EnvLoader
   module_function
 

data/lib/evilution/config/file_loader.rb

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 
 require "yaml"
+require_relative "../config"
 
 module Evilution::Config::FileLoader
   module_function

data/lib/evilution/config/sources.rb

@@ -1,5 +1,6 @@
 # frozen_string_literal: true
 
+require_relative "../config"
 require_relative "file_loader"
 require_relative "env_loader"
 

data/lib/evilution/config/validators/example_targeting_cache.rb

@@ -1,5 +1,6 @@
 # frozen_string_literal: true
 
+require_relative "../validators"
 require_relative "base"
 
 class Evilution::Config::Validators::ExampleTargetingCache < Evilution::Config::Validators::Base

data/lib/evilution/config/validators/example_targeting_fallback.rb

@@ -1,5 +1,6 @@
 # frozen_string_literal: true
 
+require_relative "../validators"
 require_relative "base"
 
 class Evilution::Config::Validators::ExampleTargetingFallback < Evilution::Config::Validators::Base

data/lib/evilution/config/validators/example_targeting_strategy.rb

@@ -1,5 +1,6 @@
 # frozen_string_literal: true
 
+require_relative "../validators"
 require_relative "base"
 
 class Evilution::Config::Validators::ExampleTargetingStrategy < Evilution::Config::Validators::Base

data/lib/evilution/config/validators/fail_fast.rb

@@ -1,5 +1,6 @@
 # frozen_string_literal: true
 
+require_relative "../validators"
 require_relative "base"
 
 class Evilution::Config::Validators::FailFast < Evilution::Config::Validators::Base

data/lib/evilution/config/validators/hooks.rb

@@ -1,5 +1,6 @@
 # frozen_string_literal: true
 
+require_relative "../validators"
 require_relative "base"
 
 class Evilution::Config::Validators::Hooks < Evilution::Config::Validators::Base

data/lib/evilution/config/validators/ignore_patterns.rb

@@ -1,5 +1,6 @@
 # frozen_string_literal: true
 
+require_relative "../validators"
 require_relative "base"
 
 class Evilution::Config::Validators::IgnorePatterns < Evilution::Config::Validators::Base

data/lib/evilution/config/validators/integration.rb

@@ -1,5 +1,6 @@
 # frozen_string_literal: true
 
+require_relative "../validators"
 require_relative "base"
 
 class Evilution::Config::Validators::Integration < Evilution::Config::Validators::Base

data/lib/evilution/config/validators/isolation.rb

@@ -1,5 +1,6 @@
 # frozen_string_literal: true
 
+require_relative "../validators"
 require_relative "base"
 
 class Evilution::Config::Validators::Isolation < Evilution::Config::Validators::Base

data/lib/evilution/config/validators/jobs.rb

@@ -1,5 +1,6 @@
 # frozen_string_literal: true
 
+require_relative "../validators"
 require_relative "base"
 
 class Evilution::Config::Validators::Jobs < Evilution::Config::Validators::Base

data/lib/evilution/config/validators/preload.rb

@@ -1,5 +1,6 @@
 # frozen_string_literal: true
 
+require_relative "../validators"
 require_relative "base"
 
 class Evilution::Config::Validators::Preload < Evilution::Config::Validators::Base

data/lib/evilution/config/validators/profile.rb

@@ -1,5 +1,6 @@
 # frozen_string_literal: true
 
+require_relative "../validators"
 require_relative "base"
 
 class Evilution::Config::Validators::Profile < Evilution::Config::Validators::Base

data/lib/evilution/config/validators/spec_mappings.rb

@@ -1,6 +1,8 @@
 # frozen_string_literal: true
 
+require_relative "../validators"
 require_relative "base"
+require_relative "../../diagnostic"
 
 class Evilution::Config::Validators::SpecMappings < Evilution::Config::Validators::Base
   def self.call(value)
@@ -48,7 +50,7 @@ class Evilution::Config::Validators::SpecMappings < Evilution::Config::Validator
         specs.each do |spec_path|
           next if File.exist?(spec_path)
 
-          warn "[evilution] spec_mappings[#{source.inspect}]: #{spec_path} not found, skipping"
+          Evilution::Diagnostic.warn("[evilution] spec_mappings[#{source.inspect}]: #{spec_path} not found, skipping")
         end
       end
     end

data/lib/evilution/config/validators/spec_pattern.rb

@@ -1,5 +1,6 @@
 # frozen_string_literal: true
 
+require_relative "../validators"
 require_relative "base"
 
 class Evilution::Config::Validators::SpecPattern < Evilution::Config::Validators::Base

data/lib/evilution/config/validators/warmup.rb

@@ -0,0 +1,17 @@
+# frozen_string_literal: true
+
+require_relative "../validators"
+require_relative "base"
+
+# `warmup: rails` warms Rails' lazily-initialised state in the parent after
+# the preload; `none` (the default) skips it. `false` is accepted as `none`
+# so `warmup: false` reads naturally in .evilution.yml.
+class Evilution::Config::Validators::Warmup < Evilution::Config::Validators::Base
+  ALLOWED = %i[none rails].freeze
+
+  def self.call(value)
+    return :none if value == false
+
+    coerce_symbol!(value, allowed: ALLOWED, name: "warmup")
+  end
+end

data/lib/evilution/config/validators.rb

@@ -1,4 +1,6 @@
 # frozen_string_literal: true
 
+require_relative "../config"
+
 module Evilution::Config::Validators
 end