evilution 0.32.0 → 0.34.0

data/lib/evilution/spec_resolver.rb

@@ -3,6 +3,11 @@
 class Evilution::SpecResolver
   STRIPPABLE_PREFIXES = %w[lib/ app/].freeze
   CONTROLLER_PREFIX = "controllers/"
+  # Conventional test subdirectories appended to @test_dir. Real-world gems
+  # frequently park specs under spec/unit or spec/lib (test/unit, test/lib)
+  # rather than mirroring the lib/ tree 1:1 (EV-z7f5 / GH #1325).
+  CONVENTIONAL_SUBDIRS = %w[unit lib].freeze
+  MINITEST_SUFFIX = "_test.rb"
 
   def initialize(test_dir: "spec", test_suffix: "_spec.rb", request_dir: "requests")
     @test_dir = test_dir
@@ -23,8 +28,54 @@ class Evilution::SpecResolver
     Array(source_paths).filter_map { |path| call(path) }.uniq
   end
 
+  # Like #call, but returns an ARRAY of test files and additionally covers the
+  # dir-grouped layout (a source file's tests live in a directory named after
+  # the source basename, e.g. lib/x/branch.rb -> test/unit/branch/*_test.rb,
+  # rather than a single mirror file). The deterministic file mirror from #call
+  # always wins; only when no mirror file exists is the first matching grouped
+  # directory expanded into its test files (EV-bi41). Returns nil when nothing
+  # resolves.
+  def resolve_specs(source_path, spec_pattern: nil)
+    return nil if source_path.nil? || source_path.empty?
+
+    file = call(source_path, spec_pattern: spec_pattern)
+    return [file] if file
+
+    resolve_grouped_dir(source_path, spec_pattern: spec_pattern)
+  end
+
+  # Best-guess candidate for an unresolved source, found by basename glob
+  # rather than the deterministic path mirroring used by #call. Used only to
+  # enrich the "no matching test" hint (EV-z7f5 / GH #1325) — never to pick a
+  # test to run — so a fuzzy substring match is acceptable here. Returns the
+  # shallowest match, or nil when nothing resembles the basename.
+  def suggest(source_path)
+    return nil if source_path.nil? || source_path.empty?
+
+    stem = File.basename(normalize_path(source_path), ".rb")
+    return nil if stem.empty?
+
+    suggestion_globs(stem).flat_map { |glob| glob_relative(glob) }.uniq.min_by(&:length)
+  end
+
   private
 
+  # Glob for project-relative paths. Mirrors #call's project_relative_exists?
+  # contract: when run inside an isolated worker chdir'd into a sandbox, glob
+  # against PROJECT_ROOT so suggestions still find real project files. base:
+  # already yields paths relative to the root, matching the CWD-glob shape.
+  def glob_relative(glob)
+    return Dir.glob(glob) unless Evilution.in_isolated_worker?
+
+    Dir.glob(glob, base: Evilution::PROJECT_ROOT)
+  end
+
+  def suggestion_globs(stem)
+    globs = ["#{@test_dir}/**/*#{stem}*#{@test_suffix}"]
+    globs << "#{@test_dir}/**/test_#{stem}.rb" if @test_suffix == MINITEST_SUFFIX
+    globs
+  end
+
   # Existence check that succeeds against the current CWD. When the caller
   # is an isolated worker that chdir'd into a per-mutation sandbox (Evilution
   # signals this via in_isolated_worker?), also try PROJECT_ROOT so the
@@ -40,6 +91,56 @@ class Evilution::SpecResolver
     candidates.select { |path| File.fnmatch?(pattern, path, File::FNM_PATHNAME | File::FNM_EXTGLOB) }
   end
 
+  # Dir-grouped resolution: find the first candidate directory that exists and
+  # expand it into its test files. Ranked below #call's file mirrors (callers
+  # try #call first), so a 1:1 spec always wins when present.
+  def resolve_grouped_dir(source_path, spec_pattern: nil)
+    dir = directory_candidates(normalize_path(source_path)).find { |c| project_relative_dir?(c) }
+    return nil unless dir
+
+    files = test_files_in(dir)
+    files = filter_by_pattern(files, spec_pattern) if spec_pattern
+    files.empty? ? nil : files
+  end
+
+  # The grouped-directory analogue of #mirror_candidates: cross every
+  # conventional root with each layout variant, using the source basename
+  # (suffix dropped) as a DIRECTORY name rather than a test FILE name.
+  def directory_candidates(source_path)
+    stripped = strip_source_prefix(source_path)
+    mirror_variants(stripped).flat_map { |variant| grouped_dir_candidates(variant) }.uniq
+  end
+
+  def grouped_dir_candidates(variant)
+    dir, _, file = variant.rpartition("/")
+    name = file.delete_suffix(@test_suffix)
+    return [] if name.empty?
+
+    relative = dir.empty? ? name : "#{dir}/#{name}"
+    roots.map { |root| "#{root}/#{relative}" }
+  end
+
+  def strip_source_prefix(source_path)
+    base = source_path.sub(/\.rb\z/, @test_suffix)
+    prefix = STRIPPABLE_PREFIXES.find { |p| source_path.start_with?(p) }
+    prefix ? base.delete_prefix(prefix) : base
+  end
+
+  # Every test file under a grouped directory: the mirrored suffix plus, for
+  # minitest/test-unit, the `test_` prefix convention. Sorted for determinism.
+  def test_files_in(dir)
+    globs = ["#{dir}/**/*#{@test_suffix}", ("#{dir}/**/test_*.rb" if @test_suffix == MINITEST_SUFFIX)]
+    globs.compact.flat_map { |glob| glob_relative(glob) }.uniq.sort
+  end
+
+  # Directory analogue of #project_relative_exists?.
+  def project_relative_dir?(path)
+    return true if File.directory?(path)
+    return false unless Evilution.in_isolated_worker?
+
+    File.directory?(File.expand_path(path, Evilution::PROJECT_ROOT))
+  end
+
   def normalize_path(path)
     path = path.delete_prefix("./")
     if path.start_with?("/")
@@ -52,18 +153,55 @@ class Evilution::SpecResolver
   def candidate_test_paths(source_path)
     base = source_path.sub(/\.rb\z/, @test_suffix)
     prefix = STRIPPABLE_PREFIXES.find { |p| source_path.start_with?(p) }
+    stripped = prefix ? base.delete_prefix(prefix) : base
+
+    primary = mirror_candidates(stripped)
+    primary.unshift(controller_to_request_test(stripped)) if prefix
+    primary.compact!
 
-    candidates = if prefix
-                   stripped = base.delete_prefix(prefix)
-                   request_test = controller_to_request_test(stripped)
-                   [request_test, "#{@test_dir}/#{stripped}", "#{@test_dir}/#{base}"].compact
-                 else
-                   ["#{@test_dir}/#{base}"]
-                 end
+    fallbacks = primary.flat_map { |c| parent_fallback_candidates(c) }
 
-    fallbacks = candidates.flat_map { |c| parent_fallback_candidates(c) }.uniq
+    (primary + fallbacks + prefix_convention_candidates(stripped)).uniq
+  end
+
+  # Conventional roots that may hold tests: the mirrored root plus the common
+  # spec/unit, spec/lib (test/unit, test/lib) buckets.
+  def roots
+    [@test_dir, *CONVENTIONAL_SUBDIRS.map { |d| "#{@test_dir}/#{d}" }]
+  end
+
+  # Cross every conventional root with every layout variant of the stripped
+  # source path: the full mirror, the mirror with the leading gem-namespace
+  # dir dropped, and the bare basename. Full mirrors rank above dropped ones
+  # so a 1:1 layout always wins when present.
+  def mirror_candidates(stripped)
+    mirror_variants(stripped).flat_map do |variant|
+      roots.map { |root| "#{root}/#{variant}" }
+    end
+  end
 
-    candidates + fallbacks
+  def mirror_variants(stripped)
+    segments = stripped.split("/")
+    variants = [stripped]
+    variants << segments[1..].join("/") if segments.length > 1
+    variants << segments.last if segments.length > 2
+    variants.uniq
+  end
+
+  # Test::Unit / minitest gems frequently name files with a `test_` PREFIX
+  # (test/test_connection_pool.rb) instead of the mirrored `_test.rb` suffix.
+  # Only meaningful when resolving against the minitest suffix.
+  def prefix_convention_candidates(stripped)
+    return [] unless @test_suffix == MINITEST_SUFFIX
+
+    mirror_variants(stripped).flat_map do |variant|
+      dir, _, file = variant.rpartition("/")
+      name = file.delete_suffix(@test_suffix)
+      next [] if name.empty?
+
+      relative = dir.empty? ? "test_#{name}.rb" : "#{dir}/test_#{name}.rb"
+      roots.map { |root| "#{root}/#{relative}" }
+    end
   end
 
   def controller_to_request_test(stripped_path)

data/lib/evilution/spec_selector.rb

@@ -19,12 +19,23 @@ class Evilution::SpecSelector
       return existing unless existing.empty?
     end
 
-    resolved = @spec_resolver.call(source_path, spec_pattern: @spec_pattern)
-    resolved ? [resolved] : nil
+    resolved = resolve_via_resolver(source_path)
+    resolved && !resolved.empty? ? resolved : nil
   end
 
   private
 
+  # Prefer the array-returning #resolve_specs, but fall back to the older single-file #call contract so a custom
+  # resolver that only implements #call keeps working.
+  def resolve_via_resolver(source_path)
+    if @spec_resolver.respond_to?(:resolve_specs)
+      @spec_resolver.resolve_specs(source_path, spec_pattern: @spec_pattern)
+    else
+      file = @spec_resolver.call(source_path, spec_pattern: @spec_pattern)
+      file ? [file] : nil
+    end
+  end
+
   def mapping_for(source_path)
     @spec_mappings[normalize(source_path)]
   end
@@ -40,8 +51,7 @@ class Evilution::SpecSelector
     normalized.delete_prefix("./")
   end
 
-  # Same semantics as Evilution::SpecResolver#project_relative_exists? — see
-  # that method for the EV-wqxu / GH #1278 rationale.
+  # Same semantics as Evilution::SpecResolver#project_relative_exists?
   def project_relative_exists?(path)
     return true if File.exist?(path)
     return false unless Evilution.in_isolated_worker?

data/lib/evilution/version.rb

@@ -1,5 +1,5 @@
 # frozen_string_literal: true
 
 module Evilution
-  VERSION = "0.32.0"
+  VERSION = "0.34.0"
 end

data/lib/evilution.rb

@@ -100,6 +100,7 @@ require_relative "evilution/mutator/registry"
 require_relative "evilution/equivalent"
 require_relative "evilution/equivalent/heuristic"
 require_relative "evilution/equivalent/detector"
+require_relative "evilution/process_supervisor"
 require_relative "evilution/isolation"
 require_relative "evilution/isolation/fork"
 require_relative "evilution/isolation/in_process"

data/lib/tasks/stress.rake

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+
+require "rspec/core/rake_task"
+
+# RUN_STRESS lifts the default :stress exclusion in spec_helper. Set via a
+# prerequisite so it runs before the RSpec task, without polluting other tasks.
+task :stress_env do
+  ENV["RUN_STRESS"] = "1"
+end
+
+desc "Run parallel/isolation stress + load specs (tagged :stress, slow)"
+RSpec::Core::RakeTask.new(stress: :stress_env) do |t|
+  t.pattern = "spec/evilution/parallel/stress_spec.rb"
+  t.rspec_opts = "--tag stress"
+end

data/scripts/canary_manifest.yml

@@ -0,0 +1,47 @@
+# Canary manifest for the EV-7ydn validation run of scripts/compare_targeting.
+#
+#   scripts/compare_targeting scripts/canary_manifest.yml --out .artifacts/targeting_validation.md
+#
+# Each repo is run through evilution three times (full_file / lexical / coverage)
+# over the SAME mutation set; the gate is total lost_kills == 0 (coverage must
+# never lose a kill full-file caught) before the default flips to coverage.
+#
+# `dir:` assumes the EV-rxob canary checkout layout (/tmp/ev-canaries/<repo>,
+# already `bundle install`ed). Adjust to your checkout. Per-repo `args` mirror
+# the EV-rxob R2 findings: repos whose specs are NOT lib-mirrored need an
+# explicit --spec (EV-z7f5 / GH #1325), or auto spec-resolution yields 0.0 and
+# the comparison is meaningless.
+#
+# Start with the R2 infra=none tier that ran clean; extend with the DB-tier
+# repos once their services are up.
+
+repos:
+  # --- R2 PASS, lib-mirrored specs (auto-resolution works) ---
+  - name: thoughtbot/factory_bot
+    dir: /tmp/ev-canaries/factory_bot
+    args: ["lib", "--jobs", "4"]
+
+  - name: jnunemaker/httparty
+    dir: /tmp/ev-canaries/httparty
+    args: ["lib", "--jobs", "4"]
+
+  - name: rubocop/rubocop
+    dir: /tmp/ev-canaries/rubocop
+    args: ["lib", "--jobs", "4"]
+
+  - name: rack/rack
+    dir: /tmp/ev-canaries/rack
+    args: ["lib", "--jobs", "4"]
+
+  # --- Non-lib-mirrored specs: explicit --spec required (EV-z7f5 / GH #1325) ---
+  - name: bblimke/webmock
+    dir: /tmp/ev-canaries/webmock
+    args: ["lib", "--spec", "spec/unit", "--jobs", "4"]
+
+  - name: doorkeeper-gem/doorkeeper
+    dir: /tmp/ev-canaries/doorkeeper
+    args: ["lib", "--spec", "spec", "--jobs", "4"]
+
+  - name: ruby-concurrency/concurrent-ruby
+    dir: /tmp/ev-canaries/concurrent-ruby
+    args: ["lib", "--spec", "spec", "--jobs", "4"]

data/scripts/compare_targeting

@@ -0,0 +1,277 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+# Targeting-mode comparison harness.
+#
+# For each repo in a manifest, runs evilution over the SAME mutation set under
+# three example-targeting modes -- full_file (baseline), lexical (current), and
+# coverage (new) -- joins the per-mutation results, and emits a per-repo table:
+#   score_full / score_lexical / score_coverage,
+#   lost_kills  (mutations full_file KILLED but coverage did NOT -> the gate),
+#   wall_ratio_lexical / wall_ratio_coverage (vs the full_file baseline).
+#
+# The accuracy gate is lost_kills == 0: coverage targeting must never lose a kill
+# the full-file run would catch. Speed is reported, not gated.
+#
+# Usage:
+#   scripts/compare_targeting MANIFEST.yml [--out report.md]
+#
+# MANIFEST.yml:
+#   repos:
+#     - name: acme/foo
+#       dir:  /checkouts/foo        # already bundled
+#       args: ["lib/foo.rb", "--jobs", "4"]
+#
+# The actual canary execution (checkout + bundle over the EV-rxob manifest) is
+# the validation run, EV-7ydn; this harness is the reusable comparison engine.
+
+require "json"
+require "yaml"
+require "open3"
+require "optparse"
+require "digest"
+
+module CompareTargeting
+  MODES = %w[full_file lexical coverage].freeze
+  KEY_FIELDS = %w[file line operator].freeze
+
+  class ConfigError < StandardError; end
+
+  module_function
+
+  # Stable per-mutation identity across modes: only TARGETING differs between
+  # runs, so the same mutation has the same file/line/operator (+ diff to
+  # separate distinct mutations sharing a line+operator).
+  def key_for(detail)
+    base = KEY_FIELDS.map { |field| detail[field] }.join(":")
+    digest = detail["diff"].to_s
+    digest.empty? ? base : "#{base}##{Digest::SHA256.hexdigest(digest)[0, 8]}"
+  end
+
+  # The label shown in lost-kill output: file:line:operator (no diff hash).
+  def label_for(detail)
+    KEY_FIELDS.map { |field| detail[field] }.join(":")
+  end
+
+  # One mode's run, indexed by mutation key.
+  class ModeResult
+    CATEGORIES = %w[killed survived neutral equivalent unresolved unparseable timed_out errors].freeze
+
+    def self.from_json(data)
+      by_key = {}
+      CATEGORIES.each do |category|
+        Array(data[category]).each do |detail|
+          key = CompareTargeting.key_for(detail)
+          by_key[key] ||= {
+            status: detail["status"],
+            duration: (detail["duration"] || 0.0).to_f,
+            label: CompareTargeting.label_for(detail)
+          }
+        end
+      end
+      new(by_key)
+    end
+
+    def initialize(by_key)
+      @by_key = by_key
+    end
+
+    def keys
+      @by_key.keys
+    end
+
+    def status(key)
+      @by_key.dig(key, :status)
+    end
+
+    def label(key)
+      @by_key.dig(key, :label)
+    end
+
+    def killed_count
+      @by_key.count { |_, value| value[:status] == "killed" }
+    end
+
+    # Measurable = the run actually observed a verdict: killed or survived.
+    # unresolved/equivalent/errors/unparseable are excluded from the score.
+    def measurable_count
+      @by_key.count { |_, value| %w[killed survived].include?(value[:status]) }
+    end
+
+    def total_duration
+      @by_key.values.sum { |value| value[:duration] }
+    end
+  end
+
+  # Joins the three modes for one repo and derives the comparison metrics.
+  class Comparison
+    def initialize(full_file:, lexical:, coverage:)
+      @modes = { "full_file" => full_file, "lexical" => lexical, "coverage" => coverage }
+    end
+
+    def score(mode)
+      result = @modes.fetch(mode)
+      return 0.0 if result.measurable_count.zero?
+
+      result.killed_count.to_f / result.measurable_count
+    end
+
+    def wall_ratio(mode)
+      baseline = @modes.fetch("full_file").total_duration
+      return 0.0 if baseline.zero?
+
+      @modes.fetch(mode).total_duration / baseline
+    end
+
+    # Mutation labels that full_file KILLED but `mode` failed to kill -- the
+    # lost kills that must be zero before coverage can become the default.
+    def lost_kills(mode = "coverage")
+      full = @modes.fetch("full_file")
+      other = @modes.fetch(mode)
+      full.keys.select { |key| full.status(key) == "killed" && other.status(key) != "killed" }
+               .map { |key| full.label(key) }
+    end
+
+    def to_row(repo)
+      {
+        repo: repo,
+        score_full: score("full_file"),
+        score_lexical: score("lexical"),
+        score_coverage: score("coverage"),
+        lost_kills: lost_kills("coverage").size,
+        wall_ratio_lexical: wall_ratio("lexical"),
+        wall_ratio_coverage: wall_ratio("coverage")
+      }
+    end
+  end
+
+  # Renders the per-repo rows as a markdown table with a PASS/FAIL gate line.
+  class TableReporter
+    COLUMNS = %w[repo score_full score_lexical score_coverage lost_kills
+                 wall_ratio_lexical wall_ratio_coverage].freeze
+
+    def initialize(rows)
+      @rows = rows
+    end
+
+    def to_markdown
+      (table_lines + ["", gate_line]).join("\n")
+    end
+
+    private
+
+    def table_lines
+      [header_row, separator_row, *@rows.map { |row| data_row(row) }]
+    end
+
+    def header_row
+      "| #{COLUMNS.join(" | ")} |"
+    end
+
+    def separator_row
+      "| #{COLUMNS.map { "---" }.join(" | ")} |"
+    end
+
+    def data_row(row)
+      "| #{COLUMNS.map { |col| format_cell(row[col.to_sym]) }.join(" | ")} |"
+    end
+
+    def gate_line
+      total_lost = @rows.sum { |row| row[:lost_kills] }
+      gate = total_lost.zero? ? "PASS" : "FAIL"
+      "GATE (total lost_kills == 0): #{gate} (#{total_lost} lost kills across #{@rows.size} repos)"
+    end
+
+    def format_cell(value)
+      value.is_a?(Float) ? format("%.3f", value) : value.to_s
+    end
+  end
+
+  # Runs evilution for one (repo, mode) and parses the JSON into a ModeResult.
+  # command_runner is injected so the pure pipeline is testable without a real
+  # mutation run.
+  class ModeRunner
+    # Run the inner `bundle exec evilution` in the TARGET repo's bundler context.
+    # If this harness is itself launched under `bundle exec`, Bundler exports
+    # BUNDLE_GEMFILE/RUBYOPT into the child, which would make the inner bundle
+    # resolve against evilution's Gemfile instead of the target repo's. Strip
+    # that inherited bundler env first.
+    DEFAULT_RUNNER = lambda do |cmd, dir|
+      stdout, stderr, status =
+        if defined?(Bundler)
+          Bundler.with_unbundled_env { Open3.capture3(*cmd, chdir: dir) }
+        else
+          Open3.capture3(*cmd, chdir: dir)
+        end
+      raise ConfigError, "evilution failed in #{dir}: #{stderr}" unless status.success?
+
+      stdout
+    end
+
+    def initialize(command_runner: DEFAULT_RUNNER)
+      @command_runner = command_runner
+    end
+
+    def run(repo_dir:, evilution_args:, mode:)
+      cmd = ["bundle", "exec", "evilution", *evilution_args,
+             "--example-targeting", mode, "--format", "json"]
+      json = @command_runner.call(cmd, repo_dir)
+      ModeResult.from_json(JSON.parse(json))
+    end
+  end
+
+  # Drives the manifest: every repo through every mode, then the table.
+  class Harness
+    def initialize(mode_runner: ModeRunner.new)
+      @mode_runner = mode_runner
+    end
+
+    def call(repos)
+      rows = repos.map { |repo| compare_repo(repo) }
+      TableReporter.new(rows).to_markdown
+    end
+
+    def compare_repo(repo)
+      results = MODES.to_h do |mode|
+        [mode, @mode_runner.run(repo_dir: repo.fetch(:dir), evilution_args: repo.fetch(:args), mode: mode)]
+      end
+      Comparison.new(**results.transform_keys(&:to_sym)).to_row(repo.fetch(:name))
+    end
+  end
+
+  def load_manifest(path)
+    data = YAML.safe_load_file(path, symbolize_names: true)
+    raise ConfigError, "manifest must list repos:" unless data.is_a?(Hash) && data[:repos].is_a?(Array)
+
+    data[:repos]
+  rescue Errno::ENOENT
+    raise ConfigError, "manifest not found: #{path}"
+  rescue Psych::SyntaxError => e
+    raise ConfigError, "manifest is not valid YAML: #{e.message}"
+  end
+end
+
+if __FILE__ == $PROGRAM_NAME
+  out_path = nil
+  parser = OptionParser.new do |opts|
+    opts.banner = "Usage: scripts/compare_targeting MANIFEST.yml [--out report.md]"
+    opts.on("--out PATH", "Write the markdown report to PATH (default: stdout)") { |p| out_path = p }
+  end
+  parser.parse!
+
+  manifest_path = ARGV.first
+  unless manifest_path
+    warn parser.banner
+    exit 2
+  end
+
+  begin
+    repos = CompareTargeting.load_manifest(manifest_path)
+    report = CompareTargeting::Harness.new.call(repos)
+    out_path ? File.write(out_path, report) : puts(report)
+    exit 0
+  rescue CompareTargeting::ConfigError => e
+    warn "Error: #{e.message}"
+    exit 2
+  end
+end

data/scripts/compare_targeting.example.yml

@@ -0,0 +1,24 @@
+# Example manifest for scripts/compare_targeting (EV-51d4).
+#
+# Each repo is run through evilution three times -- once per example-targeting
+# mode (full_file, lexical, coverage) -- over the SAME mutation set, and the
+# harness emits a per-repo table with the lost_kills gate and wall-time ratios.
+#
+# Repos must already be checked out and `bundle install`ed; the harness only
+# runs evilution inside them. The real EV-rxob canary list is wired up in the
+# validation run (EV-7ydn).
+#
+#   scripts/compare_targeting scripts/compare_targeting.example.yml --out report.md
+
+repos:
+  - name: thoughtbot/factory_bot
+    dir: /tmp/ev-canaries/factory_bot
+    args: ["lib", "--jobs", "4"]
+
+  - name: bblimke/webmock
+    dir: /tmp/ev-canaries/webmock
+    args: ["lib", "--jobs", "4"]
+
+  - name: rack/rack
+    dir: /tmp/ev-canaries/rack
+    args: ["lib", "--jobs", "4"]