mutineer 1.6.0 → 1.7.0

data/lib/mutineer/config.rb

@@ -1,9 +1,25 @@
 # frozen_string_literal: true
 
+require "did_you_mean"
 require "etc"
 require "yaml"
 
 module Mutineer
+  # The phrase every 1.x warning uses for behaviour that 2.0 turns into an
+  # error, so the 2.0 release can find each of those sites with one search.
+  BECOMES_ERROR_IN_2_0 = "this becomes an error in Mutineer 2.0"
+
+  # A "did you mean" hint for a misspelled name, using the stdlib spell
+  # checker that Ruby itself uses for NameError.
+  #
+  # @param word [String] the name the user wrote.
+  # @param candidates [Array<String>] the names that are valid here.
+  # @return [String] ` (did you mean "x"?)` for the closest candidate, or "" when none is close.
+  def self.did_you_mean(word, candidates)
+    match = DidYouMean::SpellChecker.new(dictionary: candidates).correct(word.to_s).first
+    match ? " (did you mean #{match.inspect}?)" : ""
+  end
+
   # Raised by the config layer instead of calling exit/abort. A data class must
   # never kill the host process. The CLI rescues this and maps it to exit 2.
   class ConfigError < StandardError; end
@@ -36,7 +52,7 @@ module Mutineer
     ConfigOption.new(field: :framework, type: :enum, yaml_key: "framework", flag: "--framework",
                      values: %w[minitest rspec]),
     ConfigOption.new(field: :verbose, type: :bool, yaml_key: "verbose", flag: "--verbose"),
-    ConfigOption.new(field: :ignore, type: :string_list, yaml_key: "ignore"),
+    ConfigOption.new(field: :ignore, type: :ignore_list, yaml_key: "ignore"),
     ConfigOption.new(field: :baseline, type: :string, yaml_key: "baseline", flag: "--baseline"),
     ConfigOption.new(field: :fail_fast, type: :bool, yaml_key: "fail_fast", flag: "--fail-fast"),
     ConfigOption.new(field: :matrix, type: :bool, yaml_key: "matrix", flag: "--matrix"),
@@ -46,6 +62,8 @@ module Mutineer
     ConfigOption.new(field: :capture_timeout, type: :positive_int, yaml_key: "capture_timeout",
                      flag: "--capture-timeout"),
     ConfigOption.new(field: :cache_dir, type: :string, yaml_key: "cache_dir", flag: "--cache-dir"),
+    ConfigOption.new(field: :allow_empty, type: :bool, yaml_key: "allow_empty", flag: "--allow-empty"),
+    ConfigOption.new(field: :coverage_from, type: :string, flag: "--coverage-from"),
     ConfigOption.new(field: :format, type: :enum, flag: "--format", values: %w[human json html]),
     ConfigOption.new(field: :strategy, type: :enum, flag: "--strategy", values: %w[reload redefine],
                      aliases: STRATEGY_ALIASES),
@@ -74,11 +92,11 @@ module Mutineer
     :sources, :tests, :operators, :threshold, :only, :dry_run,
     :cache_dir, :project_root, :load_paths,
     :jobs, :format, :output, :strategy, :require_paths,
-    :boot, :rails, :since, :framework, :verbose, :ignore,
+    :boot, :rails, :since, :framework, :verbose, :ignore, :ignore_reasons,
     # :daemon is user-facing (--daemon flag + KNOWN_KEYS + boolean coerce).
     # :daemon_timeout stays programmatic (set by tests/Runner; no flag yet).
     :baseline, :baseline_epsilon, :fail_fast, :test_command,
-    :daemon, :daemon_timeout, :timeout, :capture_timeout, :matrix,
+    :daemon, :daemon_timeout, :timeout, :capture_timeout, :matrix, :allow_empty, :coverage_from,
     keyword_init: true
   ) do
     # Config file name.
@@ -109,10 +127,12 @@ module Mutineer
       self.rails         = false if rails.nil?
       self.verbose       = false if verbose.nil?
       self.ignore        ||= []
+      self.ignore_reasons ||= {}
       self.baseline_epsilon ||= 0.0
       self.fail_fast     = false if fail_fast.nil?
       self.daemon        = false if daemon.nil?
       self.matrix        = false if matrix.nil?
+      self.allow_empty   = false if allow_empty.nil?
     end
 
     # True when the user wrote `key`, on the command line or in the config
@@ -180,12 +200,17 @@ module Mutineer
       raw.each do |key, value|
         ks = key.to_s
         unless KNOWN_KEYS.include?(ks)
-          warn "mutineer: unknown config key #{ks.inspect} in #{name} " \
-               "(known: #{KNOWN_KEYS.join(', ')}); ignored"
+          warn "mutineer: unknown config key #{ks.inspect} in #{name}#{Mutineer.did_you_mean(ks, KNOWN_KEYS)}; " \
+               "ignored, and #{BECOMES_ERROR_IN_2_0} (known: #{KNOWN_KEYS.join(', ')})"
           next
         end
         field = field_for(ks)
         parsed = parse(field, value, file: name, defer_operators: defer_operators)
+        if field == :ignore
+          out[:ignore] = parsed.fetch(:ids)
+          out[:ignore_reasons] = parsed.fetch(:reasons)
+          next
+        end
         if field == :operators
           parsed = filter_operators(parsed, name)
           if parsed.empty? && !defer_operators
@@ -220,7 +245,9 @@ module Mutineer
           config.strategy = "redefine"
         end
         unless config.daemon
-          if config.jobs.to_i > 1
+          # A typed --no-daemon already chose the serial run; only an explicit
+          # --jobs N still needs telling that --daemon is the way to get N.
+          if config.jobs.to_i > 1 && (!config.explicit?(:daemon) || config.explicit?(:jobs))
             warn "[mutineer] --rails without --daemon runs serially (shared test DB); " \
                  "forcing --jobs 1. Use --daemon for safe --jobs N."
           end
@@ -235,6 +262,24 @@ module Mutineer
       config
     end
 
+    # Mutineer 2.0 makes the daemon the default under --rails for Minitest, so a
+    # run that would change says so now. The CLI calls this after test pairing,
+    # which can still switch the framework to rspec. The runs that 2.0 keeps on this path
+    # stay quiet: RSpec, --test-command, --matrix and --fail-fast cannot use the
+    # daemon, a --dry-run runs no tests, and a written `daemon` value (true or
+    # false) is the user's choice.
+    #
+    # @api private
+    # @param config [Mutineer::Config] the resolved config.
+    # @return [void]
+    def self.warn_rails_default_change(config)
+      return unless config.rails && !config.explicit?(:daemon) && config.framework == "minitest"
+      return if config.test_command || config.matrix || config.fail_fast || config.dry_run
+
+      warn "[mutineer] in Mutineer 2.0, --rails uses --daemon by default: parallel workers (unless " \
+           "--jobs 1) and the reload strategy. Pass --no-daemon (or daemon: false) to keep this run as it is."
+    end
+
     # Pick rspec when a MAJORITY of the given test files end with _spec.rb;
     # otherwise minitest. Empty/ambiguous -> minitest (the safe default).
     #
@@ -299,6 +344,8 @@ module Mutineer
 
         prefix = file ? "#{file}: " : ""
         raise ConfigError, "#{prefix}unknown #{field} #{value.to_s.inspect}. Expected: #{opt.values.join(', ')}"
+      when :ignore_list
+        parse_ignore_list(value, file)
       when :string_list
         items = Array(value).map(&:to_s)
         # Only `operators` treats [] as "run these" rather than "use the
@@ -319,6 +366,7 @@ module Mutineer
         # A blank cache directory (an unset CI variable) would put coverage.json
         # in the project root, so it is an error, not the default.
         raise ConfigError, "#{origin} must be a directory, not blank #{got}" if field == :cache_dir && value.to_s.strip.empty?
+        raise ConfigError, "#{origin} must be a file, not blank #{got}" if field == :coverage_from && value.to_s.strip.empty?
 
         value.to_s
       when :since
@@ -350,6 +398,98 @@ module Mutineer
       f if f&.finite?
     end
 
+    # Reads an `ignore:` value into ids plus a reason for each mapping.
+    # A bare string or integer is an id with no reason. YAML reads an unquoted
+    # all-digit id as an integer, and the id text is those decimal digits.
+    # A mapping may hold `id` and `reason` only. A mapping with no id, or with
+    # any other key, is dropped and warned: 2.0 will reject it. Callers that
+    # match ids keep the id list.
+    #
+    # @api private
+    # @param value [Object] the raw YAML value.
+    # @param file [String, nil] config file name.
+    # @return [Hash] `:ids` is an array of id strings. `:reasons` maps an id to its text.
+    def self.parse_ignore_list(value, file)
+      label = file || CONFIG_FILE
+      case value
+      when nil then return { ids: [], reasons: {} }
+      when String, Integer then return { ids: [ignore_id_text(value)], reasons: {} }
+      when Array then items = value
+      when Hash then items = [value]
+      else
+        warn "mutineer: ignore in #{label} must be a list; ignored, and #{BECOMES_ERROR_IN_2_0}"
+        return { ids: [], reasons: {} }
+      end
+
+      ids = []
+      reasons = {}
+      items.each_with_index do |item, index|
+        number = index + 1
+        id = ignore_id_text(item)
+        if id
+          ids << id unless id.empty? || ids.include?(id)
+        elsif item.is_a?(Hash)
+          mapped, reason = ignore_mapping(item, number, label)
+          next if mapped.nil?
+
+          ids << mapped unless ids.include?(mapped)
+          reasons[mapped] = reason if reason
+        else
+          warn "mutineer: ignore entry #{number} in #{label} is not an id or a mapping; " \
+               "ignored, and #{BECOMES_ERROR_IN_2_0}"
+        end
+      end
+      { ids: ids, reasons: reasons }
+    end
+
+    # Text of one ignore id. An integer is the decimal digits YAML kept.
+    # A leading zero is already gone by then, so quote an id that needs one.
+    # Any other type is not an id.
+    #
+    # @api private
+    # @param value [Object] a list item, a scalar ignore value, or a mapping id.
+    # @return [String, nil] the id text, or nil when the value cannot be an id.
+    def self.ignore_id_text(value)
+      case value
+      when String then value
+      when Integer then value.to_s
+      end
+    end
+
+    # One `ignore:` mapping. Returns the id and reason, or nil when the entry
+    # is not usable.
+    #
+    # @api private
+    # @param item [Hash] the mapping.
+    # @param number [Integer] 1-based index in the list.
+    # @param label [String] config file name.
+    # @return [Array(String, String), Array(nil, nil)]
+    def self.ignore_mapping(item, number, label)
+      unknown = item.keys.map(&:to_s) - %w[id reason]
+      unless unknown.empty?
+        names = unknown.map(&:inspect).join(", ")
+        warn "mutineer: ignore entry #{number} in #{label} has unknown key #{names}; " \
+             "ignored, and #{BECOMES_ERROR_IN_2_0}"
+        return [nil, nil]
+      end
+
+      id = ignore_id_text(item["id"] || item[:id])
+      unless id.is_a?(String) && !id.strip.empty?
+        warn "mutineer: ignore entry #{number} in #{label} has no id; ignored, and #{BECOMES_ERROR_IN_2_0}"
+        return [nil, nil]
+      end
+
+      reason = item.key?("reason") ? item["reason"] : item[:reason]
+      if !reason.nil? && !reason.is_a?(String)
+        warn "mutineer: ignore entry #{number} in #{label} reason must be text; " \
+             "ignored, and #{BECOMES_ERROR_IN_2_0}"
+        return [nil, nil]
+      end
+
+      text = reason.to_s.strip
+      [id, text.empty? ? nil : text]
+    end
+
     # Drop (with a warning) operator names the registry does not know.
     # Referenced lazily so config.rb carries no load-order dependency on the
     # registry; by the time a config is parsed at runtime, it is loaded.
@@ -363,8 +503,8 @@ module Mutineer
       names.select do |n|
         next true if known.include?(n)
 
-        warn "mutineer: unknown operator #{n.inspect} in #{file_name} " \
-             "(known: #{known.join(', ')}); ignored"
+        warn "mutineer: unknown operator #{n.inspect} in #{file_name}#{Mutineer.did_you_mean(n, known)}; " \
+             "ignored, and #{BECOMES_ERROR_IN_2_0} (known: #{known.join(', ')})"
         false
       end
     end

data/lib/mutineer/coverage_map.rb

@@ -10,10 +10,14 @@ require_relative "minitest_integration"
 require_relative "test_runners"
 require_relative "child_stdout"
 require_relative "orphan_guard"
+require_relative "redaction"
 require_relative "project_path"
 require_relative "pairing"
 
 module Mutineer
+  # A `--coverage-from` capture that cannot be read, or does not fit this run.
+  class CoverageFromError < StandardError; end
+
   # Maps `(source_file, line) -> [test_files]` so each mutant runs only against
   # the tests that actually exercise its line. Built once, then queried per
   # mutant via #tests_for. Persisted to .mutineer/coverage.json with a
@@ -90,11 +94,12 @@ module Mutineer
     def initialize(source_paths:, test_paths:, cache_dir: ".mutineer",
                    load_paths: ["lib"], project_root: Dir.pwd,
                    capture_timeout: DEFAULT_CAPTURE_TIMEOUT, boot_path: nil,
-                   framework: "minitest", verbose: false, require_paths: [])
+                   framework: "minitest", verbose: false, require_paths: [], coverage_from: nil)
       @source_paths = Array(source_paths)
       @require_paths = Array(require_paths)
       @test_paths   = Array(test_paths)
       @cache_dir    = cache_dir
+      @coverage_from = coverage_from
       @load_paths   = Array(load_paths)
       @project_root = project_root
       @capture_timeout = capture_timeout
@@ -367,13 +372,20 @@ module Mutineer
     # @return [Mutineer::CoverageMap] self.
     def cached_or(after_fork: nil)
       @digest = compute_digest
+      return reuse_capture(after_fork) if @coverage_from
+
       cached = read_cache
       # A cache from before load methods (#209) or test timings (#203) were
-      # saved rebuilds once. Boot mode reads its load lines and methods from
-      # the live boot instead (#build_via_fork), but its map from before #209
-      # lacks the `def` lines of the methods each test called.
+      # saved rebuilds once. A cache that does not record this process's
+      # PARALLEL_WORKERS value rebuilds too. Rails reads that value inside
+      # parallelize, so a map from another worker count selects the wrong
+      # tests. The daemon writes this same file. Boot mode reads its load
+      # lines and methods from the live boot instead (#build_via_fork), but
+      # its map from before #209 lacks the `def` lines of the methods each
+      # test called.
       if cached && cached["digest"] == @digest && dependencies_match?(cached) &&
-         cached.key?("load_methods") && cached.key?("timings")
+         cached.key?("load_methods") && cached.key?("timings") &&
+         parallel_workers_match?(cached)
         @map = cached["map"] || {}
         @timings = cached["timings"] || {}
         unless @boot_path
@@ -397,6 +409,137 @@ module Mutineer
       self
     end
 
+    # Takes the map from the `--coverage-from` capture instead of capturing,
+    # keeping only this run's sources, then runs the tests clean. A mutant runs
+    # only the tests that cover its line, often one file alone, so each test
+    # that covers one of this run's sources runs alone, as on a cache hit: a
+    # test that fails alone here would otherwise kill every mutant it runs. The
+    # other tests never run alone in this run, so only the combined check
+    # covers them.
+    #
+    # @api private
+    # @param after_fork [Proc, nil] boot-mode fork hook.
+    # @return [Mutineer::CoverageMap] self.
+    # @raise [Mutineer::CoverageFromError] when the capture does not fit.
+    def reuse_capture(after_fork)
+      cached = accepted_capture
+      sources = source_fingerprints.keys.to_set
+      @map = cached["map"].select { |key, _| sources.include?(key_source(key)) }
+      @timings = cached["timings"] || {}
+      @failed_test_files = []
+      @failed_clean_tests = []
+      @loaded_dependencies = cached["dependencies"]
+      covering = @map.values.flatten.to_set
+      verify_cached_clean(after_fork: after_fork,
+                          test_paths: @test_paths.select { |t| covering.include?(relativize(t)) })
+      verify_combined_clean(after_fork: after_fork)
+      save
+      self
+    end
+
+    # The parsed `--coverage-from` capture, once it fits this run.
+    #
+    # @api private
+    # @return [Hash] the parsed coverage.json.
+    # @raise [Mutineer::CoverageFromError] when it cannot be read or does not fit.
+    def accepted_capture
+      if File.expand_path(@coverage_from) == File.expand_path(cache_path)
+        raise CoverageFromError, "--coverage-from #{@coverage_from} is this run's own cache; give the run another --cache-dir"
+      end
+
+      cached = JSON.parse(File.read(@coverage_from))
+      reason = capture_mismatch(cached)
+      raise CoverageFromError, "--coverage-from #{@coverage_from} cannot be reused: #{reason}" if reason
+
+      cached
+    rescue SystemCallError, JSON::ParserError => e
+      raise CoverageFromError, "--coverage-from #{@coverage_from} cannot be read: #{e.message}"
+    end
+
+    # Why a capture does not fit this run, or nil when it does.
+    #
+    # @api private
+    # @param cached [Object] the parsed coverage.json.
+    # @return [String, nil]
+    def capture_mismatch(cached)
+      return "it is not a coverage cache" unless cached.is_a?(Hash) && cached["map"].is_a?(Hash)
+      unless cached["sources"] && cached["owners"] && cached["inputs"]
+        return "it has no source fingerprints; capture it again with this version and --rails/--boot"
+      end
+      return "its entries are not the shape a coverage cache saves; capture it again" unless well_formed_capture?(cached)
+      return "its tests, --require files, boot file, load paths or framework differ" unless cached["inputs"] == compute_digest(inputs_only: true)
+
+      fingerprints = source_fingerprints
+      missing = fingerprints.keys - cached["sources"].keys
+      return "it was not captured over #{missing.join(', ')}" if missing.any?
+
+      changed = fingerprints.reject { |rel, fingerprint| cached["sources"][rel] == fingerprint }.keys
+      return "#{changed.join(', ')} changed since the capture" if changed.any?
+
+      owners = ownership_paths - cached["owners"]
+      return "#{owners.join(', ')} appeared since the capture and can own a split test" if owners.any?
+
+      # The capture can list owners of sources outside this run, so compare
+      # with the disk, not with this run's owners.
+      gone = cached["owners"].reject { |rel| File.file?(absolute(rel)) }
+      return "#{gone.join(', ')} disappeared since the capture and could own a split test" if gone.any?
+
+      return "a file its tests loaded changed since the capture" unless dependencies_match?(cached)
+      unless parallel_workers_match?(cached)
+        return "it was captured with PARALLEL_WORKERS=#{cached['parallel_workers'].inspect}, " \
+               "this run has #{ENV.fetch('PARALLEL_WORKERS', nil).inspect}"
+      end
+
+      failed = Array(cached["failed_test_files"])
+      return "capture failed for #{failed.join(', ')}" if failed.any?
+
+      moved = load_mismatch(cached, fingerprints.keys.to_set)
+      return nil if moved.empty?
+
+      "lines of #{moved.join(', ')} ran at load in only one of the capture and this run: loading the capture's " \
+        "other sources ran them. Under --strategy redefine, eager-load in both (config.eager_load = true)"
+    end
+
+    # True when every entry the reuse reads has the type {#save} writes, so a
+    # hand-edited or foreign file is rejected with a reason, not a backtrace.
+    #
+    # @api private
+    # @param cached [Hash] the parsed coverage.json.
+    # @return [Boolean]
+    def well_formed_capture?(cached)
+      strings = ->(value) { value.is_a?(Array) && value.all?(String) }
+      cached["inputs"].is_a?(String) && strings.(cached["owners"]) &&
+        cached["sources"].is_a?(Hash) && cached["sources"].all? { |k, v| k.is_a?(String) && v.is_a?(String) } &&
+        cached["map"].all? { |k, v| k.is_a?(String) && strings.(v) } &&
+        %w[load_lines load_methods].all? { |key| cached[key].nil? || strings.(cached[key]) } &&
+        (cached["timings"].nil? || cached["timings"].is_a?(Hash))
+    end
+
+    # This run's sources whose load lines or load methods differ in the capture.
+    #
+    # @api private
+    # @param cached [Hash] the parsed coverage.json.
+    # @param sources [Set<String>] project-relative source paths.
+    # @return [Array<String>] the sources that differ, sorted.
+    def load_mismatch(cached, sources)
+      owned = ->(keys) { Array(keys).select { |key| sources.include?(key_source(key)) }.to_set }
+      moved = (owned.(cached["load_lines"]) ^ owned.(@load_lines)) + (owned.(cached["load_methods"]) ^ owned.(@load_methods))
+      moved.map { |key| key_source(key) }.uniq.sort
+    end
+
+    # Fingerprint of each source, by project-relative path.
+    #
+    # @return [Hash{String => String}]
+    def source_fingerprints
+      @source_paths.to_h { |p| [relativize(absolute(p)), file_fingerprint(absolute(p))] }
+    end
+
+    # The source path of a "file:line" or "file:line:column" key.
+    #
+    # @param key [String] a map, load line or load method key.
+    # @return [String]
+    def key_source(key) = key.sub(/(?::\d+)+\z/, "")
+
     # Runs standalone coverage capture.
     #
     # @api private
@@ -578,7 +721,8 @@ module Mutineer
     def fail_test(test_path, reason)
       rel = relativize(test_path)
       @failed_test_files << rel
-      warn "[mutineer] coverage skipped for #{rel}: #{reason}"
+      # The reason can quote an app error, and so a credentialed DATABASE_URL.
+      warn "[mutineer] coverage skipped for #{rel}: #{Redaction.text(reason)}"
       nil
     end
 
@@ -623,9 +767,10 @@ module Mutineer
     #
     # @api private
     # @param after_fork [Proc, nil] boot-mode fork hook.
+    # @param test_paths [Array<String>] the tests to run, each alone.
     # @return [void]
-    def verify_cached_clean(after_fork: nil)
-      @test_paths.each do |test_path|
+    def verify_cached_clean(after_fork: nil, test_paths: @test_paths)
+      test_paths.each do |test_path|
         rel = relativize(test_path)
         next if @failed_test_files.include?(rel)
 
@@ -1178,18 +1323,32 @@ module Mutineer
       end
     end
 
+    # True when `cached` records the same +PARALLEL_WORKERS+ value this process
+    # has now. Rails reads that variable inside +parallelize+ and can run one
+    # test file in several workers. A missing key is a map saved before this
+    # value was stored. In-process runs and the daemon write this same file.
+    #
+    # @api private
+    # @param cached [Hash] parsed coverage.json.
+    # @return [Boolean]
+    def parallel_workers_match?(cached)
+      cached.key?("parallel_workers") && cached["parallel_workers"] == ENV.fetch("PARALLEL_WORKERS", nil)
+    end
+
     # Digest each file's ROLE + relative path + content length + content, plus
     # the load_paths. Without role/path/length delimiters the digest collides
     # (("ab","c") == ("a","bc")) and is blind to source/test role swaps, silently
     # accepting a stale cached map.
-    def compute_digest
+    #
+    # @param inputs_only [Boolean] leave out the sources and their owners.
+    def compute_digest(inputs_only: false)
       d = Digest::SHA256.new
-      digest_group(d, "source", @source_paths)
+      digest_group(d, "source", @source_paths) unless inputs_only
       digest_group(d, "test", @test_paths)
       # One group per file: digest_group sorts, and the load order matters.
       @require_paths.each { |p| digest_group(d, "require", [digest_path(p)]) }
       digest_group(d, "boot", [digest_path(@boot_path)]) if @boot_path
-      ownership_paths.each do |rel|
+      (inputs_only ? [] : ownership_paths).each do |rel|
         d.update("owner\0")
         d.update(rel)
         d.update("\0")
@@ -1305,7 +1464,11 @@ module Mutineer
       data = { "digest" => @digest, "failed_test_files" => @failed_test_files,
                "dependencies" => @loaded_dependencies, "map" => @map,
                "load_lines" => @load_lines.to_a.sort, "load_methods" => @load_methods.to_a.sort,
-               "timings" => @timings }
+               "timings" => @timings, "parallel_workers" => ENV.fetch("PARALLEL_WORKERS", nil) }
+      if @boot_path
+        data.merge!("inputs" => compute_digest(inputs_only: true), "sources" => source_fingerprints,
+                    "owners" => ownership_paths)
+      end
       tmp = "#{cache_path}.tmp"
       File.write(tmp, JSON.generate(data))
       File.rename(tmp, cache_path) # atomic swap