mutineer 1.4.0 → 1.5.0

data/lib/mutineer/changed_lines.rb

@@ -59,6 +59,8 @@ module Mutineer
     # A file that is not in the index has no usable diff, so `new_file_diff`
     # supplies one that marks every line new. That includes a file the ref still
     # has, deleted and then written again: git prints only a deletion hunk for it.
+    # A Git-ignored file outside the index contributes no changed lines, with a
+    # warning, so the skip does not look like a file with no changes.
     # Paths are literal, so a name such as `file[1].rb` is not read as a glob.
     #
     # A failure is warned, never silent: an empty result means "no changed
@@ -73,7 +75,13 @@ module Mutineer
       out, _err, status = Open3.capture3(
         "git", "--literal-pathspecs", "-C", project_root, "diff", "--unified=0", ref, "--", abs_file
       )
-      return(indexed?(abs_file, project_root) ? out : new_file_diff(abs_file)) if status.success?
+      if status.success?
+        return out if indexed?(abs_file, project_root)
+        return new_file_diff(abs_file) unless ignored?(abs_file, project_root)
+
+        warn "[mutineer] #{abs_file} is ignored by Git; its lines will not be mutated (--since)"
+        return ""
+      end
 
       warn "[mutineer] git diff failed for #{abs_file}; its lines will not be mutated (--since)"
       ""
@@ -95,6 +103,21 @@ module Mutineer
       known.success?
     end
 
+    # Tells whether Git ignores the file (`.gitignore`, `.git/info/exclude` or
+    # the global excludes file).
+    #
+    # @param abs_file [String] absolute path of the file.
+    # @param project_root [String] repository root for `git -C`.
+    # @return [Boolean] true when Git ignores the file. False when Git cannot
+    #   tell, such as for a path beyond a symlink or inside a submodule (exit
+    #   128): that file is scored in full, as a file Git does not ignore is.
+    def ignored?(abs_file, project_root)
+      _out, _err, status = Open3.capture3(
+        "git", "-C", project_root, "check-ignore", "--quiet", "--", abs_file
+      )
+      status.success?
+    end
+
     # Returns a diff that marks a file as entirely new. It returns `""` when
     # the file is empty or cannot be read (after a warning). A file outside the
     # index has every line new; read as "unchanged", `--since` would score

data/lib/mutineer/cli.rb

@@ -10,6 +10,7 @@ require_relative "pairing"
 require_relative "changed_lines"
 require_relative "runner"
 require_relative "reporter"
+require_relative "kill_matrix"
 require_relative "baseline"
 require_relative "mutator_registry"
 
@@ -28,11 +29,12 @@ module Mutineer
       Usage: mutineer [options] <command> [args]
 
       Commands:
-        run [options] <source...> --test <test...>   Mutate, run, and report
+        run [options] <source...> --test <test> [--test <test>...]
+                                                     Mutate, run, and report
         run --dry-run [options] <source...>          Print candidate mutations only
 
       Run options:
-        --test FILE          Test file covering the sources (repeatable)
+        --test FILE          Test file covering the sources (one per flag; repeat it)
         --operators LIST     Comma-separated operator names (default: Tier 1 set)
         --threshold FLOAT    Fail (exit 1) when score < FLOAT (default: 0 = off)
         --baseline FILE      Fail (exit 1) on NEW survivors / score drop vs a prior
@@ -41,12 +43,22 @@ module Mutineer
         --only NAME          Restrict to one fully-qualified subject
         --since REF          Only mutate lines changed since git REF (e.g. origin/main)
         --no-since           Disable diff scoping (a typed no beats a .mutineer.yml since:)
-        --jobs N             Parallel worker count (default: processor count)
+        --jobs N             Parallel worker count (default: processor count);
+                             --test-command, --fail-fast, or --rails
+                             without --daemon forces 1
         --strategy NAME      reload (whole-file) or redefine (surgical); default: reload
+        --timeout SECONDS    Per-mutant time limit for in-process runs (default: 10;
+                             not with --daemon or --test-command); a mutant over it
+                             is a timeout, excluded from the score
+        --capture-timeout SECONDS  Time limit for each coverage-capture subprocess
+                             and the clean check (default: 120; not with --test-command)
+        --cache-dir DIR      Directory for the coverage cache (default: .mutineer;
+                             not with --test-command)
         --framework NAME     minitest or rspec (default: auto-detect from --test names)
         --boot FILE          Require FILE once in the parent to boot the app env, then
                              fork per mutant (Rails apps; requires --test)
-        --rails              Sugar for --boot config/environment --strategy redefine
+        --rails              Boot config/environment; without --daemon, defaults to
+                             redefine and serial execution
         --test-command CMD   Run the target suite in the app's own runtime as a
                              subprocess (for apps on Ruby < 3.4). CMD must contain
                              %{files}. Scrubs Mutineer Ruby PATH pins; set RAILS_ENV
@@ -57,7 +69,13 @@ module Mutineer
         --format human|json|html  Report format (default: human)
         --output FILE        Write the report to FILE instead of stdout
         --dry-run            List mutations without executing
-        --fail-fast          Stop at the first surviving mutant
+        --fail-fast          Stop at the first surviving mutant (--no-fail-fast beats a
+                             .mutineer.yml fail_fast:)
+        --matrix             Run every covering test for each mutant and report blind and
+                             redundant tests; the JSON report also lists each mutant's
+                             killers (in-process only; not with --daemon, --test-command,
+                             --fail-fast or --dry-run; --no-matrix beats a .mutineer.yml
+                             matrix:)
         --verbose            Surface the real error when a fork capture fails (alias: --debug)
 
       Options:
@@ -88,7 +106,10 @@ module Mutineer
         end
         o.on("--list-operators") { show_operators = true }
         o.on("--dry-run") { opts[:dry_run] = true }
-        o.on("--fail-fast") { opts[:fail_fast] = true }
+        # A typed --no-* beats the .mutineer.yml key, so a file key that
+        # conflicts with another flag can be turned off for one run.
+        o.on("--[no-]fail-fast") { |on| opts[:fail_fast] = on }
+        o.on("--[no-]matrix") { |on| opts[:matrix] = on }
         o.on("--only NAME") { |v| opts[:only] = v }
         o.on("--since REF") { |v| opts[:since] = Config.parse(:since, v) }
         # A typed "no" must beat a .mutineer.yml `since:` key: the key is present
@@ -100,6 +121,9 @@ module Mutineer
         end
         o.on("--threshold FLOAT") { |v| opts[:threshold] = Config.parse(:threshold, v) }
         o.on("--jobs N") { |v| opts[:jobs] = Config.parse(:jobs, v) }
+        o.on("--timeout SECONDS") { |v| opts[:timeout] = Config.parse(:timeout, v) }
+        o.on("--capture-timeout SECONDS") { |v| opts[:capture_timeout] = Config.parse(:capture_timeout, v) }
+        o.on("--cache-dir DIR") { |v| opts[:cache_dir] = Config.parse(:cache_dir, v) }
         o.on("--strategy STRAT") { |v| opts[:strategy] = Config.parse(:strategy, v) }
         o.on("--framework NAME") { |v| opts[:framework] = Config.parse(:framework, v) }
         o.on("--boot FILE") { |v| opts[:boot] = v }
@@ -138,7 +162,7 @@ module Mutineer
 
       begin
         file_path = Config.find_file
-        file_hash = file_path ? Config.from_file(file_path) : {}
+        file_hash = file_path ? Config.from_file(file_path, defer_operators: opts.key?(:operators)) : {}
         config = Config.resolve(opts, file_hash)
       rescue Mutineer::ConfigError => e
         # The lib layer raises instead of killing the host; the CLI maps a
@@ -150,8 +174,14 @@ module Mutineer
       case argv.first
       when "run"
         warn_config_root_mismatch(file_path, config.project_root) if file_path
-        # A directory source expands to its **/*.rb files; literal files pass
-        # through. Test inference (when --test is omitted) happens in validate!.
+        tests_as_sources = argv[1..].grep(%r{(\A|/)(test|spec)/.*_(test|spec)\.rb\z})
+        if config.explicit?(:tests) && tests_as_sources.any?
+          warn "mutineer: #{tests_as_sources.join(', ')} looks like a test file, not a source. " \
+               "--test takes one file; repeat it for each test file (--test a_test.rb --test b_test.rb)"
+          exit 2
+        end
+        # A directory source expands to its **/*.rb files; a file inside the
+        # project becomes its root-relative path (#104). Test inference (when --test is omitted) happens in validate!.
         config.sources = Pairing.expand_sources(argv[1..], project_root: config.project_root)
         run(config)
       else
@@ -224,6 +254,8 @@ module Mutineer
     # @param config [Mutineer::Config] run configuration.
     # @return [void]
     def self.validate!(config)
+      # First, so a conflict is reported before another check rewrites the config.
+      validate_matrix!(config) if config.matrix
       validate_test_command!(config) if config.test_command
 
       validate_since!(config) if config.since
@@ -277,6 +309,7 @@ module Mutineer
              "(surgical redefine needs a shared VM; the subprocess has its own)"
         exit 2
       end
+      warn_unused_settings(config, "--test-command", %i[timeout capture_timeout cache_dir])
       return unless config.jobs > 1
 
       warn "[mutineer] --test-command runs serially (no per-worker DB isolation yet); forcing --jobs 1."
@@ -305,6 +338,7 @@ module Mutineer
              "(rspec is not implemented on the daemon path yet)"
         exit 2
       end
+      warn_unused_settings(config, "--daemon", %i[timeout])
       return if config.strategy == "reload"
 
       # --rails defaults strategy to redefine; daemon always whole-file loads.
@@ -313,6 +347,66 @@ module Mutineer
       config.strategy = "reload"
     end
 
+    # Where each setting a backend may ignore applies, for the warning.
+    UNUSED_SETTING_SCOPE = {
+      timeout: "in-process runs only",
+      capture_timeout: "in-process and --daemon coverage capture",
+      cache_dir: "the in-process and --daemon coverage cache"
+    }.freeze
+
+    # A backend that never reads a setting the user set would silently do
+    # nothing with it, so say so. `--test-command` builds no coverage map and
+    # has its own runtime, so it reads none of {UNUSED_SETTING_SCOPE}; the
+    # daemon reads the capture limit and the cache directory but keeps its own
+    # per-mutant limit.
+    #
+    # @api private
+    # @param config [Mutineer::Config] run configuration.
+    # @param backend [String] the backend's flag.
+    # @param keys [Array<Symbol>] the settings that backend ignores.
+    # @return [void]
+    def self.warn_unused_settings(config, backend, keys)
+      keys.each do |key|
+        next unless config.explicit?(key)
+
+        applies = UNUSED_SETTING_SCOPE.fetch(key)
+        warn "[mutineer] #{config.origin(key)} has no effect with #{backend} (it applies to #{applies}); ignoring it."
+      end
+    end
+
+    # --matrix runs on the in-process backend and needs every mutant's whole
+    # covering run, so a backend that cannot name the failing test, or a run that
+    # stops early, is a usage error (exit 2), never a quietly partial matrix.
+    #
+    # @api private
+    # @param config [Mutineer::Config] run configuration.
+    # @return [void]
+    def self.validate_matrix!(config)
+      conflict, reason =
+        if config.dry_run
+          [:dry_run, "a dry run runs no tests, so it has no matrix"]
+        elsif config.daemon
+          [:daemon, "the kill matrix runs on the in-process backend only"]
+        elsif config.test_command
+          [:test_command, "the external suite reports pass or fail, not which test failed"]
+        elsif config.fail_fast
+          [:fail_fast, "a fail-fast run is partial, so blind and redundant tests would be wrong"]
+        end
+      return unless conflict
+
+      # Say how to turn off whichever side came from the file, for this run.
+      hint =
+        if config.origin(:matrix) != "--matrix"
+          "; pass --no-matrix to run without it"
+        elsif conflict == :fail_fast && config.origin(:fail_fast) != "--fail-fast"
+          "; pass --no-fail-fast to run without it"
+        else
+          ""
+        end
+      warn "mutineer: #{config.origin(:matrix)} cannot be combined with #{config.origin(conflict)} (#{reason}#{hint})"
+      exit 2
+    end
+
     # --since needs a real git repo and a resolvable ref; either failure is a
     # usage error (exit 2) so CI sees "bad invocation," not "tests too weak."
     #
@@ -354,9 +448,10 @@ module Mutineer
     end
 
     # Auto-pair sources to tests by path convention when no --test was given
-    # (explicit --test wins). Each source with an inferred test on disk joins the
-    # run; a source with none is dropped with a one-line stderr warning and the
-    # run continues with the rest. If every source is dropped: in boot mode the
+    # (explicit --test wins). Each source with at least one inferred test on
+    # disk joins the run; a source with none is dropped with a one-line stderr
+    # warning and the run continues with the rest. Split files for one source
+    # are all kept (#87). If every source is dropped: in boot mode the
     # dedicated --boot/--rails-requires-test check reports it; otherwise exit 2
     # with a usage message. The framework is re-detected from the inferred set
     # unless it was set explicitly (a spec-only project loads/reports as rspec).
@@ -368,14 +463,14 @@ module Mutineer
       return unless config.tests.empty?
 
       paired = config.sources.filter_map do |s|
-        t = Pairing.infer_test(s, project_root: config.project_root, prefer: config.framework)
-        [s, t] if t
+        tests = Pairing.infer_tests(s, project_root: config.project_root, prefer: config.framework)
+        [s, tests] unless tests.empty?
       end
       (config.sources - paired.map(&:first)).each do |s|
         warn "[mutineer] no test found by convention for #{s}; skipping"
       end
       config.sources = paired.map(&:first)
-      config.tests   = paired.map(&:last).uniq
+      config.tests   = paired.flat_map(&:last).uniq
       config.framework = Config.detect_framework(config.tests) unless config.explicit?(:framework)
 
       return unless config.sources.empty?
@@ -426,7 +521,8 @@ module Mutineer
 
       aggregate, source_map, extras = Runner.execute(config)
       warn_legacy_ignore_matches(extras[:legacy_ignore_matches])
-      reporter = Reporter.new(aggregate, source_map)
+      matrix = KillMatrix.new(aggregate.results) if config.matrix
+      reporter = Reporter.new(aggregate, source_map, matrix: matrix)
 
       # Diff the current run against the baseline (preflighted above) by the
       # stable survivor id. The delta is rendered inline (human section / additive

data/lib/mutineer/config.rb

@@ -39,8 +39,13 @@ module Mutineer
     ConfigOption.new(field: :ignore, type: :string_list, yaml_key: "ignore"),
     ConfigOption.new(field: :baseline, type: :string, yaml_key: "baseline", flag: "--baseline"),
     ConfigOption.new(field: :fail_fast, type: :bool, yaml_key: "fail_fast", flag: "--fail-fast"),
+    ConfigOption.new(field: :matrix, type: :bool, yaml_key: "matrix", flag: "--matrix"),
     ConfigOption.new(field: :test_command, type: :string, yaml_key: "test_command", flag: "--test-command"),
     ConfigOption.new(field: :daemon, type: :bool, yaml_key: "daemon", flag: "--daemon"),
+    ConfigOption.new(field: :timeout, type: :positive_int, yaml_key: "timeout", flag: "--timeout"),
+    ConfigOption.new(field: :capture_timeout, type: :positive_int, yaml_key: "capture_timeout",
+                     flag: "--capture-timeout"),
+    ConfigOption.new(field: :cache_dir, type: :string, yaml_key: "cache_dir", flag: "--cache-dir"),
     ConfigOption.new(field: :format, type: :enum, flag: "--format", values: %w[human json html]),
     ConfigOption.new(field: :strategy, type: :enum, flag: "--strategy", values: %w[reload redefine],
                      aliases: STRATEGY_ALIASES),
@@ -58,10 +63,13 @@ module Mutineer
   # Config loading and the CLI > file > default precedence merge live here; each
   # layer holds only the keys the user wrote, and Config#explicit? reports them.
   #
+  # `matrix` (--matrix) runs every covering test for each mutant and reports
+  # which tests kill it (see KillMatrix); it never changes a verdict.
+  #
   # Boot mode adds: boot (a file to require ONCE in the parent so the app env,
   # e.g. Rails, is booted before forking; sources are then NOT manually required)
-  # and rails (sugar: defaults boot to config/environment and strategy to redefine,
-  # and reconnects ActiveRecord per fork).
+  # and rails (sugar: defaults boot to config/environment, prefers redefine without
+  # a daemon, and reconnects ActiveRecord per fork).
   Config = Struct.new(
     :sources, :tests, :operators, :threshold, :only, :dry_run,
     :cache_dir, :project_root, :load_paths,
@@ -70,7 +78,7 @@ module Mutineer
     # :daemon is user-facing (--daemon flag + KNOWN_KEYS + boolean coerce).
     # :daemon_timeout stays programmatic (set by tests/Runner; no flag yet).
     :baseline, :baseline_epsilon, :fail_fast, :test_command,
-    :daemon, :daemon_timeout,
+    :daemon, :daemon_timeout, :timeout, :capture_timeout, :matrix,
     keyword_init: true
   ) do
     # Config file name.
@@ -81,9 +89,12 @@ module Mutineer
 
     # @param explicit [Array<Symbol>] fields the user wrote (CLI or file). Derived
     #   values fill only the others; a programmatic Config.new writes none.
-    def initialize(explicit: [], **kwargs)
+    # @param from_file [Array<Symbol>] the explicit fields whose value came from
+    #   the config file (the command line did not override them).
+    def initialize(explicit: [], from_file: [], **kwargs)
       super(**kwargs)
       @explicit = explicit.to_a.dup.freeze
+      @from_file = from_file.to_a.dup.freeze
       self.sources       ||= []
       self.tests         ||= []
       self.threshold     ||= 0.0
@@ -101,6 +112,7 @@ module Mutineer
       self.baseline_epsilon ||= 0.0
       self.fail_fast     = false if fail_fast.nil?
       self.daemon        = false if daemon.nil?
+      self.matrix        = false if matrix.nil?
     end
 
     # True when the user wrote `key`, on the command line or in the config
@@ -114,6 +126,17 @@ module Mutineer
       @explicit.include?(key)
     end
 
+    # Where the user set `key`, for messages: the config-file key (as
+    # `name in .mutineer.yml`) when its value came from the file, else the
+    # command-line flag.
+    #
+    # @param key [Symbol] Config field name (a row of the option schema).
+    # @return [String] e.g. `"--fail-fast"` or `"fail_fast in .mutineer.yml"`.
+    def origin(key)
+      opt = CONFIG_OPTIONS.find { |o| o.field == key } or raise ArgumentError, "unknown option #{key.inspect}"
+      @from_file.include?(key) && opt.yaml_key ? "#{opt.yaml_key} in #{CONFIG_FILE}" : (opt.flag || opt.yaml_key)
+    end
+
     # Walk from `start` toward `home`, returning the first .mutineer.yml path found
     # or nil. Checks `home` itself, then stops; if `start` is above `home`
     # (e.g. /tmp), the walk continues to the filesystem root. Pure discovery;
@@ -134,10 +157,18 @@ module Mutineer
     end
 
     # Parse a .mutineer.yml into a symbol-keyed hash of recognized keys. Unknown
-    # keys / unknown operator names emit a one-line stderr warning and are ignored.
-    # A YAML syntax error raises ConfigError: never a silent fallback to defaults,
-    # and never an exit from the lib layer.
-    def self.from_file(path)
+    # keys emit a one-line stderr warning and are ignored. Unknown operator names
+    # warn and are dropped. If that leaves no names, the file is an error: an
+    # empty operator list would run nothing and exit 0. Pass
+    # +defer_operators: true+ only when the command line replaces that list, so
+    # a blank or all-unknown file list does not block +--operators+. A YAML
+    # syntax error raises ConfigError: never a silent fallback to defaults, and
+    # never an exit from the lib layer.
+    #
+    # @param path [String] config file path.
+    # @param defer_operators [Boolean] keep an empty operator list for a CLI override.
+    # @return [Hash{Symbol => Object}]
+    def self.from_file(path, defer_operators: false)
       raw = YAML.safe_load(File.read(path)) || {}
       name = File.basename(path)
       unless raw.is_a?(Hash)
@@ -154,8 +185,13 @@ module Mutineer
           next
         end
         field = field_for(ks)
-        parsed = parse(field, value, file: name)
-        parsed = filter_operators(parsed, name) if field == :operators
+        parsed = parse(field, value, file: name, defer_operators: defer_operators)
+        if field == :operators
+          parsed = filter_operators(parsed, name)
+          if parsed.empty? && !defer_operators
+            raise ConfigError, "#{name}: operators must name at least one known operator"
+          end
+        end
         out[field] = parsed
       end
       out
@@ -173,7 +209,7 @@ module Mutineer
     # @return [Mutineer::Config]
     def self.resolve(cli_opts, file_hash)
       user = file_hash.merge(cli_opts)
-      config = new(**user, explicit: user.keys)
+      config = new(**user, explicit: user.keys, from_file: file_hash.keys - cli_opts.keys)
 
       # --rails sugar: boot config/environment. Prefer redefine only for the
       # in-process path (daemon is whole-file reload only). In-process --rails
@@ -227,9 +263,12 @@ module Mutineer
     # @param value [Object] raw CLI string or YAML value.
     # @param file [String, nil] config file name when the value came from it;
     #   nil when it came from the command line.
+    # @param defer_operators [Boolean] when true, a blank operator list is
+    #   returned instead of raising, so +--operators+ can replace it. The flag
+    #   itself still rejects a blank list.
     # @return [Object] the typed value.
     # @raise [Mutineer::ConfigError] when the value does not fit the field's type.
-    def self.parse(field, value, file: nil)
+    def self.parse(field, value, file: nil, defer_operators: false)
       opt = CONFIG_OPTIONS.find { |o| o.field == field } or raise ArgumentError, "unknown option #{field.inspect}"
       origin = file ? "#{file}: #{opt.yaml_key}" : opt.flag
       got = "(got: #{value.inspect})"
@@ -260,7 +299,16 @@ module Mutineer
 
         prefix = file ? "#{file}: " : ""
         raise ConfigError, "#{prefix}unknown #{field} #{value.to_s.inspect}. Expected: #{opt.values.join(', ')}"
-      when :string_list then Array(value).map(&:to_s)
+      when :string_list
+        items = Array(value).map(&:to_s)
+        # Only `operators` treats [] as "run these" rather than "use the
+        # default". A blank key then makes no mutants and exits 0. An empty
+        # `require` or `ignore` matches the default, so those stay valid.
+        if field == :operators && !defer_operators && ([nil, true, false].include?(value) || items.empty? || items.all? { |item| item.strip.empty? })
+          raise ConfigError, "#{origin} must name at least one operator, not blank #{got}"
+        end
+
+        items
       when :string
         # A key written with no value (`baseline:`) parses as nil. Keeping nil
         # would switch the feature off without a word; main failed here, so the
@@ -268,6 +316,9 @@ module Mutineer
         # `only: false` must not become the subject name "false" either: it
         # matches nothing, so the run has no mutants and still exits 0.
         raise ConfigError, "#{origin} must be a string #{got}" if [nil, true, false].include?(value)
+        # A blank cache directory (an unset CI variable) would put coverage.json
+        # in the project root, so it is an error, not the default.
+        raise ConfigError, "#{origin} must be a directory, not blank #{got}" if field == :cache_dir && value.to_s.strip.empty?
 
         value.to_s
       when :since