saboteur 0.3.0 → 0.5.0

data/lib/saboteur/custom_map.rb

@@ -0,0 +1,170 @@
+# SPDX-License-Identifier: MIT
+# frozen_string_literal: true
+
+module Saboteur
+  # Explicit source-file -> spec-file mapping from config/saboteur_custom_map.rb:
+  #
+  #   SABOTEUR_CUSTOM_MAP = {
+  #     "app/services/tax_sync.rb" => ["spec/services/tax_sync_spec.rb"],
+  #     "app/services/**/*.rb"     => ["spec/services/**/*_spec.rb"],
+  #     %r{\Aapp/jobs/(.*)\.rb\z}  => ['spec/jobs/\1_spec.rb'],
+  #     "app/generated/**/*.rb"    => [], # intentionally untested
+  #   }.freeze
+  #
+  # The file loads before the app boots, so it must be plain Ruby.
+  class CustomMap
+    CONSTANT = :SABOTEUR_CUSTOM_MAP
+
+    Entry = Data.define(:key, :specs)
+
+    attr_reader :entries, :root
+
+    def self.load(path, root: Dir.pwd)
+      return new([], root: root) unless path && File.file?(path)
+
+      Object.send(:remove_const, CONSTANT) if Object.const_defined?(CONSTANT)
+      Kernel.load(path)
+      unless Object.const_defined?(CONSTANT)
+        raise ConfigError, "#{path}: expected it to define #{CONSTANT} = { ... }.freeze"
+      end
+
+      data = Object.const_get(CONSTANT)
+      raise ConfigError, "#{path}: #{CONSTANT} must be a Hash, got #{data.class}" unless data.is_a?(Hash)
+
+      new(data.map { |key, specs| validate_entry(path, root, key, specs) }, root: root)
+    rescue SyntaxError, LoadError => e
+      raise ConfigError, "#{path}: #{e.message.lines.first.to_s.chomp}"
+    rescue NameError => e
+      raise ConfigError, name_error_message(path, e)
+    ensure
+      Object.send(:remove_const, CONSTANT) if Object.const_defined?(CONSTANT)
+    end
+
+    def self.name_error_message(path, error)
+      line = error.backtrace_locations&.find { |loc| loc.path == path }&.lineno
+      "#{path}:#{line}: #{error.message.lines.first.to_s.chomp} " \
+        "- this file loads before the app boots; use plain Ruby or ENV"
+    end
+
+    def self.validate_entry(path, root, key, specs)
+      unless key.is_a?(String) || key.is_a?(Regexp)
+        raise ConfigError, "#{path}: key #{key.inspect} must be a String path, glob, or Regexp"
+      end
+      unless specs.is_a?(Array)
+        raise ConfigError, "#{path}: #{key.inspect} maps to #{specs.inspect}; wrap spec paths in an array"
+      end
+
+      specs.each { |spec| validate_spec(path, root, key, spec) }
+      Entry.new(key: key, specs: specs)
+    end
+
+    def self.validate_spec(path, root, key, spec)
+      unless spec.is_a?(String)
+        raise ConfigError, "#{path}: #{key.inspect} maps to #{spec.inspect}; spec paths must be Strings"
+      end
+      return if key.is_a?(Regexp) && spec.match?(/\\\d/) # backreference filled in at match time
+
+      expanded = File.expand_path(spec, root)
+      unless expanded.start_with?("#{root}/")
+        raise ConfigError, "#{path}: #{key.inspect} maps to #{spec.inspect}, which is outside the project root"
+      end
+      return if Dir.glob(expanded).any?
+
+      raise ConfigError, "#{path}: #{key.inspect} maps to #{spec.inspect}, which matches no file"
+    end
+
+    private_class_method :validate_entry, :validate_spec
+
+    def initialize(entries, root: Dir.pwd)
+      @entries = entries
+      @root = root
+    end
+
+    def empty?
+      @entries.empty?
+    end
+
+    # Spec paths for a source file, nil when no entry matches, :excluded when
+    # the winning entry maps to an empty array (intentionally untested).
+    def specs_for(file)
+      exact = @entries.find { |entry| entry.key == file }
+      return resolve(exact, nil) if exact
+
+      glob = glob_entry(file)
+      return resolve(glob, nil) if glob
+
+      regexp_specs(file)
+    end
+
+    # String keys (exact or glob) that match nothing under the root.
+    def stale_keys
+      @entries.select { |entry| entry.key.is_a?(String) }
+              .reject { |entry| Dir.glob(File.join(@root, entry.key)).any? }
+              .map(&:key)
+    end
+
+    # Literal String keys written twice (Ruby silently keeps the last).
+    # Detection is line-based and only sees double-quoted one-line keys;
+    # single-quoted, multi-line or %q keys that repeat go unreported.
+    def duplicate_keys(path)
+      seen = Hash.new(0)
+      File.readlines(path).each do |line|
+        seen[::Regexp.last_match(1)] += 1 if line =~ /^\s*"(.+?)"\s*=>/
+      end
+      seen.select { |_, count| count > 1 }.keys
+    end
+
+    def self.glob?(pattern, file)
+      File.fnmatch?(pattern, file, File::FNM_PATHNAME | File::FNM_EXTGLOB)
+    end
+
+    private
+
+    def glob_entry(file)
+      globs = @entries.select { |entry| entry.key.is_a?(String) && self.class.glob?(entry.key, file) }
+      return nil if globs.empty?
+
+      best = globs.group_by { |entry| specificity(entry.key) }.max_by { |score, _| score }.last
+      if best.size > 1
+        raise ConfigError,
+              "custom map keys #{best.map { |e| e.key.inspect }.join(' and ')} both match #{file} " \
+              "with equal specificity; make one more specific"
+      end
+      best.first
+    end
+
+    def regexp_specs(file)
+      matches = @entries.filter_map do |entry|
+        next unless entry.key.is_a?(Regexp)
+
+        match = entry.key.match(file)
+        [entry, match] if match
+      end
+      return nil if matches.empty?
+      # Exclusion is unanimous: a file is intentionally untested only when
+      # every matching regexp entry maps to an empty array. One mapped entry
+      # is enough to keep the file tested; unmapped entries add nothing.
+      return :excluded if matches.all? { |entry, _| entry.specs.empty? }
+
+      matches.reject { |entry, _| entry.specs.empty? }
+             .flat_map { |entry, match| resolve(entry, match) }
+    end
+
+    def specificity(key)
+      key.count("^*?[]{}\\")
+    end
+
+    def resolve(entry, match)
+      return :excluded if entry.specs.empty?
+
+      specs = entry.specs
+      if match
+        # A backreference to a group that did not participate expands to "";
+        # that is never a real spec path, so drop it here rather than letting
+        # a junk entry reach the run.
+        specs = specs.map { |spec| spec.gsub(/\\(\d+)/) { match[::Regexp.last_match(1).to_i].to_s } }.reject(&:empty?)
+      end
+      specs
+    end
+  end
+end

data/lib/saboteur/doctor.rb

@@ -5,11 +5,13 @@ require "coverage"
 require_relative "baseline_health"
 
 module Saboteur
-  # Reports the current environment and runs baseline checks, never mutations.
+  # Reports the current environment, never mutations. The two unchanged suite
+  # passes are slow on a large suite, so they run only when asked for.
   class Doctor
-    def initialize(config, out: $stdout)
+    def initialize(config, out: $stdout, suite: false)
       @config = config
       @out = out
+      @suite = suite
     end
 
     def call
@@ -17,20 +19,32 @@ module Saboteur
       check_runtime
       check_coverage
       check_boot_files
+      check_custom_map
+      check_ignore_subjects
       hooks = Hooks.new
       integration = boot_target(hooks)
       @out.puts "Integration: #{@config['integration']} (loaded)"
       @out.puts "Rails: #{@config['rails'] ? 'enabled' : 'disabled'}"
       @out.puts "SimpleCov: #{defined?(::SimpleCov) ? 'present; start suppressed for this process' : 'not loaded'}"
       check_coverage
-      tests = BaselineHealth.new(@config, integration: integration, hooks: hooks).call
-      @out.puts "Baseline: #{tests.size} tests passed twice in the same order with seed #{@config['seed']}"
-      @out.puts "Healthy in these two runs. This does not prove the suite is free of flaky tests."
+      check_suite(integration, hooks)
       true
     end
 
     private
 
+    def check_suite(integration, hooks)
+      unless @suite
+        @out.puts "Baseline: not run. Pass --suite to run the unchanged suite twice " \
+                  "(add --test PATH to limit it to some specs)."
+        return
+      end
+
+      tests = BaselineHealth.new(@config, integration: integration, hooks: hooks).call
+      @out.puts "Baseline: #{tests.size} tests passed twice in the same order with seed #{@config['seed']}"
+      @out.puts "Healthy in these two runs. This does not prove the suite is free of flaky tests."
+    end
+
     def boot_target(hooks)
       Engine::Boot.new(@config, hooks).load_target
     rescue LoadError, SyntaxError, SystemExit, StandardError => e
@@ -51,17 +65,51 @@ module Saboteur
     def check_coverage
       state = Coverage.state
       @out.puts "Coverage: #{state}"
-      return if Coverage.supported?(:lines) && state == :suspended
+      return if Coverage.supported?(:lines) && Coverage.supported?(:methods) && state == :suspended
+
+      raise Error, "Coverage must be suspended with line and method support before boot; use the saboteur " \
+                   "executable, disable SimpleCov and remove other Coverage.start calls"
+    end
+
+    # Match against all included source definitions, not the current diff or
+    # selected tests. A limited run must not make a valid ignore look stale.
+    def check_ignore_subjects
+      ignores = IgnoreList.load(@config.path("ignore_file"))
+      return if ignores.entries.empty?
+
+      finder = SubjectFinder.new
+      parsed = @config.included_files.map { |file| finder.call(file) }
+      if parsed.any? { |result| result.errors.any? }
+        @out.puts "Warning: ignore subject check incomplete: an included file does not parse"
+        return
+      end
+      subjects = parsed.flat_map(&:subjects).to_set(&:id)
+      ignores.entries.map(&:subject).uniq.reject { |subject| subjects.include?(subject) }.each do |subject|
+        @out.puts "Warning: ignore subject #{subject} matches no discovered subject in included files; " \
+                  "check include/exclude settings and the exact Class#method or Class.method form"
+      end
+    end
+
+    def check_custom_map
+      path = @config.path("custom_map")
+      map = CustomMap.load(path, root: @config.root)
+      return @out.puts("Custom map: none (optional)") if map.empty?
+
+      duplicates = map.duplicate_keys(path)
+      if duplicates.any?
+        @out.puts "Warning: custom map keys written twice, Ruby keeps the last: #{duplicates.join(', ')}"
+      end
+      stale = map.stale_keys
+      raise ConfigError, "#{path}: stale keys matching no file: #{stale.join(', ')}" if stale.any?
 
-      raise Error, "Coverage must be suspended with line support before boot; use the saboteur executable, " \
-                   "disable SimpleCov and remove other Coverage.start calls"
+      @out.puts "Custom map: #{map.entries.size} entries (loaded)"
     end
 
     def check_boot_files
       %w[require hooks].each do |key|
         @config[key].each do |file|
           path = File.expand_path(file, @config.root)
-          raise ConfigError, "#{key} file missing: #{file}; fix .saboteur.yml" unless File.file?(path)
+          raise ConfigError, "#{key} file missing: #{file}; fix config/saboteur.rb" unless File.file?(path)
 
           @out.puts "#{key}: #{file} (found)"
         end

data/lib/saboteur/engine/baseline_guard.rb

@@ -10,6 +10,10 @@ module Saboteur
     # or mutation source is used. This catches observed instability, not all
     # possible flaky behaviour.
     class BaselineGuard
+      EXCERPT_LINES = 10
+      EXCERPT_WIDTH = 200
+      EXCERPT_SCAN_LINES = 5000
+
       def initialize(config:, integration:, hooks:, supervisor:, diagnostics:)
         @config = config
         @integration = integration
@@ -45,15 +49,32 @@ module Saboteur
 
       def check(number)
         Execution::ForkReadiness.prepare_heap
-        result = @supervisor.run(timeout: @config["coverage_timeout"]) { |writer| record(writer, number) }
+        heartbeat = @diagnostics.progress_heartbeat("baseline check #{number}").watch(log_path(number))
+        result = @supervisor.run(timeout: @config["coverage_timeout"], on_tick: heartbeat.method(:tick)) do |writer|
+          record(writer, number)
+        end
         refuse(number, "the suite stopped: #{result.stopped_by} (#{result.detail})") if result.stopped_by
+        # Before the payload checks: a child killed by a signal (the OOM
+        # killer, say) writes nothing, and the signal is the explanation. A
+        # child the supervisor itself killed after it had reported is not a
+        # failure: its payload is checked below.
+        refuse(number, unsuccessful_exit(result.exit_status, result.memory_pressure)) if result.child_failed?
         refuse(number, "the child did not report a result") if result.payload.empty?
-        refuse(number, "the child exited unsuccessfully") if
-          result.exit_status.is_a?(Process::Status) && !result.exit_status.success?
         refuse(number, "the suite left child processes behind") if result.leftover_pids.any?
         parse_result(number, result.payload)
       end
 
+      # A signaled child (the OOM killer, say) can leave an empty log,
+      # which reads like a suite failure; name the signal. The OOM hint is
+      # printed only when the supervisor saw the process tree's memory grow.
+      def unsuccessful_exit(status, memory_pressure)
+        return "the child exited with status #{status.exitstatus}" unless status.signaled?
+
+        signal = "SIG#{Signal.signame(status.termsig)}"
+        hint = signal == "SIGKILL" && memory_pressure ? " - possible OOM, check memory" : ""
+        "the child was killed by #{signal}#{hint}"
+      end
+
       def parse_result(number, payload)
         data = JSON.parse(payload)
         refuse(number, "the child reported an invalid result") unless data.is_a?(Hash)
@@ -91,8 +112,26 @@ module Saboteur
 
       def refuse(number, reason)
         raise Error, "baseline check #{number} failed at seed #{@config['seed']}: #{reason}. " \
-                     "See the log at #{log_path(number)}. Fix the suite or its setup, then run saboteur again " \
-                     "with the same seed. No mutants were started."
+                     "#{failure_excerpt(number)}See the full log at #{log_path(number)}. Fix the suite or its " \
+                     "setup, then run saboteur again with the same seed. No mutants were started."
+      end
+
+      # The first lines of the failure, inline, so the cause is readable
+      # without opening the log: RSpec's "Failures:" section when the log has
+      # one, otherwise the last lines of the log.
+      def failure_excerpt(number)
+        lines = log_lines(number)
+        return "" if lines.empty?
+
+        start = lines.index { |line| line.start_with?("Failures:") }
+        shown = start ? lines[start, EXCERPT_LINES] : lines.last(EXCERPT_LINES)
+        "The log says:\n#{shown.map { |line| "  #{line.slice(0, EXCERPT_WIDTH)}" }.join("\n")}\n"
+      end
+
+      def log_lines(number)
+        File.foreach(log_path(number), chomp: true).first(EXCERPT_SCAN_LINES).reject { |line| line.strip.empty? }
+      rescue SystemCallError
+        []
       end
     end
   end

data/lib/saboteur/engine/boot.rb

@@ -62,6 +62,7 @@ module Saboteur
         @rails = rails = Frameworks::Rails.new(@config, @hooks) if @config["rails"]
         rails&.before_boot
         @hooks.run(:before_boot)
+        add_spec_to_load_path if @config["rails"]
         @config["require"].each { |file| require File.expand_path(file, @root) }
         integration = Integration.fetch(@config["integration"]).new(@config)
         integration.setup
@@ -72,6 +73,14 @@ module Saboteur
 
       private
 
+      # rspec puts spec/ on the load path, so a boot file under spec/ (a
+      # dummy app's rails_helper, say) can require its siblings. Match
+      # that, or the same require list fails to load.
+      def add_spec_to_load_path
+        path = File.join(@root, "spec")
+        $LOAD_PATH.unshift(path) unless $LOAD_PATH.include?(path)
+      end
+
       def load_hooks
         Saboteur.current_hooks = @hooks
         @config["hooks"].each { |file| load File.expand_path(file, @root) }

data/lib/saboteur/engine/bytecode_equivalence.rb

@@ -0,0 +1,95 @@
+# SPDX-License-Identifier: MIT
+# frozen_string_literal: true
+
+module Saboteur
+  class Engine
+    # Trivial compiler equivalence: a mutant whose method compiles to the
+    # same instruction sequence as the original cannot change behaviour, so
+    # running it can only produce a false survivor. The comparison walks the
+    # compiled instruction tree and drops only position metadata (line
+    # numbers, line events, source spans, node ids, file paths). Operands,
+    # including string and numeric literals, are compared exactly (2 and 2.0 differ, and so do +0.0 and -0.0).
+    class BytecodeEquivalence
+      ISEQ_MAGIC = "YARVInstructionSequence/SimpleDataFormat"
+      # Operand types whose eql? is exact. Anything else (an unknown object,
+      # a type that could hold a Float) is never called equal, so its
+      # mutant still runs.
+      PLAIN = [Integer, String, Symbol, Rational, Regexp, NilClass, TrueClass, FalseClass].freeze
+      POSITION_EVENT = /\ARUBY_EVENT_/
+
+      # True when mutated_text compiles to the same instructions as the
+      # original subject source under the same compile options. Anything the
+      # compiler cannot judge counts as not equivalent, so the mutant still
+      # runs. frozen_string_literal: whether the file carries that magic
+      # comment, which changes how string literals compile.
+      def equivalent?(original_text, mutated_text, frozen_string_literal: false)
+        original = fingerprint_of_original(original_text, frozen_string_literal)
+        return false unless original
+
+        exact?(original, fingerprint(mutated_text, frozen_string_literal))
+      end
+
+      private
+
+      # Mutants of one method arrive together, so the last original is enough.
+      def fingerprint_of_original(text, frozen)
+        key = [text, frozen]
+        return @original_fingerprint if key == @original_key
+
+        @original_key = key
+        @original_fingerprint = fingerprint(text, frozen)
+      end
+
+      def fingerprint(text, frozen)
+        options = { frozen_string_literal: frozen }
+        strip_positions(RubyVM::InstructionSequence.compile(text, nil, nil, 1, options).to_a)
+      rescue SyntaxError, StandardError
+        nil
+      end
+
+      # Structural equality that never coerces: an Integer is not a Float,
+      # and +0.0 is not -0.0 (1.0 / +0.0 and 1.0 / -0.0 are different
+      # infinities), which eql? alone would not tell apart.
+      def exact?(left, right)
+        case left
+        when Array then right.is_a?(Array) && left.size == right.size && left.zip(right).all? { |l, r| exact?(l, r) }
+        when Hash, Complex, Range then right.instance_of?(left.class) && exact?(parts(left), parts(right))
+        when Float then right.is_a?(Float) && [left].pack("G") == [right].pack("G")
+        when *PLAIN then left.eql?(right)
+        else false
+        end
+      end
+
+      # The values a composite operand is made of, compared one by one.
+      def parts(value)
+        case value
+        when Hash then value.to_a
+        when Complex then value.rectangular
+        else [value.begin, value.end, value.exclude_end?]
+        end
+      end
+
+      def strip_positions(node)
+        return node unless node.is_a?(Array)
+        return node.map { |child| strip_positions(child) } unless node.first == ISEQ_MAGIC
+
+        _magic, major, minor, format, _misc, name, _path, _real_path, _first_line, type, locals, params, catches,
+          body = node
+        [major, minor, format, name, type, locals, strip_positions(params), strip_positions(catches),
+         strip_body(body)]
+      end
+
+      def strip_body(body)
+        body.reject { |entry| position?(entry) }.map { |entry| strip_positions(entry) }
+      end
+
+      # Line numbers and line/call/return events depend only on layout.
+      def position?(entry)
+        case entry
+        when Integer, POSITION_EVENT then true
+        else false
+        end
+      end
+    end
+  end
+end

data/lib/saboteur/engine/canary.rb

@@ -0,0 +1,74 @@
+# SPDX-License-Identifier: MIT
+# frozen_string_literal: true
+
+require "open3"
+
+module Saboteur
+  class Engine
+    # Watches the working tree while mutants run (issue #102). A mutant that
+    # slips past the hazard guard and deletes tracked files must abort the
+    # run, not be reported as a survivor. The snapshot is the tracked files
+    # (git ls-files) when the tree is a repository, else every file under the
+    # root; the tool's own cache directory is excluded in both cases, and the
+    # presence of .git is watched separately.
+    class Canary
+      GIT_DIR = ".git"
+
+      def initialize(root, cache_dir: ".saboteur")
+        @root = root
+        @cache_dir = cache_dir.to_s
+        @snapshot = inventory
+      end
+
+      # After a subject's mutants: any file present at snapshot time and gone
+      # now means a mutant deleted it. digest names the most recent mutant,
+      # the likely cause.
+      def check!(subject: nil, digest: nil)
+        missing = @snapshot - inventory
+        return if missing.empty?
+
+        of = " of #{subject.id}" if subject
+        culprit = digest ? " The most recent mutant, #{digest}#{of}, is the likely cause." : ""
+        raise Saboteur::CanaryTripped,
+              "canary: #{missing.size} watched file(s) vanished from the working tree while mutants ran " \
+              "(first: #{missing.first}).#{culprit} The canary watches tracked files and .git presence " \
+              "and only detects the loss after the fact; it cannot restore anything. Recover from your " \
+              "own backup, a fresh checkout, or a reclone - a deleted .git or uncommitted work is gone " \
+              "unless you have another copy."
+      end
+
+      private
+
+      def inventory
+        files = git? ? git_tracked_files : walk_files
+        files << GIT_DIR if File.exist?(File.join(@root, GIT_DIR))
+        files.uniq
+      end
+
+      def git?
+        File.directory?(File.join(@root, GIT_DIR)) && system("git", "--version", %i[out err] => File::NULL)
+      end
+
+      def git_tracked_files
+        out, status = Open3.capture2("git", "ls-files", "-z", chdir: @root)
+        return walk_files unless status.success?
+
+        # ls-files reads the index, which still names a deleted file; keep
+        # only paths present in the worktree so a deletion shows up.
+        out.split("\0").reject { |rel| cache_path?(rel) }
+           .select { |rel| File.exist?(File.join(@root, rel)) || File.symlink?(File.join(@root, rel)) }
+      end
+
+      def walk_files
+        Dir.glob(File.join(@root, "**", "*"), File::FNM_DOTMATCH)
+           .select { |path| File.file?(path) }
+           .map { |path| path.delete_prefix("#{@root}/") }
+           .reject { |rel| rel.start_with?("#{GIT_DIR}/") || cache_path?(rel) }
+      end
+
+      def cache_path?(rel)
+        rel.start_with?("#{@cache_dir}/")
+      end
+    end
+  end
+end

data/lib/saboteur/engine/candidate_selection.rb

@@ -6,7 +6,7 @@ module Saboteur
     # Selects a bounded list for each covered subject before workers fork.
     # Subject skips are not mutants and do not enter the mutation score.
     class CandidateSelection
-      attr_reader :skipped_subjects
+      attr_reader :skipped_subjects, :selected
 
       def initialize(candidates:, map:, limit:, diagnostics:)
         @candidates = candidates
@@ -14,21 +14,34 @@ module Saboteur
         @limit = limit
         @diagnostics = diagnostics
         @skipped_subjects = {}
+        @selected = {}
       end
 
       def for(subject, scope: nil)
-        if @map.tests_for(subject).empty?
-          @skipped_subjects[subject] = "no coverage"
-          return []
+        reason = subject.skip_reason || (skip_reason_for(subject) if @map.tests_for(subject).empty?)
+        if reason
+          @skipped_subjects[subject] = reason
+          return @selected[subject] = []
         end
 
         list = @candidates.for(subject)
         list = scope.restrict(subject, list) if scope
         omitted = list.size - @limit
         if omitted.positive?
+          @diagnostics.record_omission(subject, count: omitted, limit: @limit)
           @diagnostics.warn("#{subject.id}: omitted #{omitted} mutant(s) above max_mutants_per_method (#{@limit})")
         end
-        list.first(@limit)
+        @selected[subject] = list.first(@limit)
+      end
+
+      private
+
+      # Load-only code is not "no coverage": it ran, just never inside a
+      # test, and no forked test can re-run it (see CoverageMap::RAN_AT_LOAD).
+      def skip_reason_for(subject)
+        return "no coverage" unless @map.ran_at_load?(subject.file, subject.coverage_lines)
+
+        Selection::CoverageMap::RAN_AT_LOAD
       end
     end
   end