saboteur 0.4.0 → 0.6.0

data/lib/saboteur/engine/baseline_guard.rb

@@ -10,6 +10,10 @@ module Saboteur
     # or mutation source is used. This catches observed instability, not all
     # possible flaky behaviour.
     class BaselineGuard
+      EXCERPT_LINES = 10
+      EXCERPT_WIDTH = 200
+      EXCERPT_SCAN_LINES = 5000
+
       def initialize(config:, integration:, hooks:, supervisor:, diagnostics:)
         @config = config
         @integration = integration
@@ -45,29 +49,29 @@ module Saboteur
 
       def check(number)
         Execution::ForkReadiness.prepare_heap
-        heartbeat = @diagnostics.progress_heartbeat("baseline check #{number}")
+        heartbeat = @diagnostics.progress_heartbeat("baseline check #{number}").watch(log_path(number))
         result = @supervisor.run(timeout: @config["coverage_timeout"], on_tick: heartbeat.method(:tick)) do |writer|
           record(writer, number)
         end
         refuse(number, "the suite stopped: #{result.stopped_by} (#{result.detail})") if result.stopped_by
         # Before the payload checks: a child killed by a signal (the OOM
-        # killer, say) writes nothing, and the signal is the explanation.
-        if result.exit_status.is_a?(Process::Status) && !result.exit_status.success?
-          refuse(number, unsuccessful_exit(result.exit_status))
-        end
+        # killer, say) writes nothing, and the signal is the explanation. A
+        # child the supervisor itself killed after it had reported is not a
+        # failure: its payload is checked below.
+        refuse(number, unsuccessful_exit(result.exit_status, result.memory_pressure)) if result.child_failed?
         refuse(number, "the child did not report a result") if result.payload.empty?
         refuse(number, "the suite left child processes behind") if result.leftover_pids.any?
         parse_result(number, result.payload)
       end
 
       # A signaled child (the OOM killer, say) can leave an empty log,
-      # which reads like a suite failure; name the signal and hint the
-      # likely cause instead.
-      def unsuccessful_exit(status)
+      # which reads like a suite failure; name the signal. The OOM hint is
+      # printed only when the supervisor saw the process tree's memory grow.
+      def unsuccessful_exit(status, memory_pressure)
         return "the child exited with status #{status.exitstatus}" unless status.signaled?
 
         signal = "SIG#{Signal.signame(status.termsig)}"
-        hint = signal == "SIGKILL" ? " - possible OOM, check memory" : ""
+        hint = signal == "SIGKILL" && memory_pressure ? " - possible OOM, check memory" : ""
         "the child was killed by #{signal}#{hint}"
       end
 
@@ -108,8 +112,26 @@ module Saboteur
 
       def refuse(number, reason)
         raise Error, "baseline check #{number} failed at seed #{@config['seed']}: #{reason}. " \
-                     "See the log at #{log_path(number)}. Fix the suite or its setup, then run saboteur again " \
-                     "with the same seed. No mutants were started."
+                     "#{failure_excerpt(number)}See the full log at #{log_path(number)}. Fix the suite or its " \
+                     "setup, then run saboteur again with the same seed. No mutants were started."
+      end
+
+      # The first lines of the failure, inline, so the cause is readable
+      # without opening the log: RSpec's "Failures:" section when the log has
+      # one, otherwise the last lines of the log.
+      def failure_excerpt(number)
+        lines = log_lines(number)
+        return "" if lines.empty?
+
+        start = lines.index { |line| line.start_with?("Failures:") }
+        shown = start ? lines[start, EXCERPT_LINES] : lines.last(EXCERPT_LINES)
+        "The log says:\n#{shown.map { |line| "  #{line.slice(0, EXCERPT_WIDTH)}" }.join("\n")}\n"
+      end
+
+      def log_lines(number)
+        File.foreach(log_path(number), chomp: true).first(EXCERPT_SCAN_LINES).reject { |line| line.strip.empty? }
+      rescue SystemCallError
+        []
       end
     end
   end

data/lib/saboteur/engine/bytecode_equivalence.rb

@@ -0,0 +1,95 @@
+# SPDX-License-Identifier: MIT
+# frozen_string_literal: true
+
+module Saboteur
+  class Engine
+    # Trivial compiler equivalence: a mutant whose method compiles to the
+    # same instruction sequence as the original cannot change behaviour, so
+    # running it can only produce a false survivor. The comparison walks the
+    # compiled instruction tree and drops only position metadata (line
+    # numbers, line events, source spans, node ids, file paths). Operands,
+    # including string and numeric literals, are compared exactly (2 and 2.0 differ, and so do +0.0 and -0.0).
+    class BytecodeEquivalence
+      ISEQ_MAGIC = "YARVInstructionSequence/SimpleDataFormat"
+      # Operand types whose eql? is exact. Anything else (an unknown object,
+      # a type that could hold a Float) is never called equal, so its
+      # mutant still runs.
+      PLAIN = [Integer, String, Symbol, Rational, Regexp, NilClass, TrueClass, FalseClass].freeze
+      POSITION_EVENT = /\ARUBY_EVENT_/
+
+      # True when mutated_text compiles to the same instructions as the
+      # original subject source under the same compile options. Anything the
+      # compiler cannot judge counts as not equivalent, so the mutant still
+      # runs. frozen_string_literal: whether the file carries that magic
+      # comment, which changes how string literals compile.
+      def equivalent?(original_text, mutated_text, frozen_string_literal: false)
+        original = fingerprint_of_original(original_text, frozen_string_literal)
+        return false unless original
+
+        exact?(original, fingerprint(mutated_text, frozen_string_literal))
+      end
+
+      private
+
+      # Mutants of one method arrive together, so the last original is enough.
+      def fingerprint_of_original(text, frozen)
+        key = [text, frozen]
+        return @original_fingerprint if key == @original_key
+
+        @original_key = key
+        @original_fingerprint = fingerprint(text, frozen)
+      end
+
+      def fingerprint(text, frozen)
+        options = { frozen_string_literal: frozen }
+        strip_positions(RubyVM::InstructionSequence.compile(text, nil, nil, 1, options).to_a)
+      rescue SyntaxError, StandardError
+        nil
+      end
+
+      # Structural equality that never coerces: an Integer is not a Float,
+      # and +0.0 is not -0.0 (1.0 / +0.0 and 1.0 / -0.0 are different
+      # infinities), which eql? alone would not tell apart.
+      def exact?(left, right)
+        case left
+        when Array then right.is_a?(Array) && left.size == right.size && left.zip(right).all? { |l, r| exact?(l, r) }
+        when Hash, Complex, Range then right.instance_of?(left.class) && exact?(parts(left), parts(right))
+        when Float then right.is_a?(Float) && [left].pack("G") == [right].pack("G")
+        when *PLAIN then left.eql?(right)
+        else false
+        end
+      end
+
+      # The values a composite operand is made of, compared one by one.
+      def parts(value)
+        case value
+        when Hash then value.to_a
+        when Complex then value.rectangular
+        else [value.begin, value.end, value.exclude_end?]
+        end
+      end
+
+      def strip_positions(node)
+        return node unless node.is_a?(Array)
+        return node.map { |child| strip_positions(child) } unless node.first == ISEQ_MAGIC
+
+        _magic, major, minor, format, _misc, name, _path, _real_path, _first_line, type, locals, params, catches,
+          body = node
+        [major, minor, format, name, type, locals, strip_positions(params), strip_positions(catches),
+         strip_body(body)]
+      end
+
+      def strip_body(body)
+        body.reject { |entry| position?(entry) }.map { |entry| strip_positions(entry) }
+      end
+
+      # Line numbers and line/call/return events depend only on layout.
+      def position?(entry)
+        case entry
+        when Integer, POSITION_EVENT then true
+        else false
+        end
+      end
+    end
+  end
+end

data/lib/saboteur/engine/canary.rb

@@ -0,0 +1,74 @@
+# SPDX-License-Identifier: MIT
+# frozen_string_literal: true
+
+require "open3"
+
+module Saboteur
+  class Engine
+    # Watches the working tree while mutants run (issue #102). A mutant that
+    # slips past the hazard guard and deletes tracked files must abort the
+    # run, not be reported as a survivor. The snapshot is the tracked files
+    # (git ls-files) when the tree is a repository, else every file under the
+    # root; the tool's own cache directory is excluded in both cases, and the
+    # presence of .git is watched separately.
+    class Canary
+      GIT_DIR = ".git"
+
+      def initialize(root, cache_dir: ".saboteur")
+        @root = root
+        @cache_dir = cache_dir.to_s
+        @snapshot = inventory
+      end
+
+      # After a subject's mutants: any file present at snapshot time and gone
+      # now means a mutant deleted it. digest names the most recent mutant,
+      # the likely cause.
+      def check!(subject: nil, digest: nil)
+        missing = @snapshot - inventory
+        return if missing.empty?
+
+        of = " of #{subject.id}" if subject
+        culprit = digest ? " The most recent mutant, #{digest}#{of}, is the likely cause." : ""
+        raise Saboteur::CanaryTripped,
+              "canary: #{missing.size} watched file(s) vanished from the working tree while mutants ran " \
+              "(first: #{missing.first}).#{culprit} The canary watches tracked files and .git presence " \
+              "and only detects the loss after the fact; it cannot restore anything. Recover from your " \
+              "own backup, a fresh checkout, or a reclone - a deleted .git or uncommitted work is gone " \
+              "unless you have another copy."
+      end
+
+      private
+
+      def inventory
+        files = git? ? git_tracked_files : walk_files
+        files << GIT_DIR if File.exist?(File.join(@root, GIT_DIR))
+        files.uniq
+      end
+
+      def git?
+        File.directory?(File.join(@root, GIT_DIR)) && system("git", "--version", %i[out err] => File::NULL)
+      end
+
+      def git_tracked_files
+        out, status = Open3.capture2("git", "ls-files", "-z", chdir: @root)
+        return walk_files unless status.success?
+
+        # ls-files reads the index, which still names a deleted file; keep
+        # only paths present in the worktree so a deletion shows up.
+        out.split("\0").reject { |rel| cache_path?(rel) }
+           .select { |rel| File.exist?(File.join(@root, rel)) || File.symlink?(File.join(@root, rel)) }
+      end
+
+      def walk_files
+        Dir.glob(File.join(@root, "**", "*"), File::FNM_DOTMATCH)
+           .select { |path| File.file?(path) }
+           .map { |path| path.delete_prefix("#{@root}/") }
+           .reject { |rel| rel.start_with?("#{GIT_DIR}/") || cache_path?(rel) }
+      end
+
+      def cache_path?(rel)
+        rel.start_with?("#{@cache_dir}/")
+      end
+    end
+  end
+end

data/lib/saboteur/engine/candidate_selection.rb

@@ -6,7 +6,7 @@ module Saboteur
     # Selects a bounded list for each covered subject before workers fork.
     # Subject skips are not mutants and do not enter the mutation score.
     class CandidateSelection
-      attr_reader :skipped_subjects
+      attr_reader :skipped_subjects, :selected
 
       def initialize(candidates:, map:, limit:, diagnostics:)
         @candidates = candidates
@@ -14,21 +14,34 @@ module Saboteur
         @limit = limit
         @diagnostics = diagnostics
         @skipped_subjects = {}
+        @selected = {}
       end
 
       def for(subject, scope: nil)
-        if @map.tests_for(subject).empty?
-          @skipped_subjects[subject] = "no coverage"
-          return []
+        reason = subject.skip_reason || (skip_reason_for(subject) if @map.tests_for(subject).empty?)
+        if reason
+          @skipped_subjects[subject] = reason
+          return @selected[subject] = []
         end
 
         list = @candidates.for(subject)
         list = scope.restrict(subject, list) if scope
         omitted = list.size - @limit
         if omitted.positive?
+          @diagnostics.record_omission(subject, count: omitted, limit: @limit)
           @diagnostics.warn("#{subject.id}: omitted #{omitted} mutant(s) above max_mutants_per_method (#{@limit})")
         end
-        list.first(@limit)
+        @selected[subject] = list.first(@limit)
+      end
+
+      private
+
+      # Load-only code is not "no coverage": it ran, just never inside a
+      # test, and no forked test can re-run it (see CoverageMap::RAN_AT_LOAD).
+      def skip_reason_for(subject)
+        return "no coverage" unless @map.ran_at_load?(subject.file, subject.coverage_lines)
+
+        Selection::CoverageMap::RAN_AT_LOAD
       end
     end
   end

data/lib/saboteur/engine/candidates.rb

@@ -2,6 +2,7 @@
 # frozen_string_literal: true
 
 require "prism"
+require_relative "../inline_suppression"
 
 module Saboteur
   class Engine
@@ -12,7 +13,10 @@ module Saboteur
       def self.build_sources(config)
         root = File.realpath(config.root)
         config["sources"].map do |entry|
-          next MutationSource::Builtin.new(arid: config["arid"]) if entry == "builtin"
+          if entry == "builtin"
+            next MutationSource::Builtin.new(arid: config["arid"], operators: config["operators"],
+                                             exclude_operators: config["exclude_operators"])
+          end
 
           file, const = entry.split(":", 2)
           raise ConfigError, "mutation source #{entry.inspect} must be builtin or path.rb:ClassName" unless const
@@ -22,37 +26,88 @@ module Saboteur
         end
       end
 
-      def initialize(sources, diagnostics)
+      def initialize(sources, diagnostics, hazard_guard: nil, bytecode: BytecodeEquivalence.new)
         @sources = sources
         @diagnostics = diagnostics
+        @hazard_guard = hazard_guard
+        @bytecode = bytecode
       end
 
       def for(subject)
-        return [] unless parses?(subject.source)
+        unless parses?(subject.source)
+          @diagnostics.record_generation(subject, source: "subject", reason: "invalid_subject_syntax")
+          return []
+        end
 
         seen = Set[subject.source]
-        list = @sources.flat_map { |source| source.mutations_for(subject) }.filter_map do |mutation|
+        list = @sources.flat_map { |source| proposals(source, subject) }.filter_map do |mutation|
           candidate(subject, mutation, seen)
         end
+        list = suppress_inline(list, subject)
+        list = suppress_hazards(list, subject)
         list.sort_by.with_index { |(mutation, _), index| [mutation.coarse ? 0 : 1, index] }
       end
 
       private
 
+      def suppress_inline(list, subject)
+        directives = InlineSuppression.new(subject.file, subject.file_source)
+        list.reject do |mutation, _text|
+          directive = directives.reason(subject, mutation)
+          @diagnostics.record_inline_suppression(subject, mutation, directive) if directive
+          directive
+        end
+      end
+
+      # The hazard guard (issue #102) refuses mutants that would leave a
+      # destructive call running with altered inputs; each refusal is counted.
+      def suppress_hazards(list, subject)
+        return list unless @hazard_guard
+
+        kept, suppressed = @hazard_guard.partition(list, subject)
+        suppressed.each { |(mutation, _)| @diagnostics.record_hazard_suppression(subject, mutation) }
+        kept
+      end
+
+      def proposals(source, subject)
+        proposals = source.mutations_for(subject)
+        if proposals.empty?
+          @diagnostics.record_generation(subject, source: source_name(source), reason: "empty_generation")
+        end
+        proposals
+      end
+
+      def source_name(source)
+        name = source.name if source.respond_to?(:name)
+        name || "anonymous"
+      end
+
+      def record_drop(subject, mutation, reason)
+        @diagnostics.record_generation(subject, source: mutation.source, operator: mutation.operator, reason: reason)
+        nil
+      end
+
       def candidate(subject, mutation, seen)
         text = subject.apply(mutation.edits)
-        return if seen.include?(text)
+        return record_drop(subject, mutation, "unchanged") if text == subject.source
+        return record_drop(subject, mutation, "duplicate") if seen.include?(text)
 
         seen << text
         unless parses?(text)
           @diagnostics.count_invalid_mutation
-          return
+          return record_drop(subject, mutation, "invalid_syntax")
         end
 
+        return record_drop(subject, mutation, "equivalent_bytecode") if equivalent?(subject, text)
+
         [mutation, text]
       rescue ArgumentError => e
         @diagnostics.warn("#{subject.id}: dropped #{mutation.source} mutation: #{e.message}")
-        nil
+        record_drop(subject, mutation, "invalid_edit")
+      end
+
+      def equivalent?(subject, text)
+        @bytecode.equivalent?(subject.source, text, frozen_string_literal: subject.frozen_literals)
       end
 
       def parses?(text)

data/lib/saboteur/engine/coverage_store.rb

@@ -15,12 +15,13 @@ module Saboteur
       LOCKFILES = %w[Gemfile.lock gems.locked].freeze
       LOG = "coverage-build.log"
 
-      def initialize(config:, integration:, hooks:, supervisor:, diagnostics:)
+      def initialize(config:, integration:, hooks:, supervisor:, diagnostics:, load_lines: {})
         @config = config
         @integration = integration
         @hooks = hooks
         @supervisor = supervisor
         @diagnostics = diagnostics
+        @load_lines = load_lines
         @root = File.realpath(config.root)
       end
 
@@ -31,6 +32,8 @@ module Saboteur
 
         map = refuse_empty_map(build(files), files)
         map.save(path)
+        File.write(File.join(@config.path("cache_dir"), "preview.json"),
+                   JSON.generate(path: path, fingerprint: RunFingerprint.new(config: @config).snapshot.digest))
         map
       end
 
@@ -57,42 +60,59 @@ module Saboteur
         # The suite child forked next inherits this heap; pack it first, to
         # reduce later copying of shared pages.
         Execution::ForkReadiness.prepare_heap
-        heartbeat = @diagnostics.progress_heartbeat("building coverage map")
+        heartbeat = @diagnostics.progress_heartbeat("building coverage map").watch(log_path)
         result = @supervisor.run(timeout: @config["coverage_timeout"], on_tick: heartbeat.method(:tick)) do |writer|
           record(writer, files)
         end
+        map_from(result)
+      end
+
+      def map_from(result)
         raise Error, "coverage map build stopped: #{result.detail}" if result.stopped_by
         raise Error, "coverage map build failed: #{died(result.exit_status)}" if result.payload.empty?
 
         data = JSON.parse(result.payload)
         raise Error, "coverage map build failed: #{data['error']}" if data["error"]
 
-        Selection::CoverageMap.from_json(result.payload)
+        with_parent_load_lines(Selection::CoverageMap.from_json(result.payload))
+      end
+
+      # The parent's load lines merge into the freshly built map: the forked
+      # build child cannot inherit the parent's Coverage counters.
+      def with_parent_load_lines(map)
+        @load_lines.empty? ? map : map.with_load_lines(@load_lines)
       end
 
-      # A whole-suite build that records no line in any included file fails
-      # the run (exit 3): usually the setup hid the app from Coverage
-      # (Bootsnap's compile cache is the default Rails case); it can also
-      # mean no test in the suite exercises any included file. Either way a
-      # run on this map would test nothing, and exit 0 would lie. With
-      # explicit tests (--test) an empty map is a legitimate outcome
+      # A whole-suite build that records nothing - no line hit and no method
+      # call - in any included file fails the run (exit 3): usually no test
+      # in the suite exercises these files, or the setup hid the app from
+      # Coverage (Bootsnap's compile cache is the default Rails case). An
+      # all-endless-def file records method calls only and passes. Either way
+      # a run on a truly empty map would test nothing, and exit 0 would lie.
+      # With explicit tests (--test) an empty map is a legitimate outcome
       # instead: the user narrowed the suite to specs that may not touch
       # these sources, so the map passes with a warning. Applies to a cached
       # map too, so a map cached before this check cannot keep a run green,
       # and the guard runs before saving, so an empty map is never cached.
       def refuse_empty_map(map, files)
-        return map if files.empty? || map.lines.any? { |_file, by_line| by_line.any? }
+        return map if files.empty? || !map.empty?
 
         if @config["tests"].empty?
-          raise Error, "Coverage recorded no lines in #{files.size} included files during a whole-suite " \
-                       "build. Is Bootsnap's compile cache on? If not, no test may exercise these files. " \
-                       "Run `saboteur doctor`."
+          raise Error, "Coverage recorded nothing in #{files.size} included files during a whole-suite " \
+                       "build. No test in the suite may exercise these files, or the setup hid the app " \
+                       "from Coverage#{bootsnap_note}. Run `saboteur doctor`."
         end
-        @diagnostics.warn("Coverage recorded no lines in #{files.size} included files; " \
+        @diagnostics.warn("Coverage recorded nothing in #{files.size} included files; " \
                           "the selected specs may not exercise them.")
         map
       end
 
+      # Bootsnap is named only when it is loaded: blaming a library the app
+      # does not use sends the reader to the wrong place.
+      def bootsnap_note
+        " (Bootsnap's compile cache is the default Rails case)" if defined?(::Bootsnap)
+      end
+
       def died(status)
         how = if status&.signaled? then "was killed by signal #{Signal.signame(status.termsig)}"
               else "exited with status #{status&.exitstatus}"