saboteur 0.4.0 → 0.6.0

data/lib/saboteur/engine/scheduler.rb

@@ -12,10 +12,15 @@ module Saboteur
     # (VerdictCheck). Each mutant runs only the tests that executed its edit,
     # likeliest killers first, with a timeout from their recorded durations.
     class Scheduler
+      WHOLE_FILE_HINT = "; add it to c.whole_file to re-evaluate the whole file"
+
       # A subject to run, with what its mutants need. killers counts the
       # tests that killed earlier mutants of this subject; slowdown is how
       # much slower than recorded its tests ran in a child (see calibrate).
-      Plan = Struct.new(:subject, :tests, :killers, :slowdown, keyword_init: true)
+      # wrapper names a known def-time wrapper (Execution::Wrapper), so the
+      # child re-evaluates it together with the def.
+      Plan = Struct.new(:subject, :tests, :killers, :slowdown, :wrapper,
+                        keyword_init: true)
 
       # Tests that killed more earlier mutants first; otherwise keep the order.
       def self.prioritize(tests, killers)
@@ -34,20 +39,24 @@ module Saboteur
       def call(subject, candidates)
         return [] if candidates.empty?
 
-        reason = subject.skip_reason || load_problem(subject)
+        reason, wrapper = subject.skip_reason ? [subject.skip_reason, nil] : load_problem(subject)
         return skip_all(candidates, subject, reason) if reason
+        if wrapper == :sorbet && !subject.sig_source
+          return skip_all(candidates, subject,
+                          "sorbet sig not found next to the def; cannot preserve runtime type checks")
+        end
 
         tests = @map.tests_for(subject)
-        return skip_all(candidates, subject, "no coverage") if tests.empty?
+        return unloadable(candidates, subject) if tests.empty?
 
-        plan = Plan.new(subject:, tests:, killers: Hash.new(0), slowdown: 1.0)
+        plan = Plan.new(subject:, tests:, killers: Hash.new(0), slowdown: 1.0, wrapper: wrapper)
         reused = all_reused(plan, candidates)
         return reused if reused
 
         results = run_candidates(candidates) do |mutation, text|
           reuse(plan, mutation, text) || execute(plan, mutation, text)
         end
-        checked(subject, results)
+        checked(plan, results)
       end
 
       # The per-mutant timeout: timeout_factor times what the tests took
@@ -64,9 +73,11 @@ module Saboteur
 
       # Verdicts that did not hold up are warned about once per subject;
       # only checked verdicts are stored for --incremental.
-      def checked(subject, results)
+      def checked(plan, results)
+        subject = plan.subject
         run_unchanged = lambda do |tests, timeout|
-          @runner.run(subject: subject, text: subject.source, test_ids: tests, timeout: timeout)
+          @runner.run(subject: subject, text: subject.source, test_ids: tests, timeout: timeout,
+                      eval_prefix: eval_prefix(plan))
                  .tap { |outcome| note_leftovers(subject, "verdict check", outcome) }
         end
         check = VerdictCheck.new(run_unchanged: run_unchanged, baseline_timeout: @config["baseline_timeout"])
@@ -100,7 +111,7 @@ module Saboteur
         return nil unless @cache
 
         tests = tests_for(plan, mutation)
-        return skip_all([[mutation, text]], plan.subject, "no coverage").first if tests.empty?
+        return uncoverable(plan, mutation, text) if tests.empty?
 
         hit = @cache.lookup(plan.subject, mutation, tests)
         return nil unless hit
@@ -112,14 +123,14 @@ module Saboteur
 
       def execute(plan, mutation, text)
         tests = tests_for(plan, mutation)
-        return skip_all([[mutation, text]], plan.subject, "no coverage").first if tests.empty?
+        return uncoverable(plan, mutation, text) if tests.empty?
 
-        timeout = timeout_for(tests, plan.slowdown)
-        outcome = @runner.run(subject: plan.subject, text: text, test_ids: tests, timeout: timeout, mutation: mutation)
-        outcome = calibrate(plan, mutation, text, tests, timeout) if outcome.status == :timeout
+        outcome = run_mutant(plan, mutation, text, tests)
+        note_unreachable(plan, outcome) if outcome.status == :unreachable
         result = MutantResult.new(subject: plan.subject, mutation: mutation, status: outcome.status,
                                   mutated_source: text, reason: outcome.message, runtime: outcome.runtime,
-                                  killed_by: outcome.failed_ids, covered_by: tests, kill_cause: outcome.kill_cause)
+                                  killed_by: outcome.failed_ids, covered_by: tests, kill_cause: outcome.kill_cause,
+                                  executed: outcome.executed_ids)
         outcome.failed_ids.each { |id| plan.killers[id] += 1 }
         note_leftovers(plan.subject, "#{mutation.operator} #{result.digest}", outcome)
         @diagnostics.progress(result)
@@ -130,8 +141,12 @@ module Saboteur
       # body, so it gets every test of the method), prior killers first, then
       # fastest-first.
       def tests_for(plan, mutation)
+        return self.class.prioritize(plan.tests, plan.killers) if mutation.coarse
+
         line_numbers = mutation.edits.flat_map { |edit| plan.subject.coverage_lines_for(edit) }.uniq
-        tests = plan.tests & @map.tests_for_lines(plan.subject.file, line_numbers)
+        tests = plan.tests & @map.tests_for_edit(plan.subject, line_numbers)
+        branch_tests = @map.tests_for_branches(plan.subject, mutation.edits)
+        tests &= branch_tests if branch_tests
         self.class.prioritize(tests, plan.killers)
       end
 
@@ -139,6 +154,52 @@ module Saboteur
         [@config["timeout_factor"] * runtime, @config["timeout_min"]].max
       end
 
+      # What a subject no test ran gets. Code that only ran while the app
+      # loaded is not "no coverage" - it ran - but no forked test can re-run
+      # it, so it gets its own status, out of the score, with a hint.
+      def unloadable(candidates, subject, line_numbers = nil)
+        return skip_all(candidates, subject, "no coverage") unless
+          @map.ran_at_load?(subject.file, line_numbers || subject.coverage_lines)
+
+        candidates.map do |mutation, text|
+          MutantResult.new(subject: subject, mutation: mutation, status: :ran_at_load, mutated_source: text,
+                           reason: Selection::CoverageMap::RAN_AT_LOAD)
+        end
+      end
+
+      # One mutation whose edited lines no test ran (see unloadable).
+      def uncoverable(plan, mutation, text)
+        line_numbers = mutation.edits.flat_map { |edit| plan.subject.coverage_lines_for(edit) }.uniq
+        result, = unloadable([[mutation, text]], plan.subject, line_numbers)
+        return result if result.status == :ran_at_load
+        return result unless @map.tests_for_branches(plan.subject, mutation.edits) == []
+
+        skipped, = skip_all([[mutation, text]], plan.subject, "no coverage (branch not taken)")
+        skipped
+      end
+
+      def run_mutant(plan, mutation, text, tests)
+        timeout = timeout_for(tests, plan.slowdown)
+        outcome = @runner.run(subject: plan.subject, text: text, test_ids: tests, timeout: timeout,
+                              mutation: mutation, eval_prefix: eval_prefix(plan))
+        outcome.status == :timeout ? calibrate(plan, mutation, text, tests, timeout) : outcome
+      end
+
+      # Each mutant is checked separately: changing a body can change which
+      # definition a test reaches, so one unreachable verdict cannot skip later work.
+      def note_unreachable(plan, outcome)
+        @diagnostics.warn("#{plan.subject.id}: #{outcome.message}")
+      end
+
+      # The text a known def-time wrapper needs re-evaluated with the def.
+      # Sorbet's sig sits on the lines right before it; without it the def
+      # evaluates alone and runtime type checks silently drop for the child.
+      def eval_prefix(plan)
+        return nil unless plan.wrapper == :sorbet
+
+        plan.subject.sig_source
+      end
+
       # Recorded durations come from one warm process; a fresh child can be
       # slower (lazy loading, first queries). So a timeout is checked at
       # once: the unchanged method runs the same tests with
@@ -147,7 +208,7 @@ module Saboteur
       # the measured slowdown. A timeout that remains is the mutant's own.
       def calibrate(plan, mutation, text, tests, timeout)
         unchanged = @runner.run(subject: plan.subject, text: plan.subject.source, test_ids: tests,
-                                timeout: @config["baseline_timeout"])
+                                timeout: @config["baseline_timeout"], eval_prefix: eval_prefix(plan))
         note_leftovers(plan.subject, "timeout check", unchanged)
         return unchanged_failed(unchanged) unless unchanged.status == :survived
 
@@ -156,7 +217,8 @@ module Saboteur
         longer = deadline(unchanged.runtime)
         return Execution::Runner::Outcome.new(status: :timeout, runtime: timeout) if longer <= timeout
 
-        @runner.run(subject: plan.subject, text: text, test_ids: tests, timeout: longer, mutation: mutation)
+        @runner.run(subject: plan.subject, text: text, test_ids: tests, timeout: longer, mutation: mutation,
+                    eval_prefix: eval_prefix(plan))
       end
 
       def unchanged_failed(unchanged)
@@ -167,18 +229,28 @@ module Saboteur
       end
 
       # The method saboteur would redefine must be the one Ruby is running.
+      # Returns [skip reason, wrapper]: a known def-time wrapper (sorbet's
+      # sig is one, see Execution::Wrapper) is not a shadowing - the child
+      # re-evaluates it together with the def, so the check passes.
       def load_problem(subject)
+        return [nil, nil] if @reevaluator.whole_file?(subject)
+
         owner = @reevaluator.resolve_owner(subject)
         target = subject.singleton ? owner.singleton_class : owner
         method = own_method(target, subject.name)
-        return "method not loaded" unless method
+        return ["method not loaded", nil] unless method
 
         file, line = method.source_location
-        return nil if file && Selection::CoverageMap.canonical(file) == subject.file && line == subject.start_line
+        if file && Selection::CoverageMap.canonical(file) == subject.file && line == subject.start_line
+          return [nil, nil]
+        end
+
+        wrapper = Execution::Wrapper.around(method, file)
+        return [nil, wrapper] if wrapper
 
-        "shadowed definition (runtime method at #{file}:#{line})"
+        ["shadowed definition (runtime method at #{file}:#{line})#{WHOLE_FILE_HINT}", nil]
       rescue Error => e
-        "owner not loaded: #{e.message}"
+        ["owner not loaded: #{e.message}#{WHOLE_FILE_HINT}", nil]
       end
 
       def own_method(target, name)

data/lib/saboteur/engine/subject_selection.rb

@@ -33,7 +33,9 @@ module Saboteur
       def find(files)
         finder = SubjectFinder.new
         files.flat_map do |file|
+          InlineSuppression.new(file, File.read(file))
           parsed = finder.call(file)
+          @diagnostics.record_discovery_failure(file, reason: "invalid_file_syntax") unless parsed.errors.empty?
           parsed.errors.first(1).each do |message|
             @diagnostics.warn("#{file.delete_prefix("#{@root}/")} does not parse: #{message}")
           end
@@ -54,9 +56,30 @@ module Saboteur
       def by_change(subjects, ref)
         @changed_lines = Selection::GitDiff.changed_lines(ref, root: @root)
                                            .transform_keys { |file| Selection::CoverageMap.canonical(file) }
-        subjects.select do |subject|
+        changed = subjects.select do |subject|
           @changed_lines[subject.file]&.any? { |line| subject.contains_line?(line) }
         end
+        drop_unchanged_syntax(changed, ref)
+      end
+
+      # A method whose only changes are comments, blank lines or spacing has
+      # nothing new to test.
+      def drop_unchanged_syntax(subjects, ref)
+        return subjects if subjects.empty?
+
+        comparison = UnchangedSyntax.new(Selection::GitDiff.merge_base(ref, root: @root), root: @root)
+        same, kept = subjects.partition { |subject| comparison.unchanged?(subject) }
+        report_unchanged_syntax(same)
+        kept
+      end
+
+      def report_unchanged_syntax(subjects)
+        return if subjects.empty?
+
+        names = subjects.first(5).map(&:id).join(", ")
+        more = subjects.size > 5 ? ", ..." : ""
+        @diagnostics.inform("#{subjects.size} changed method(s) differ only in comments or formatting, " \
+                            "so they are skipped: #{names}#{more}")
       end
     end
   end

data/lib/saboteur/engine/test_guess.rb

@@ -0,0 +1,89 @@
+# SPDX-License-Identifier: MIT
+# frozen_string_literal: true
+
+module Saboteur
+  class Engine
+    # --guess-tests: the specs that sit next to the changed source files by
+    # naming convention (lib/a/b.rb -> spec/a/b_spec.rb). A guess, not a
+    # coverage fact: a spec that only reaches the file through another one
+    # is not found. Packwerk packs (a directory with a package.yml) map to the
+    # spec directory inside the same pack, and controllers also map to request
+    # specs. The run scopes to the guesses only when every changed file has one.
+    class TestGuess
+      Result = Data.define(:specs, :unmatched, :changed)
+
+      SOURCE_ROOTS = %w[lib app].freeze
+
+      def initialize(config)
+        @root = File.realpath(config.root)
+        @config = config
+      end
+
+      # specs: existing spec files (relative paths); unmatched: changed
+      # source files with no guessed spec; changed: how many source files
+      # changed.
+      def call
+        changed = changed_sources
+        guesses = changed.to_h { |file| [file, guess_for(file)] }
+        Result.new(specs: guesses.values.compact.uniq.sort, unmatched: guesses.select { |_, spec| spec.nil? }.keys,
+                   changed: changed.size)
+      end
+
+      private
+
+      # Changed files that saboteur would mutate, relative to the root.
+      def changed_sources
+        included = @config.included_files.to_set
+        changed = Selection::GitDiff.changed_lines(@config["since"], root: @root).keys
+        changed.map { |file| Selection::CoverageMap.canonical(file) }
+               .select { |file| included.include?(file) }
+               .map { |file| file.delete_prefix("#{@root}/") }.sort
+      end
+
+      def guess_for(file)
+        candidates_for(file).find { |path| File.file?(File.join(@root, path)) }
+      end
+
+      # Candidate specs for a source file, most specific first: the nearest
+      # package.yml pack's own spec directory, then the app-wide one.
+      def candidates_for(file)
+        pack = pack_of(file)
+        # pack is nil for files outside packs: "#{nil}/" is "/", which
+        # never prefixes a relative path, so delete_prefix is a no-op there.
+        base = file.delete_prefix("#{pack}/")
+        spec_root = pack ? "#{pack}/spec" : "spec"
+        stem = base.delete_suffix(".rb")
+        rest = stem.split("/", 2)
+        rooted = rest.size == 2 && SOURCE_ROOTS.include?(rest.first)
+        names = rooted ? [rest.last, "lib/#{rest.last}", stem] : [stem]
+        names = names.flat_map { |name| [name, *request_names(name)] }.uniq
+        names.map { |name| "#{spec_root}/#{name}_spec.rb" }
+      end
+
+      # Rails tests controllers as request specs: spec/requests/users_spec.rb
+      # or spec/requests/users_controller_spec.rb for controllers/users_controller.rb.
+      def request_names(name)
+        return [] unless name.start_with?("controllers/")
+
+        path = name.delete_prefix("controllers/")
+        ["requests/#{path}", "requests/#{path.delete_suffix('_controller')}"].uniq
+      end
+
+      # The nearest ancestor directory holding a package.yml (a packwerk
+      # pack), relative to the root. The root itself is not a pack here:
+      # its specs follow the plain mapping.
+      def pack_of(file)
+        dir = File.dirname(file)
+        until dir == "."
+          return dir if pack_dirs.include?(dir)
+
+          dir = File.dirname(dir)
+        end
+      end
+
+      def pack_dirs
+        @pack_dirs ||= Dir.glob("**/package.yml", base: @root).to_set { |path| File.dirname(path) }
+      end
+    end
+  end
+end

data/lib/saboteur/engine/unchanged_syntax.rb

@@ -0,0 +1,87 @@
+# SPDX-License-Identifier: MIT
+# frozen_string_literal: true
+
+require "prism"
+require_relative "../inline_suppression"
+
+module Saboteur
+  class Engine
+    # Tells whether a method still has, in the working tree, the syntax it had
+    # at the merge base of the --since ref. A change to a comment, a blank
+    # line or spacing leaves it the same, so no mutant of it asks a question
+    # the change raised. Comparison is of the Prism tree without locations
+    # and comments, plus the frozen_string_literal magic comment and the
+    # `sig` in front of the def, which both change what runs. `__LINE__` is
+    # the one node whose value is its location, so it keeps its line: a
+    # method that moved, or one with a line added above its `__LINE__`, is
+    # not the same.
+    #
+    # Anything that cannot be compared says the method changed: a file or
+    # method that is new at the base, a base that does not parse, a method
+    # name defined twice in a file.
+    class UnchangedSyntax
+      def initialize(base, root:)
+        @base = base
+        @root = root
+        @base_subjects = {}
+        @current_subjects = {}
+      end
+
+      def unchanged?(subject)
+        before = base_subject(subject)
+        return false unless before
+
+        return false unless suppression_signature(before) == suppression_signature(subject)
+
+        before.frozen_literals == subject.frozen_literals && before.sig_source == subject.sig_source &&
+          self.class.digest(before.node) == self.class.digest(subject.node)
+      end
+
+      # The node as plain data: type and fields, never locations (except
+      # the line a `__LINE__` stands for).
+      def self.digest(value)
+        case value
+        when Prism::SourceLineNode then [value.type, value.location.start_line]
+        when Prism::Node
+          fields = value.deconstruct_keys(nil).except(:node_id)
+          [value.type, fields.transform_values { |field| digest(field) }]
+        when Array then value.map { |item| digest(item) }
+        when Prism::Location then nil
+        else value
+        end
+      end
+
+      private
+
+      def suppression_signature(subject)
+        InlineSuppression.new(subject.file, subject.file_source).signature(subject)
+      end
+
+      def base_subject(subject)
+        return nil unless current_subjects(subject).key?(subject.id)
+
+        base_subjects(subject.file)[subject.id]
+      end
+
+      # Only the names defined once: with two defs of a name, which one a
+      # change belongs to is a guess.
+      def unique_by_id(source, file)
+        subjects = SubjectFinder.new.call(file, source: source).subjects
+        counts = subjects.map(&:id).tally
+        subjects.select { |subject| counts[subject.id] == 1 }.to_h { |subject| [subject.id, subject] }
+      end
+
+      def current_subjects(subject)
+        file = subject.file
+        @current_subjects.fetch(file) { @current_subjects[file] = unique_by_id(subject.file_source, file) }
+      end
+
+      def base_subjects(file)
+        @base_subjects.fetch(file) do
+          text = Selection::GitDiff.file_at(@base, file.delete_prefix("#{@root}/"), root: @root)
+          @base_subjects[file] = text ? unique_by_id(text, file) : {}
+        end
+      end
+    end
+  end
+end

data/lib/saboteur/engine/verdict_check.rb

@@ -5,7 +5,7 @@ module Saboteur
   class Engine
     # Checks each kill against the method re-evaluated unchanged: a kill
     # stands only if the same tests, in the same order, up to and including
-    # the killer, pass without the mutant (so a test that fails only after
+    # the last killer, pass without the mutant (so a test that fails only after
     # another test left state behind cannot claim a kill). A kill with no
     # failing test named (an exit, a suite-level failure) is checked on all
     # the mutant's tests. A kill that fails the check becomes an error.
@@ -39,11 +39,12 @@ module Saboteur
         result.status == :killed && result.killed_by.empty?
       end
 
-      # The tests the mutant ran, in order, up to and including the killer.
+      # The tests the mutant ran, in order, up to and including the last
+      # killer: a matrix run lists every failing test, and each of them must
+      # pass under the unchanged method in this same sequence.
       def sequence(result)
-        killer = result.killed_by.first
-        position = result.covered_by.index(killer)
-        position ? result.covered_by.first(position + 1) : [killer]
+        position = result.killed_by.filter_map { |test| result.covered_by.index(test) }.max
+        position ? result.covered_by.first(position + 1) : [result.killed_by.first]
       end
 
       # Runs each distinct kill sequence once. Returns those that fail, each
@@ -67,7 +68,7 @@ module Saboteur
 
         if named_kill?(result) && failing_runs.key?(sequence(result))
           why = failing_runs[sequence(result)]
-          result.as_error("killing test #{result.killed_by.first} also fails with the method unchanged" \
+          result.as_error("killing test #{sequence(result).last} also fails with the method unchanged" \
                           "#{": #{why}" if why}", rejected_kill: true)
         elsif failing.include?(result)
           result.as_error("tests also fail with the method unchanged", rejected_kill: true)