saboteur 0.5.0 → 0.7.0

data/lib/saboteur/engine/test_guess.rb

@@ -1,51 +1,211 @@
 # SPDX-License-Identifier: MIT
 # frozen_string_literal: true
 
+require "pathname"
+
 module Saboteur
   class Engine
-    # --guess-tests: the specs that sit next to the changed source files by
+    # The specs that sit next to the mutated source files by
     # naming convention (lib/a/b.rb -> spec/a/b_spec.rb). A guess, not a
     # coverage fact: a spec that only reaches the file through another one
-    # is not found. The run scopes to the guesses only when every changed
-    # file has one.
+    # is not found. Packwerk packs (a directory with a package.yml) map to the
+    # spec directory inside the same pack, and controllers also map to request
+    # specs. With --since, only files whose changed lines carry a mutation
+    # subject need a guessed spec; without --since, every included file with
+    # a mutation subject needs one. A file that needs a spec and has none
+    # stops the run before boot.
     class TestGuess
-      Result = Data.define(:specs, :unmatched, :changed)
+      # hints: unmatched file -> its conventional spec path (which does
+      # not exist; that is why the file is unmatched), for the stop-before-boot
+      # error. The result also builds its own messages, so a stubbed guess in
+      # the engine's specs formats exactly like a real one.
+      Result = Data.define(:specs, :unmatched, :subjectless, :changed, :hints) do
+        # The stop-before-boot usage error: names each expected spec and the
+        # ways out as an append-only instruction - never a reconstructed
+        # command, which could carry a hostile ref or a wrong source scope.
+        def unmatched_message
+          pairs = unmatched.map { |file| "#{file} (expected #{hints.fetch(file)})" }
+          "no spec found for #{pairs.join(', ')}, so nothing ran. Re-run the same command with --test " \
+            "followed by your existing spec paths, or append --all to use the whole suite without " \
+            "changing the source scope."
+        end
+
+        # Why the subjectless files needed no spec.
+        def subjectless_message(since:)
+          detail = since ? "no changed mutation subjects" : "no mutation subjects"
+          "inferred specs: no spec is required for #{subjectless.join(', ')}: they have #{detail}"
+        end
+
+        # The warning for a guess-scoped run: the guessed specs now stand as
+        # explicit tests, with the cross-file coverage caveat that brings.
+        def scope_message(since:)
+          what = since ? "a changed file" : "a source file"
+          "inferred specs: running only #{specs.join(', ')}. A spec that reaches " \
+            "#{what} only through another file does not run, so a mutant only it would kill can " \
+            "survive, and a line no inferred spec executes is reported as NoCoverage"
+        end
+      end
 
       SOURCE_ROOTS = %w[lib app].freeze
 
-      def initialize(config)
+      def initialize(config, diagnostics: nil)
+        @diagnostics = diagnostics
         @root = File.realpath(config.root)
         @config = config
       end
 
-      # specs: existing spec files (relative paths); unmatched: changed
-      # source files with no guessed spec; changed: how many source files
-      # changed.
       def call
-        changed = changed_sources
-        guesses = changed.to_h { |file| [file, guess_for(file)] }
-        Result.new(specs: guesses.values.compact.uniq.sort, unmatched: guesses.select { |_, spec| spec.nil? }.keys,
-                   changed: changed.size)
+        @config["since"] ? call_diff : call_all
+      end
+
+      # Subject-scoped inference (--subject runs): only the named subjects'
+      # files need a guessed spec; every one of them has a subject by
+      # construction, so there is no subjectless bucket. With --since the
+      # diff narrows the scope further: a named subject whose file is
+      # unchanged, whose changed lines sit in other methods, or whose change
+      # is comment or formatting only is out of scope and needs no spec.
+      def call_for(files, patterns = [])
+        files = effective_subject_files(files, patterns) if @config["since"]
+        guesses = files.to_h { |file| [file, guess_for(file)] }
+        result(guesses, [], files.size)
+      end
+
+      # specs: existing spec files (relative paths) guessed for the files in
+      # scope; unmatched: such files with no guessed spec; subjectless: files
+      # with no mutation subject the run checks, which neither need nor get a
+      # spec; changed: how many source files are in scope; hints: each
+      # unmatched file's conventional spec path.
+      def call_diff
+        lines_by_file = changed_lines_by_file
+        subjectless = subjectless_files(lines_by_file)
+        # Only files with a changed mutation subject get a guessed spec: a
+        # subjectless file mutates nothing, so its spec would only narrow the
+        # run for no reason.
+        guesses = lines_by_file.except(*subjectless).to_h { |file| [file, guess_for(file)] }
+        result(guesses, subjectless, lines_by_file.size)
+      end
+
+      # Without --since every included file is in scope: a file needs its
+      # guessed spec unless it holds no mutation subject at all.
+      def call_all
+        finder = SubjectFinder.new
+        guesses = {}
+        subjectless = []
+        included_files.each do |file|
+          if finder.call(File.join(@root, file)).subjects.empty?
+            subjectless << file
+          else
+            guesses[file] = guess_for(file)
+          end
+        end
+        result(guesses, subjectless.sort, guesses.size + subjectless.size)
       end
 
       private
 
-      # Changed files that saboteur would mutate, relative to the root.
-      def changed_sources
+      # The named subjects' files inside the diff scope: the subject must
+      # sit on a changed line and survive the comment/formatting prune,
+      # mirroring the run's subject selection.
+      def effective_subject_files(files, patterns)
+        lines_by_file = changed_lines_by_file
+        comparison = UnchangedSyntax.new(Selection::GitDiff.merge_base(@config["since"], root: @root), root: @root)
+        finder = SubjectFinder.new
+        files.select do |file|
+          lines = lines_by_file[file]
+          next false unless lines
+
+          finder.call(File.join(@root, file)).subjects.any? do |subject|
+            patterns.any? { |pattern| SubjectSelection.match?(subject.id, pattern) } &&
+              subject.overlaps_lines?(lines) &&
+              !comparison.unchanged?(subject)
+          end
+        end
+      end
+
+      def result(guesses, subjectless, changed)
+        unmatched = guesses.select { |_, spec| spec.nil? }.keys
+        Result.new(specs: guesses.values.compact.uniq.sort,
+                   unmatched: unmatched,
+                   subjectless: subjectless,
+                   changed: changed,
+                   hints: unmatched.to_h { |file| [file, candidates_for(file).first] })
+      end
+
+      # Included files relative to the root, sorted for stable output.
+      def included_files
+        prefix = "#{@root}/"
+        @config.included_files.filter_map { |file| file.delete_prefix(prefix) if file.start_with?(prefix) }.sort
+      end
+
+      # Changed files that saboteur would mutate, relative to the root, with
+      # their changed line numbers.
+      def changed_lines_by_file
         included = @config.included_files.to_set
-        changed = Selection::GitDiff.changed_lines(@config["since"], root: @root).keys
-        changed.map { |file| Selection::CoverageMap.canonical(file) }
-               .select { |file| included.include?(file) }
-               .map { |file| file.delete_prefix("#{@root}/") }.sort
+        Selection::GitDiff.changed_lines(@config["since"], root: @root, include: included,
+                                                           diagnostics: @diagnostics)
+                          .each_with_object({}) do |(file, lines), map|
+          canonical = Selection::CoverageMap.canonical(file)
+          map[canonical.delete_prefix("#{@root}/")] = lines if included.include?(canonical)
+        end.sort.to_h
+      end
+
+      # Changed files whose changed lines carry no mutation subject: the run
+      # mutates nothing in them, so they cannot require a spec. Mirrors the
+      # run's subject selection, including its comment/formatting-only skip:
+      # a deletion whose remaining context lines are blank or comments, or a
+      # comment-only change inside a method, leaves the file subjectless.
+      def subjectless_files(lines_by_file)
+        comparison = UnchangedSyntax.new(Selection::GitDiff.merge_base(@config["since"], root: @root), root: @root)
+        finder = SubjectFinder.new
+        lines_by_file.reject do |file, lines|
+          subjects = finder.call(File.join(@root, file)).subjects
+          subjects.select { |subject| subject.overlaps_lines?(lines) }
+                  .any? { |subject| !comparison.unchanged?(subject) }
+        end.keys
       end
 
       def guess_for(file)
-        stem = file.delete_suffix(".rb")
+        candidates_for(file).find { |path| File.file?(File.join(@root, path)) }
+      end
+
+      # Candidate specs for a source file, most specific first: the nearest
+      # package.yml pack's own spec directory, then the app-wide one.
+      def candidates_for(file)
+        pack = pack_of(file)
+        # pack is nil for files outside packs: "#{nil}/" is "/", which
+        # never prefixes a relative path, so delete_prefix is a no-op there.
+        base = file.delete_prefix("#{pack}/")
+        spec_root = pack ? "#{pack}/spec" : "spec"
+        stem = base.delete_suffix(".rb")
         rest = stem.split("/", 2)
-        candidates = ["spec/#{stem}_spec.rb"]
-        candidates.unshift("spec/#{rest.last}_spec.rb", "spec/lib/#{rest.last}_spec.rb") if
-          rest.size == 2 && SOURCE_ROOTS.include?(rest.first)
-        candidates.find { |path| File.file?(File.join(@root, path)) }
+        rooted = rest.size == 2 && SOURCE_ROOTS.include?(rest.first)
+        names = rooted ? [rest.last, "lib/#{rest.last}", stem] : [stem]
+        names = names.flat_map { |name| [name, *request_names(name)] }.uniq
+        names.map { |name| "#{spec_root}/#{name}_spec.rb" }
+      end
+
+      # Rails tests controllers as request specs: spec/requests/users_spec.rb
+      # or spec/requests/users_controller_spec.rb for controllers/users_controller.rb.
+      def request_names(name)
+        return [] unless name.start_with?("controllers/")
+
+        path = name.delete_prefix("controllers/")
+        ["requests/#{path}", "requests/#{path.delete_suffix('_controller')}"].uniq
+      end
+
+      # The nearest ancestor directory holding a package.yml (a packwerk
+      # pack), relative to the root. The root itself is not a pack here:
+      # Pathname#ascend stops above "." for these relative paths.
+      def pack_of(file)
+        Pathname(file).ascend do |path|
+          dir = path.to_s
+          return dir if pack_dirs.include?(dir)
+        end
+        nil
+      end
+
+      def pack_dirs
+        @pack_dirs ||= Dir.glob("**/package.yml", base: @root).to_set { |path| File.dirname(path) }
       end
     end
   end

data/lib/saboteur/engine/thread_warnings.rb

@@ -0,0 +1,53 @@
+# SPDX-License-Identifier: MIT
+# frozen_string_literal: true
+
+module Saboteur
+  class Engine
+    # Threads do not survive fork, so a run names the ones alive when the
+    # workers fork: by the name the owner gave the thread, else the first
+    # frame of its backtrace, so the user can tell whether they matter (a
+    # connection pool reaper, Sidekiq, an SDK's background flusher).
+    # Known-harmless threads say so inline; an unknown thread gets the
+    # generic caveat (#263).
+    module ThreadWarnings
+      # Well-known fork-time threads, with the condition each assurance
+      # hangs on. Evidence: rails-mode children reconnect the database
+      # (lib/saboteur/frameworks/rails.rb, after_fork); whether the debug
+      # gem's console also attaches to forked children depends on its
+      # fork_mode, whose default is :both, i.e. attach (RUBY_DEBUG_FORK_MODE;
+      # ruby/debug's config.rb and SESSION.activate on_fork: true in
+      # session.rb).
+      KNOWN_FORK_THREADS = {
+        "AR Pool Reaper" => "ActiveRecord's connection-pool reaper: expected harmless when children " \
+                            "reconnect the database after fork (rails mode does) - it only maintains " \
+                            "the parent's pool",
+        "DEBUGGER__::SESSION@server" => "the debug gem's remote-console server: what it does in " \
+                                        "children depends on RUBY_DEBUG_FORK_MODE - with parent no " \
+                                        "debugger runs in children; with child or both (the gem's " \
+                                        "default) the debugger activates in children and can pause or " \
+                                        "interfere with supervised mutation runs. For an unattended " \
+                                        "run, do not start the remote debugger at all: drop " \
+                                        "RUBY_DEBUG_OPEN or require \"debug\" without :open"
+      }.freeze
+
+      def self.warn(diagnostics)
+        others = Thread.list - [Thread.current]
+        return if others.empty?
+
+        unknown = false
+        list = others.map do |thread|
+          name = thread.name || thread.backtrace&.first || "unknown"
+          explanation = KNOWN_FORK_THREADS[name]
+          unknown = true unless explanation
+          explanation ? "#{name} (#{explanation})" : name
+        end.join(", ")
+        message = "#{others.size} other thread(s) alive at fork time; they do not run in children: #{list}"
+        if unknown
+          message += "; a thread that does not run in a child cannot do background work a mutant's test may " \
+                     "rely on (a pool reaper, a flusher), so results can differ when one is left behind"
+        end
+        diagnostics.warn(message)
+      end
+    end
+  end
+end

data/lib/saboteur/engine/unscoped_suite_warning.rb

@@ -0,0 +1,41 @@
+# SPDX-License-Identifier: MIT
+# frozen_string_literal: true
+
+module Saboteur
+  class Engine
+    # A whole-suite run (--all, c.test_selection = :all, or a --since diff
+    # with nothing to infer from) runs the loaded suite for the baseline
+    # checks; a cold coverage cache adds one more pass. Say so - with the
+    # example count and the real pass count - before those hours pass; past
+    # a size threshold, add the way to scope it (#260). The check count
+    # comes from c.baseline_passes (#55).
+    module UnscopedSuiteWarning
+      # A suite this size pays hours for an unscoped run; below it the cost
+      # summary alone is enough, without the --test lecture (#260).
+      COSTLY_SUITE_EXAMPLES = 1_000
+      REUSED_MAP_NOTE = "; the cached coverage map is reused"
+
+      def self.call(config:, diagnostics:, integration:, full_passes:)
+        return unless config["tests"].empty?
+        return unless config["since"] || config["all"] || config["test_selection"] == "all"
+
+        examples = integration.all_tests.size
+        coverage = full_passes == config["baseline_passes"] ? REUSED_MAP_NOTE : " + 1 coverage map build"
+        message = "running the loaded suite unscoped: #{examples} examples in #{integration.test_files.size} " \
+                  "spec files; #{full_passes} full passes (#{checks_label(config)}#{coverage})."
+        if examples >= COSTLY_SUITE_EXAMPLES
+          message += " Pass --test to run only the specs you name: faster, but a mutant of a changed method is then " \
+                     "killed only by those specs, so specs elsewhere that call this code are not run. " \
+                     "The default inferred selection runs the specs named after the mutated files, " \
+                     "a guess that can miss specs elsewhere."
+        end
+        diagnostics.warn(message)
+        diagnostics.inform(message)
+      end
+
+      def self.checks_label(config)
+        "#{config['baseline_passes']} baseline check#{'s' unless config['baseline_passes'] == 1}"
+      end
+    end
+  end
+end

data/lib/saboteur/engine/verdict_check.rb

@@ -22,11 +22,14 @@ module Saboteur
         @baseline_timeout = baseline_timeout
       end
 
+      # Why a result the budget never let the check run is not a verdict.
+      BUDGET_REASON = "the --max-duration budget stopped the verdict check before it could run"
+
       def call(results)
         fresh = results.reject(&:reused?)
-        failing_runs = failing_sequences(fresh.select { |result| named_kill?(result) })
-        failing = failing_claims(fresh.select { |result| unnamed_kill?(result) })
-        results.map { |result| checked(result, failing_runs, failing) }
+        failing_runs, unchecked_runs = check_sequences(fresh.select { |result| named_kill?(result) })
+        failing, unchecked = check_claims(fresh.select { |result| unnamed_kill?(result) })
+        results.map { |result| checked(result, failing_runs, failing, unchecked_runs, unchecked) }
       end
 
       private
@@ -47,31 +50,64 @@ module Saboteur
         position ? result.covered_by.first(position + 1) : [result.killed_by.first]
       end
 
-      # Runs each distinct kill sequence once. Returns those that fail, each
-      # with why (the failure message, when there is one).
-      def failing_sequences(kills)
-        kills.map { |result| sequence(result) }.uniq.each_with_object({}) do |tests, failing|
+      # Runs each distinct kill sequence once. Returns [failing, unchecked]:
+      # failing maps a sequence to why it failed unchanged, unchecked maps
+      # the sequences the budget never let run to why - an unverified re-run
+      # is not a rejection (#296).
+      def check_sequences(kills)
+        failing = {}
+        unchecked = {}
+        kills.map { |result| sequence(result) }.uniq.each do |tests|
           outcome = @run_unchanged.call(tests, @baseline_timeout)
-          failing[tests] = outcome.message unless outcome.status == :survived
+          if outcome.status == :unverified
+            unchecked[tests] = outcome.message
+          elsif outcome.status != :survived
+            failing[tests] = outcome.message
+          end
         end
+        [failing, unchecked]
       end
 
-      # Kills with no failing test named. Returns those that fail unchanged.
-      def failing_claims(claims)
-        claims.reject do |result|
-          @run_unchanged.call(result.covered_by, @baseline_timeout).status == :survived
+      # Kills with no failing test named, checked on all their tests.
+      # Returns [failing, unchecked]: failing maps a rejected result to why
+      # it failed unchanged, unchecked maps a result the budget never let
+      # run to why.
+      def check_claims(claims)
+        failing = {}
+        unchecked = {}
+        claims.each do |result|
+          outcome = @run_unchanged.call(result.covered_by, @baseline_timeout)
+          if outcome.status == :unverified
+            unchecked[result] = outcome.message
+          elsif outcome.status != :survived
+            failing[result] = outcome.message
+          end
         end
+        [failing, unchecked]
       end
 
-      def checked(result, failing_runs, failing)
+      def checked(result, failing_runs, failing, unchecked_runs, unchecked)
         return result if result.reused?
+        return checked_named(result, failing_runs, unchecked_runs) if named_kill?(result)
+
+        if failing.key?(result)
+          why = failing[result]
+          result.as_error("tests also fail with the method unchanged#{": #{why}" if why}", rejected_kill: true)
+        elsif unchecked.key?(result)
+          result.as_unverified(unchecked[result] || BUDGET_REASON)
+        else
+          result
+        end
+      end
 
-        if named_kill?(result) && failing_runs.key?(sequence(result))
-          why = failing_runs[sequence(result)]
-          result.as_error("killing test #{sequence(result).last} also fails with the method unchanged" \
+      def checked_named(result, failing_runs, unchecked_runs)
+        tests = sequence(result)
+        if failing_runs.key?(tests)
+          why = failing_runs[tests]
+          result.as_error("killing test #{tests.last} also fails with the method unchanged" \
                           "#{": #{why}" if why}", rejected_kill: true)
-        elsif failing.include?(result)
-          result.as_error("tests also fail with the method unchanged", rejected_kill: true)
+        elsif unchecked_runs.key?(tests)
+          result.as_unverified(unchecked_runs[tests] || BUDGET_REASON)
         else
           result
         end