saboteur 0.6.0 → 0.7.0

data/lib/saboteur/engine/verdict_check.rb

@@ -22,11 +22,14 @@ module Saboteur
         @baseline_timeout = baseline_timeout
       end
 
+      # Why a result the budget never let the check run is not a verdict.
+      BUDGET_REASON = "the --max-duration budget stopped the verdict check before it could run"
+
       def call(results)
         fresh = results.reject(&:reused?)
-        failing_runs = failing_sequences(fresh.select { |result| named_kill?(result) })
-        failing = failing_claims(fresh.select { |result| unnamed_kill?(result) })
-        results.map { |result| checked(result, failing_runs, failing) }
+        failing_runs, unchecked_runs = check_sequences(fresh.select { |result| named_kill?(result) })
+        failing, unchecked = check_claims(fresh.select { |result| unnamed_kill?(result) })
+        results.map { |result| checked(result, failing_runs, failing, unchecked_runs, unchecked) }
       end
 
       private
@@ -47,31 +50,64 @@ module Saboteur
         position ? result.covered_by.first(position + 1) : [result.killed_by.first]
       end
 
-      # Runs each distinct kill sequence once. Returns those that fail, each
-      # with why (the failure message, when there is one).
-      def failing_sequences(kills)
-        kills.map { |result| sequence(result) }.uniq.each_with_object({}) do |tests, failing|
+      # Runs each distinct kill sequence once. Returns [failing, unchecked]:
+      # failing maps a sequence to why it failed unchanged, unchecked maps
+      # the sequences the budget never let run to why - an unverified re-run
+      # is not a rejection (#296).
+      def check_sequences(kills)
+        failing = {}
+        unchecked = {}
+        kills.map { |result| sequence(result) }.uniq.each do |tests|
           outcome = @run_unchanged.call(tests, @baseline_timeout)
-          failing[tests] = outcome.message unless outcome.status == :survived
+          if outcome.status == :unverified
+            unchecked[tests] = outcome.message
+          elsif outcome.status != :survived
+            failing[tests] = outcome.message
+          end
         end
+        [failing, unchecked]
       end
 
-      # Kills with no failing test named. Returns those that fail unchanged.
-      def failing_claims(claims)
-        claims.reject do |result|
-          @run_unchanged.call(result.covered_by, @baseline_timeout).status == :survived
+      # Kills with no failing test named, checked on all their tests.
+      # Returns [failing, unchecked]: failing maps a rejected result to why
+      # it failed unchanged, unchecked maps a result the budget never let
+      # run to why.
+      def check_claims(claims)
+        failing = {}
+        unchecked = {}
+        claims.each do |result|
+          outcome = @run_unchanged.call(result.covered_by, @baseline_timeout)
+          if outcome.status == :unverified
+            unchecked[result] = outcome.message
+          elsif outcome.status != :survived
+            failing[result] = outcome.message
+          end
         end
+        [failing, unchecked]
       end
 
-      def checked(result, failing_runs, failing)
+      def checked(result, failing_runs, failing, unchecked_runs, unchecked)
         return result if result.reused?
+        return checked_named(result, failing_runs, unchecked_runs) if named_kill?(result)
+
+        if failing.key?(result)
+          why = failing[result]
+          result.as_error("tests also fail with the method unchanged#{": #{why}" if why}", rejected_kill: true)
+        elsif unchecked.key?(result)
+          result.as_unverified(unchecked[result] || BUDGET_REASON)
+        else
+          result
+        end
+      end
 
-        if named_kill?(result) && failing_runs.key?(sequence(result))
-          why = failing_runs[sequence(result)]
-          result.as_error("killing test #{sequence(result).last} also fails with the method unchanged" \
+      def checked_named(result, failing_runs, unchecked_runs)
+        tests = sequence(result)
+        if failing_runs.key?(tests)
+          why = failing_runs[tests]
+          result.as_error("killing test #{tests.last} also fails with the method unchanged" \
                           "#{": #{why}" if why}", rejected_kill: true)
-        elsif failing.include?(result)
-          result.as_error("tests also fail with the method unchanged", rejected_kill: true)
+        elsif unchecked_runs.key?(tests)
+          result.as_unverified(unchecked_runs[tests] || BUDGET_REASON)
         else
           result
         end

data/lib/saboteur/engine.rb

@@ -6,25 +6,31 @@ require "json"
 require_relative "replay"
 require_relative "mutation_source/hazard_guard"
 require_relative "baseline/run_evidence"
+require_relative "engine/early_stop_recording"
+require_relative "engine/run_budget"
+require_relative "engine/thread_warnings"
 
 module Saboteur
   # A single-use instance runs one pipeline: boot the target, select subjects, fetch the
   # coverage map, then schedule each subject's mutants. The parts live in
   # engine/; this class only wires them together.
   class Engine
+    include EarlyStopRecording
+    include RunBudget
+
     # Everything besides sources and tests that can change a verdict.
     VERDICT_INPUTS = %w[integration require rspec_args rails reeval sources operators exclude_operators arid
                         timeout_factor timeout_min
                         max_memory_mb max_processes seed tests whole_file].freeze
 
     attr_reader :config, :hooks
-    # The specs --guess-tests narrowed the run to, or nil when it did not.
+    # The specs inference narrowed the run to, or nil when it did not.
     attr_reader :guessed_tests
 
-    def initialize(config, err: $stderr, digest: nil, capture_evidence: false, evidence_files: [])
+    def initialize(config, err: $stderr, digest: nil, capture_evidence: false, evidence_files: [], verbose: false)
       @config = config
       @hooks = Hooks.new
-      @diagnostics = Diagnostics.new(err)
+      @diagnostics = Diagnostics.new(err, verbose: verbose)
       @root = File.realpath(config.root)
       @skipped_subjects = {}
       @replay = Replay.new(digest: digest) if digest
@@ -38,12 +44,16 @@ module Saboteur
     end
 
     def call
-      Boot.refuse_recursion!
-      refuse_fail_fast_evidence
+      preflight!
       started = monotonic
-      guess_tests
-      announce_operators
+      start_budget_clock(started)
+      # Capture evidence inputs before inference narrows the test list: the
+      # fingerprint must match what the baseline workflow recomputes from the
+      # declared config, and the inferred specs are pinned by the hashed
+      # source files anyway.
       @run_evidence = Baseline::RunEvidence.new(config: config, extra_files: @evidence_files) if @capture_evidence
+      select_tests
+      announce_operators
       ignore_list = @replay ? IgnoreList.new([]) : IgnoreList.load(config.path("ignore_file"))
       warn_custom_map
       boot = Boot.new(config, hooks)
@@ -52,29 +62,31 @@ module Saboteur
         @rails = boot.rails
         subjects, scope = select_subjects(files)
         results = subjects.empty? ? [] : mutate(subjects, files, integration, scope, ignore_list, load_lines:)
-        @diagnostics.warnings.concat(ignore_list.apply(results))
+        @diagnostics.warnings.concat(ignore_list.apply(results, unscoped_digests: @unscoped_digests))
         build_report(results, subjects, scope, started)
       ensure
-        # The run boundary for the per-worker databases: cleanup runs however
-        # the call ends - a boot that raised partway through, a baseline or
-        # coverage failure, no subjects, no runnable work - so a host that
-        # embeds Engine and calls it repeatedly never holds copies or
-        # snapshots until process exit. cleanup_workers is idempotent; the
-        # Rails profile's at_exit handler stays as the fallback for an
-        # abrupt process exit.
-        safely_cleanup_workers(boot.rails)
+        # Cleanup runs however the call ends - a partway boot failure, no
+        # subjects, no runnable work - so a host embedding Engine never
+        # holds worker copies until process exit. Idempotent; the Rails
+        # profile's at_exit handler stays the fallback for abrupt exit.
+        begin
+          @coverage_guard&.resume # an embedded host gets its coverage back (#254)
+        ensure
+          safely_cleanup_workers(boot.rails)
+        end
       end
     end
 
-    def warn_custom_map
-      @diagnostics.warnings.concat(custom_map_warnings(load_custom_map))
+    def preflight!
+      Boot.refuse_recursion!
+      refuse_fail_fast_evidence
     end
 
+    def warn_custom_map = @diagnostics.warnings.concat(custom_map_warnings(load_custom_map))
+
     # Stale entries and duplicate literals never fail a run; they warn so the
     # map does not rot silently. doctor fails on stale keys.
-    def load_custom_map
-      @replay ? CustomMap.new([]) : CustomMap.load(config.path("custom_map"), root: config.root)
-    end
+    def load_custom_map = @replay ? CustomMap.new([]) : CustomMap.load(config.path("custom_map"), root: config.root)
 
     def custom_map_warnings(map)
       path = config.path("custom_map")
@@ -86,29 +98,80 @@ module Saboteur
 
     private
 
-    # --guess-tests with --since: scope the run to the specs found next to
-    # the changed files, as --test would, but only when each changed file
-    # has one. Otherwise the whole suite runs, as without the flag.
-    def guess_tests
-      return unless config["guess_tests"]
-
-      guess = TestGuess.new(config).call
-      message =
-        if guess.changed.zero?
-          "--guess-tests: no changed source file to guess specs for, so the whole suite runs"
-        elsif guess.unmatched.any?
-          "--guess-tests: no spec found for #{guess.unmatched.join(', ')}, so the whole suite runs " \
-            "(name specs after their source files or pass --test)"
+    # Inferred test selection, the default since 0.7.0: scope the run to the
+    # specs found next to the mutated files, as --test would. With --since
+    # only files whose changed lines carry a mutation subject need a spec;
+    # without it every included file with a subject does. A file that needs
+    # a spec and has none stops the run before boot, rather than silently
+    # narrowing the source scope or widening the test run to the whole
+    # suite. --all and c.test_selection = :all keep the whole suite.
+    # A replay keeps its recorded tests, and a --subject run infers only
+    # from the named subjects' files, so one unrelated spec-less file can
+    # never block a one-method run.
+    def select_tests
+      return unless config["tests"].empty?
+      return if @replay || config["all"] || config["test_selection"] == "all"
+
+      subjects = config["subjects"]
+      return select_subject_tests(subjects) if subjects.any?
+
+      guess = TestGuess.new(config, diagnostics: @diagnostics).call
+      @diagnostics.inform(guess.subjectless_message(since: config["since"])) if guess.subjectless.any?
+      raise ConfigError, guess.unmatched_message if guess.unmatched.any?
+
+      @diagnostics.announce(selection_message(guess))
+    end
+
+    # A --subject run infers only from the files holding the named subjects:
+    # an unrelated spec-less file must not block a one-method run. A subject
+    # file without a conventional spec stops the run before boot, like any
+    # other unmatched file. With --since only named subjects inside the diff
+    # scope count: an unchanged, comment-only or otherwise pruned subject
+    # needs no spec and cannot stop the run either.
+    def select_subject_tests(patterns)
+      files = SubjectSelection.files_matching(@root, config.included_files, patterns)
+      guess = TestGuess.new(config, diagnostics: @diagnostics).call_for(files, patterns)
+      # No file in scope: the subjects are unknown (named later, in
+      # selection) or --since left every named subject outside the diff, so
+      # the run has nothing to mutate and needs no inferred spec.
+      return if guess.changed.zero?
+
+      raise ConfigError, guess.unmatched_message if guess.unmatched.any?
+
+      @diagnostics.announce(scope_to_guesses(guess))
+    end
+
+    def selection_message(guess)
+      if guess.changed.zero?
+        target = config["since"] ? "no changed source file" : "no source file"
+        "inferred specs: #{target} to infer a spec for, so the whole suite runs"
+      elsif guess.specs.empty?
+        if config["since"]
+          "inferred specs: no changed file has a mutation subject on its changed lines, " \
+            "so there is nothing to mutate"
         else
-          # The guesses now stand as explicit tests, like --test.
-          @config = Config.new(config.to_h.merge("tests" => guess.specs, "guess_tests" => false), root: config.root)
-          @guessed_tests = guess.specs
-          "--guess-tests: running only #{guess.specs.join(', ')}. A spec that reaches a changed file only " \
-            "through another file does not run, so a mutant only it would kill can survive, and a line " \
-            "no guessed spec executes is reported as NoCoverage"
+          "inferred specs: no source file has a mutation subject, so there is nothing to mutate"
         end
-      @diagnostics.warn(message)
-      @diagnostics.inform(message)
+      else
+        scope_to_guesses(guess)
+      end
+    end
+
+    # Stop before boot: name the spec-less files and the ways out. The
+    # hinted spec paths are the conventional ones - they do not exist (that
+    # is why the files are unmatched), so the message never prints them as
+    # a runnable --test command, and it never reconstructs the source
+    # scope: the user appends --test or --all to their own command. The
+    # example is still shell-safe: the ref is user input, so it is quoted
+    # and pinned to the flag with = (a ref like "-x" must not become a
+    # switch).
+    def scope_to_guesses(guess)
+      # The guesses now stand as explicit tests, like --test. guess_tests is
+      # dropped rather than set to false: an explicit false is the deprecated
+      # opt-out and would warn.
+      @config = config.with({ "tests" => guess.specs }, except: ["guess_tests"])
+      @guessed_tests = guess.specs
+      guess.scope_message(since: config["since"])
     end
 
     # The run's immutable report, built once the last result is in.
@@ -124,7 +187,7 @@ module Saboteur
                  omissions: @diagnostics.omissions, run_evidence: complete_evidence(results, subjects),
                  hazard_suppressions: @diagnostics.hazard_suppressions, early_stop: @early_stop,
                  matrix: config["matrix"], inline_suppressions: @diagnostics.inline_suppressions,
-                 operator_policy: config.operator_policy)
+                 operator_policy: config.operator_policy, discovery_failures: @diagnostics.discovery_failures)
     end
 
     def complete_evidence(results, subjects)
@@ -149,10 +212,13 @@ module Saboteur
     end
 
     def mutate(subjects, files, integration, scope, ignore_list, load_lines: {})
-      warn_about_unscoped_suite(integration)
-      check_baseline(integration)
-      map = CoverageStore.new(config: config, integration: integration, hooks: hooks, load_lines:,
-                              supervisor: supervisor(memory_factor: 4), diagnostics: @diagnostics).fetch(files)
+      store = coverage_store(integration, load_lines)
+      full_passes = config["baseline_passes"] + (store.reusable?(files) ? 0 : 1)
+      warn_unscoped(integration, full_passes)
+      check_baseline(integration, full_passes)
+      return [] if abandon_for_budget?(subjects)
+
+      map = store.fetch(files)
       inform_about_run(subjects, map)
       cache = result_cache
       scheduler = scheduler_for(map, integration, cache)
@@ -169,14 +235,17 @@ module Saboteur
       results = run_candidates(runnable_subjects, runnable_lists, scheduler, cache, ignore_list)
       cache&.save
       results
+    rescue BudgetPhaseTimeout => e
+      budget_phase_stop(e, subjects)
     end
 
     def run_candidates(subjects, lists, scheduler, cache, ignore_list)
       @diagnostics.start_progress(lists.sum(&:size))
       distribution = Distribution.new(jobs: config["jobs"], scheduler: scheduler, diagnostics: @diagnostics,
                                       cache: cache, on_start: method(:start_worker),
-                                      after_subject: canary_check, stop_after: fail_fast_check(ignore_list))
-      distribution.run(subjects, lists).tap { record_early_stop(distribution.unreached) }
+                                      after_subject: after_subject_hook, stop_after: stop_checks(ignore_list),
+                                      halt_check: config["max_duration"] ? method(:budget_exceeded?) : nil)
+      distribution.run(subjects, lists).tap { record_early_stop(distribution) }
     ensure
       @diagnostics.finish_progress
     end
@@ -189,16 +258,15 @@ module Saboteur
       raise ConfigError, "fail_fast cannot be used with baseline evidence: it certifies the whole selection"
     end
 
-    # Subjects that were selected but never ran, with candidates, are part of
-    # the report: the verdict covers less than the selection.
-    def record_early_stop(unreached)
-      return if unreached.empty?
-
-      @early_stop = EarlyStop.new(reason: "fail_fast", unreached: unreached.map(&:id))
-    end
-
     # --fail-fast: stop after the first subject that has a survivor the
     # ignore list does not accept. Granularity is one subject.
+    # --fail-fast and --max-duration both stop before the next subject; the
+    # distribution records whatever they leave unreached.
+    def stop_checks(ignore_list)
+      checks = [fail_fast_check(ignore_list), ->(_results) { budget_exceeded? }].compact
+      ->(results) { checks.any? { |check| check.call(results) } }
+    end
+
     def fail_fast_check(ignore_list)
       return nil unless config["fail_fast"]
 
@@ -211,38 +279,40 @@ module Saboteur
 
     def inform_about_run(subjects, map)
       @diagnostics.inform("#{subjects.size} subjects, #{map.tests.size} tests in coverage map")
-      warn_about_threads
+      ThreadWarnings.warn(@diagnostics)
     end
 
-    # A --since run without --test runs the loaded suite twice for the
-    # baselines; a cold coverage cache adds one more pass. Say so
-    # before those hours pass, with the way to scope it.
-    def warn_about_unscoped_suite(integration)
-      return unless config["since"] && config["tests"].empty?
+    def coverage_store(integration, load_lines)
+      CoverageStore.new(config: config, integration: integration, hooks: hooks, load_lines:,
+                        supervisor: supervisor(memory_factor: 4), diagnostics: @diagnostics,
+                        timeout_cap: method(:budget_capped_timeout))
+    end
 
-      message = "running the loaded suite unscoped: two baseline passes; coverage may add a pass " \
-                "(#{integration.test_files.size} spec files). " \
-                "Pass --test to run only the specs you name: faster, but a mutant of a changed method is then killed " \
-                "only by those specs, so specs elsewhere that call this code are not run. " \
-                "--guess-tests does the same with the specs named after the changed files, " \
-                "a guess that can miss specs elsewhere."
-      @diagnostics.warn(message)
-      @diagnostics.inform(message)
+    def check_baseline(integration, full_passes)
+      guard = BaselineGuard.new(config: config, integration: integration, hooks: hooks,
+                                supervisor: supervisor(memory_factor: 4), diagnostics: @diagnostics,
+                                full_passes: full_passes, timeout_cap: method(:budget_capped_timeout),
+                                between_passes: config["max_duration"] ? method(:budget_exceeded?) : nil)
+      guard.call
+      # Stability is the order comparison between completed checks: the cold
+      # coverage build is not a baseline pass, and a budget break shortens
+      # the loop, so certify only what actually ran.
+      @stable_suite = guard.completed_passes >= 2
     end
 
-    def check_baseline(integration)
-      BaselineGuard.new(config: config, integration: integration, hooks: hooks,
-                        supervisor: supervisor(memory_factor: 4), diagnostics: @diagnostics).call
-      @stable_suite = true
+    # A --id replay targets one known mutant; the whole-suite cost summary
+    # and its --test lecture do not apply.
+    def warn_unscoped(integration, full_passes)
+      UnscopedSuiteWarning.call(config:, diagnostics: @diagnostics, integration:, full_passes:) unless @replay
     end
 
     def scheduler_for(map, integration, cache)
       reevaluator = Execution::Reevaluator.new(config["reeval"], whole_file: config["whole_file"])
       runner = Execution::Runner.new(integration: integration, reevaluator: reevaluator, hooks: hooks,
                                      supervisor: supervisor, log_dir: fresh_log_dir, seed: config["seed"],
-                                     sandbox: sandbox_hook)
+                                     sandbox: sandbox_hook, canary: canary)
       Scheduler.new(config: config, runner: runner, map: map, reevaluator: reevaluator, diagnostics: @diagnostics,
-                    cache: cache)
+                    cache: cache, deadline: budget_deadline)
     end
 
     # The sandbox hook for each mutant child (#152); the policy and its
@@ -287,8 +357,17 @@ module Saboteur
 
     # Checks the tree after each subject's mutants; a vanished file means a
     # mutant deleted it, so abort with the most recent mutant's digest.
+    # Per finished subject: the canary, then the --verbose marks line that
+    # makes a survivor attributable to its method mid-run (#262).
+    def after_subject_hook
+      canary = canary_check
+      lambda do |subject, results|
+        canary.call(subject, results)
+        @diagnostics.subject_progress(subject, results)
+      end
+    end
+
     def canary_check
-      canary = Canary.new(@root, cache_dir: config["cache_dir"])
       lambda do |subject, results|
         last = results.reject(&:reused?).last || results.last
         digest = last&.mutation&.digest(subject_start: subject.start_byte,
@@ -297,6 +376,9 @@ module Saboteur
       end
     end
 
+    # Shared with the runner so the hazard report wins over a log loss (#283).
+    def canary = (@canary ||= Canary.new(@root, cache_dir: config["cache_dir"]))
+
     def candidate_lists(subjects, scope, map)
       return [@replay_candidate] if @replay
 
@@ -305,6 +387,7 @@ module Saboteur
                                          limit: config["max_mutants_per_method"], diagnostics: @diagnostics)
       subjects.map { |subject| selection.for(subject, scope: scope) }.tap do
         @skipped_subjects = selection.skipped_subjects
+        @unscoped_digests = selection.unscoped_digests
         if @run_evidence
           @run_evidence.plan(subjects, subjects.map { |subject| selection.selected.fetch(subject) })
           @evidence_planned = true
@@ -333,19 +416,6 @@ module Saboteur
       end
     end
 
-    # Threads do not survive fork, so name them: with the name the owner
-    # gave the thread, else the first frame of its backtrace, so the user
-    # can tell whether they matter (a connection pool reaper, Sidekiq, an
-    # SDK's background flusher).
-    def warn_about_threads
-      others = Thread.list - [Thread.current]
-      return if others.empty?
-
-      names = others.map { |thread| thread.name || thread.backtrace&.first || "unknown" }
-      list = names.join(", ")
-      @diagnostics.warn("#{others.size} other thread(s) alive at fork time; they do not run in children: #{list}")
-    end
-
     # The lines that ran while the app and spec files loaded: the executable
     # counts from its first line, so this snapshot holds the whole boot.
     # Read here, in the parent: a forked child does not inherit Coverage
@@ -377,22 +447,22 @@ module Saboteur
     # Loads the target and stops coverage, returning what the rest of the
     # run needs from that boot.
     def boot_target(boot)
+      @diagnostics.inform("loading the application")
       integration = boot.load_target
       files = config.included_files
       [integration, files, suspend_coverage(files)]
     end
 
     def suspend_coverage(files)
-      Coverage.suspend if Coverage.state == :running
+      (@coverage_guard = CoverageGuard.new(@diagnostics)).suspend
       capture_load_lines(files)
     end
 
-    def monotonic
-      Process.clock_gettime(Process::CLOCK_MONOTONIC)
-    end
+    def monotonic = Process.clock_gettime(Process::CLOCK_MONOTONIC)
   end
 end
 
+require_relative "engine/coverage_guard"
 require_relative "engine/diagnostics"
 require_relative "engine/boot"
 require_relative "engine/subject_selection"
@@ -402,6 +472,7 @@ require_relative "engine/test_guess"
 require_relative "engine/sandbox_policy"
 require_relative "engine/coverage_store"
 require_relative "engine/baseline_guard"
+require_relative "engine/unscoped_suite_warning"
 require_relative "engine/bytecode_equivalence"
 require_relative "engine/candidates"
 require_relative "engine/candidate_selection"

data/lib/saboteur/execution/protocol.rb

@@ -2,6 +2,7 @@
 # frozen_string_literal: true
 
 require "json"
+require_relative "../text"
 
 module Saboteur
   module Execution
@@ -31,7 +32,7 @@ module Saboteur
       # Events in order; lines that are not this protocol's are ignored.
       def parse(payload)
         payload.each_line.filter_map do |line|
-          message = JSON.parse(line)
+          message = JSON.parse(Text.utf8_strict(line))
           message if message.is_a?(Hash) && message["v"] == VERSION
         rescue JSON::ParserError
           nil