saboteur 0.12.1 → 0.12.2

data/lib/saboteur/engine/pipelines.rb

@@ -104,9 +104,16 @@ module Saboteur
       # "Full run" for ignore staleness (#1206): no --since diff, no --subject
       # filter, and no narrowed include (explicit file arguments replace the
       # configured include) - only then was every in-scope subject examined, so
-      # an entry whose subject never appears really points at gone code.
+      # an entry whose subject never appears really points at gone code. The
+      # include nobody set counts as unscoped whether it is the generic
+      # default or the Rails one a config-less app gets (#1255).
       def unscoped_full_run?
-        !config["since"] && config["subjects"].empty? && config["include"] == Config::DEFAULTS["include"]
+        !config["since"] && config["subjects"].empty? && default_include?
+      end
+
+      def default_include?
+        config["include"] == Config::DEFAULTS["include"] ||
+          (config["include"] == Config::RAILS_INCLUDE && !config.explicit?("include"))
       end
 
       def external_command? = !config["test_command"].to_s.empty?

data/lib/saboteur/engine/survivor_confirmation.rb

@@ -15,13 +15,14 @@ module Saboteur
       # confirmation_budget caps the whole pass.
       MAX_SPECS_PER_SURVIVOR = 8
 
-      def initialize(config:, map:, scheduler:, diagnostics:, budget_exceeded:, cache: nil)
+      def initialize(config:, map:, scheduler:, diagnostics:, budget_exceeded:, cache: nil, load_specs: ->(_files) {})
         @config = config
         @map = map
         @scheduler = scheduler
         @diagnostics = diagnostics
         @budget_exceeded = budget_exceeded
         @cache = cache
+        @load_specs = load_specs
       end
 
       def call(results, ignore_list:, early_stop: nil)
@@ -33,19 +34,60 @@ module Saboteur
                             "stopped early (#{early_stop.reason})")
         end
         @remaining = @config["confirmation_budget"]
+        @map_silent = 0
         results.map { |result| survivors.include?(result) ? confirm(result) : result }
                .tap { |confirmed| summarize(confirmed) }
+               .tap { warn_map_silent }
       end
 
       private
 
+      # A crash while confirming one survivor downgrades it to unconfirmed
+      # with one warning; it never ends the run (#1140).
       def confirm(result)
+        confirm_checked(result)
+      rescue RefusedSpecPath => e
+        warn_refused(e)
+        result.with_confirmation("unconfirmed_map(a covering spec from the whole-suite map was refused)")
+      rescue StandardError => e
+        warn_failed(result, e)
+        unconfirmed(result, "confirmation failed: #{e.class}")
+      end
+
+      # The map named a spec that is not a spec file inside the project: the
+      # map cannot be trusted, so its survivors stay counted (#1286).
+      def warn_refused(error)
+        return if @warned_refused
+
+        @warned_refused = true
+        @diagnostics.warn("survivor confirmation refused the whole-suite coverage map: " \
+                          "#{error.message.lines.first.strip}; its survivors stay counted; " \
+                          "run `saboteur run --all` to rebuild it")
+      end
+
+      # One line per run, however many survivors hit the same failure.
+      def warn_failed(result, error)
+        return if @warned_failure
+
+        @warned_failure = true
+        @diagnostics.warn("survivor confirmation failed for #{result.subject.id} #{result.digest} " \
+                          "(#{error.class}: #{error.message.lines.map(&:strip).reject(&:empty?).first}); " \
+                          "its survivors stay unconfirmed, outside the score")
+      end
+
+      def confirm_checked(result)
         return unconfirmed(result, "the confirmation budget is spent") if @remaining <= 0
         return unconfirmed(result, "the --max-duration budget is spent") if @budget_exceeded.call
 
-        tests = unrun_covering_tests(result)
-        return unconfirmed(result, "no covering tests") if tests.empty?
+        # Nothing in the whole-suite map covers the edited lines (the tree
+        # drifted since the map was built): the map cannot confirm anything.
+        covering = covering_tests(result)
+        return map_silent(result) if covering.empty?
 
+        tests = unrun_covering_tests(result, covering)
+        return result.with_confirmation("confirmed") if tests.empty?
+
+        load_unselected_specs(tests)
         outcome = run_confirmation(result, tests)
         @remaining -= outcome.runtime || 0
         case outcome.status
@@ -58,8 +100,14 @@ module Saboteur
       # The whole-suite map's covering tests for the mutant's edited lines
       # (branch evidence intersected, mirroring the scheduler), minus the
       # tests the mutant already ran, fastest first, capped.
-      def unrun_covering_tests(result)
-        fastest_first(covering_tests(result) - (result.executed || result.covered_by)).first(MAX_SPECS_PER_SURVIVOR)
+      def unrun_covering_tests(result, covering)
+        fastest_first(covering - (result.executed || result.covered_by)).first(MAX_SPECS_PER_SURVIVOR)
+      end
+
+      # The covering specs may live in files this run never loaded; the
+      # forked confirmation child can only run examples already in its world.
+      def load_unselected_specs(tests)
+        @load_specs.call(tests.map { |id| id.sub(/\[[\d:]+\]\z/, "") })
       end
 
       def covering_tests(result)
@@ -86,11 +134,12 @@ module Saboteur
       # becomes an error; a check the budgets never let run comes back
       # unverified - outside the score and never cached.
       def checked_kill(result, tests, outcome)
-        kill = result.with_confirmation("confirmed", status: :killed, reason: outcome.message,
-                                                     killed_by: outcome.failed_ids, covered_by: tests,
-                                                     kill_cause: outcome.kill_cause,
-                                                     runtime: (result.runtime || 0) + (outcome.runtime || 0),
-                                                     executed: (result.executed || []) + outcome.executed_ids)
+        kill = result.with_confirmation(
+          "killed_by_confirmation", status: :killed, reason: outcome.message, killed_by: outcome.failed_ids,
+                                    covered_by: tests, kill_cause: outcome.kill_cause,
+                                    runtime: (result.runtime || 0) + (outcome.runtime || 0),
+                                    executed: (result.executed || []) + outcome.executed_ids
+        )
         checked = VerdictCheck.new(run_unchanged: unchanged_runner(result.subject),
                                    baseline_timeout: @config["baseline_timeout"]).call([kill]).first
         return checked unless checked.status == :killed
@@ -108,7 +157,7 @@ module Saboteur
       # (#1079). ResultCache#record replaces the subject+mutation entry, so
       # the stale survivor verdict cannot win a later lookup.
       def record_kill(result, kill)
-        @cache&.record(kill.with_confirmation("confirmed", covered_by: covering_tests(result)))
+        @cache&.record(kill.with_confirmation("killed_by_confirmation", covered_by: covering_tests(result)))
       end
 
       # The unchanged-method re-run VerdictCheck checks a kill against, in
@@ -129,6 +178,13 @@ module Saboteur
         end
       end
 
+      # The map says nothing about the edited lines (the tree drifted since it
+      # was built): the survivor stays counted and the pass says so once.
+      def map_silent(result)
+        @map_silent += 1
+        result.with_confirmation("unconfirmed_map(no covering tests in the whole-suite map)")
+      end
+
       def unconfirmed(result, reason)
         result.with_confirmation("unconfirmed(#{reason})")
       end
@@ -148,6 +204,14 @@ module Saboteur
         durations.empty? ? nil : durations.sum / durations.size
       end
 
+      def warn_map_silent
+        return if @map_silent.zero?
+
+        @diagnostics.warn("survivor confirmation: the whole-suite coverage map covers none of the edited lines of " \
+                          "#{@map_silent} survivor#{'s' unless @map_silent == 1}, so they stay counted; " \
+                          "run `saboteur run --all` to rebuild it")
+      end
+
       def summarize(results)
         counts = results.filter_map(&:confirmation).tally
         return if counts.empty?

data/lib/saboteur/engine/test_guess.rb

@@ -116,7 +116,8 @@ module Saboteur
         # Why the subjectless files needed no spec.
         def subjectless_message(since:)
           detail = since ? "no changed mutation subjects" : "no mutation subjects"
-          "inferred specs: no spec is required for #{names(subjectless)}: they have #{detail}"
+          who = subjectless.size == 1 ? "it has" : "they have"
+          "inferred specs: no spec is required for #{names(subjectless)}: #{who} #{detail}"
         end
 
         # The per-file warning when the run proceeds with the mapped

data/lib/saboteur/engine.rb

@@ -31,11 +31,6 @@ module Saboteur
     include GatedRun
     include SelectionAnnouncement
 
-    # Confirmation only guards a partial coverage map, so it is silent unless
-    # the user opted into partial_coverage = use (#1228).
-    CONFIRMATION_SKIPPED = "survivor confirmation skipped: partial_coverage = use applies only to a cached " \
-                           "whole-suite run (no --test, one coverage part)"
-
     # Everything besides sources and tests that can change a verdict.
     VERDICT_INPUTS = %w[integration require rspec_args rails reeval sources operators exclude_operators arid
                         timeout_factor timeout_min
@@ -130,6 +125,12 @@ module Saboteur
       @coverage_store&.whole_suite_map
     end
 
+    # True when this run did not build a whole-suite map: a diff, named
+    # subjects, an explicit --test or inferred specs (#1286).
+    def scoped_selection?
+      !whole_suite_run?
+    end
+
     # False when caching is off, where no coverage build can ever leave a
     # whole-suite map to consult, so advising one would be wrong (review C1).
     def whole_suite_map_lookup?
@@ -292,15 +293,21 @@ module Saboteur
       confirm_survivors(run_candidates(subjects, lists, scheduler, cache, ignore_list), scheduler, cache, ignore_list)
     end
 
+    # Confirm each live survivor against the whole-suite coverage map (#1286):
+    # the covering specs the run's own selection never loaded get one budgeted
+    # run. Scoped runs (--since, --subject, pr) are the point, so nothing here
+    # depends on partial_coverage or on the run being unscoped. Without a
+    # whole-suite map there is nothing to confirm against; the run says so
+    # elsewhere (survivors unconfirmed: no whole-suite coverage map).
     def confirm_survivors(results, scheduler, cache, ignore_list)
-      unless CoveragePartial.use?(config, config["coverage_parts"])
-        live = results.any? { |result| result.survived? && !ignore_list.accepts?(result) }
-        @diagnostics.inform(CONFIRMATION_SKIPPED) if live && config["partial_coverage"] == "use"
-        return results
-      end
+      return results if results.none? { |result| result.survived? && !ignore_list.accepts?(result) }
 
-      SurvivorConfirmation.new(config: config, map: scheduler.map, scheduler: scheduler, diagnostics: @diagnostics,
-                               budget_exceeded: method(:budget_exceeded?), cache: cache)
+      whole = whole_suite_coverage_map
+      return results if whole.nil?
+
+      SurvivorConfirmation.new(config: config, map: whole, scheduler: scheduler, diagnostics: @diagnostics,
+                               budget_exceeded: method(:budget_exceeded?), cache: cache,
+                               load_specs: ->(files) { scheduler.runner.integration.load_spec_files(files) })
                           .call(results, ignore_list: ignore_list, early_stop: @early_stop)
     end
 
@@ -350,10 +357,19 @@ module Saboteur
     # baseline's own chatter (demoted to --verbose) would start.
     def announce_quiet_start(store, files)
       @map_cached = store.reusable?(files)
-      @diagnostics.quiet_start_header(scope: config["since"] ? "--since #{config['since']}" : "whole project",
+      @diagnostics.quiet_start_header(scope: quiet_scope,
                                       seed: config["seed"], operators: config.operator_policy["effective"].size)
     end
 
+    # Names what the run covers: the diff, the --subject filter, or the
+    # whole project (P6) - a --subject run is not "whole project".
+    def quiet_scope
+      parts = []
+      parts << "--since #{config['since']}" if config["since"]
+      parts << "--subject #{config['subjects'].join(', ')}" if config["subjects"].any?
+      parts.empty? ? "whole project" : parts.join(", ")
+    end
+
     def inform_about_run(subjects, map)
       if @diagnostics.quiet_start && !@diagnostics.verbose
         @diagnostics.quiet_baseline(passes: @completed_baseline_passes, elapsed: @baseline_elapsed,
@@ -367,11 +383,13 @@ module Saboteur
     end
 
     # This run's map is the whole-suite map only when nothing scoped it:
-    # no --since, no subjects, and tests either unset or inferred over every
-    # included file (never an explicit --test list) (review C1).
+    # no --since, no subjects, and every spec file loaded (--all or
+    # c.test_selection = :all). Inferred specs are a selection, never the
+    # whole suite: a spec with no matching source file is not in the map
+    # (#1286).
     def whole_suite_run?
       config["since"].nil? && config["subjects"].empty? &&
-        (config["tests"].empty? || !@guessed_tests.nil?)
+        config["tests"].empty? && @guessed_tests.nil?
     end
 
     def coverage_store(integration, load_lines, load_methods = {})

data/lib/saboteur/execution/runner.rb

@@ -74,6 +74,9 @@ module Saboteur
       # sandbox: when given, a callable run first in every child (Landlock
       # makes the project tree read-only there, #152). It raises rather than
       # run half-restricted, which the child reports as an error.
+      # Survivor confirmation loads the spec files a scoped run left out (#1286).
+      attr_reader :integration
+
       def initialize(integration:, reevaluator:, log_dir:, hooks: Hooks.new, supervisor: Supervisor.new, seed: 1,
                      sandbox: nil, canary: nil, crash_log: nil)
         @integration = integration

data/lib/saboteur/gates.rb

@@ -205,7 +205,7 @@ module Saboteur
 
     # A subject fails a gate when at least one of its mutants survived.
     def failing_subjects(scored)
-      scored.select { |result| result.status == :survived }.map(&:subject).uniq.size
+      scored.select(&:survived?).map(&:subject).uniq.size
     end
 
     def over?(count, limit, executed)

data/lib/saboteur/integration/rspec.rb

@@ -124,6 +124,36 @@ module Saboteur
         world.example_groups.map { |group| File.expand_path(group.metadata[:file_path]) }.uniq
       end
 
+      # Loads spec files the run's own selection left out, in the parent, so a
+      # forked confirmation run can execute their examples (#1286). Files
+      # already loaded are skipped; a file that defines no group is an error,
+      # since its examples could never run.
+      def load_spec_files(paths)
+        loaded = test_files
+        absolute = paths.map { |path| File.expand_path(path, config.root) }.uniq
+        (absolute - loaded).each do |path|
+          require_inside_root(path)
+          before = world.example_groups.size
+          from_directory(config.root) { load(path) }
+          raise RefusedSpecPath, "#{path} defines no example group" if world.example_groups.size == before
+        end
+      end
+
+      # The ids come from a cached coverage map, which is untrusted input
+      # (#427): only a spec file that really lives under the project root,
+      # symlinks resolved, may be loaded.
+      def require_inside_root(path)
+        raise RefusedSpecPath, "refusing to load #{path}: not a *_spec.rb file" unless path.end_with?("_spec.rb")
+
+        root = File.realpath(config.root)
+        real = File.realpath(path)
+        return if real.start_with?("#{root}#{File::SEPARATOR}") && real.end_with?("_spec.rb")
+
+        raise RefusedSpecPath, "refusing to load #{path}: it resolves outside the project root"
+      rescue SystemCallError
+        raise RefusedSpecPath, "refusing to load #{path}: it does not exist under the project root"
+      end
+
       # Runs the tests in the order given: groups by their earliest example,
       # examples by position.
       def run(ids)

data/lib/saboteur/reporter/default_findings.rb

@@ -57,7 +57,7 @@ module Saboteur
       def list_lines(report, live)
         groups(report, live).map do |(path, line, subject_id), results|
           first, *rest = results
-          label = SurvivorClassification.label(first)
+          label = SurvivorClassification.group_label(results)
           text = "#{path}:#{line} #{subject_id}#{" [#{label}]" if label} - #{first.digest}"
           rest.any? ? "#{text} (+#{rest.size} more)" : text
         end

data/lib/saboteur/reporter/gap_kinds.rb

@@ -58,6 +58,12 @@ module Saboteur
 
       FALLBACK = "result unchecked"
 
+      # Which kind names a statement when its surviving mutants disagree
+      # (#P4): the first one listed that any of them carries. The broadest
+      # unchecked thing wins, so a branch condition is not called a
+      # boundary because one of its mutants happened to flip a comparison.
+      PRECEDENCE = ["result unchecked", "error path", "branch", "boundary", "effect", "number"].freeze
+
       # The deterministic per-kind reason printed next to a survivor's rank.
       REASONS = {
         "boundary" => "the covering specs pass without pinning this boundary",

data/lib/saboteur/reporter/next_steps.rb

@@ -20,7 +20,9 @@ module Saboteur
       end
 
       def call(report)
-        survivor = report.unignored_survivors.first
+        # The replay example is the #1 survivor of the ranked list, not
+        # whichever one the results hold first.
+        survivor = SurvivorRanking.ordered(report, report.unignored_survivors).first
         return unless survivor
 
         @out.puts "\n#{@palette.paint(:head, 'Next steps:')}"

data/lib/saboteur/reporter/score_table.rb

@@ -15,7 +15,6 @@ module Saboteur
     # it. Untested subjects are never dropped silently, even at 100 %.
     class ScoreTable
       MAX_ROWS = 15
-      MAX_UNTESTED_NAMES = 3
       # The skip reason CandidateSelection gives a subject no test covers.
       NO_COVERAGE = "no coverage"
 
@@ -63,7 +62,7 @@ module Saboteur
           detected: breakdown.detected, tested: breakdown.tested, untested: untested }
       end
 
-      # "cart.rb  100.0% (2/2) · n/a - 2 subjects - no test executes them: ...": the
+      # "cart.rb  100.0% (2/2) · n/a - 2 subjects - no test executes them": the
       # tested score and the untested count stand next to each other; a
       # file with nothing tested gets no percentage at all.
       def line(row, width)
@@ -75,18 +74,14 @@ module Saboteur
         note ? "#{base} #{@palette.glyph(:middot)} #{note}" : base
       end
 
-      # "n/a - 2 subjects - no test executes them: Cart#total, Cart#biggest":
-      # names the subjects the score cannot speak for, capped so a long
-      # list does not swallow the row; the Not scored block lists them all.
-      # Wording owns the claim (#1145): a stubbed-away subject has tests,
-      # they just never execute it.
+      # "n/a - 2 subjects - no test executes them": only the count. The
+      # Not scored block below names every subject once (P6), so the table
+      # does not repeat the names. Wording owns the claim (#1145): a
+      # stubbed-away subject has tests, they just never execute it.
       def untested_note(untested)
         return if untested.empty?
 
-        names = untested.map(&:id)
-        shown = names.first(MAX_UNTESTED_NAMES)
-        shown << "+#{names.size - MAX_UNTESTED_NAMES} more" if names.size > MAX_UNTESTED_NAMES
-        @palette.paint(:warn, "n/a - #{Wording.untested_note(untested.size)}: #{shown.join(', ')}")
+        @palette.paint(:warn, "n/a - #{Wording.untested_note(untested.size)}")
       end
     end
   end

data/lib/saboteur/reporter/stryker.rb

@@ -98,7 +98,7 @@ module Saboteur
       end
 
       def self.test_index(results)
-        names = results.flat_map { |result| result.covered_by + result.killed_by + result.executed.to_a }.uniq
+        names = results.flat_map { |result| result.covered_by + result.killed_by + result.executed.to_a }.uniq.sort
         names.each_with_index.to_h { |name, index| [name, index.to_s] }
       end
 
@@ -139,6 +139,10 @@ module Saboteur
           # Why the survivor is worth a look, as a label only: scores and
           # the exit code never change (#192).
           classification: SurvivorClassification.label(result),
+          # Additive (#1286): confirmed, killed_by_confirmation or unconfirmed.
+          # An unconfirmed survivor reports Ignored: outside the score.
+          confirmation: result.confirmation_state,
+          creditedTest: result.credited_test&.then { |name| test_ids.fetch(name) },
           killedBy: result.killed_by.map { |name| test_ids.fetch(name) }.then { |ids| ids.empty? ? nil : ids },
           coveredBy: result.covered_by.map { |name| test_ids.fetch(name) }.then { |ids| ids.empty? ? nil : ids },
           # Which of the covering tests actually ran against the mutant; absent
@@ -151,7 +155,7 @@ module Saboteur
       def self.status(result)
         case result.status
         when :killed then "Killed"
-        when :survived then "Survived"
+        when :survived then result.unconfirmed? ? "Ignored" : "Survived"
         when :timeout then "Timeout"
         when :error, :resource_limit then "RuntimeError"
         when :skipped then result.reason&.start_with?("no coverage") ? "NoCoverage" : "Ignored"
@@ -169,6 +173,7 @@ module Saboteur
         # ignore, and only wont_fix adds a suffix after the reason.
         return "ignored: #{result.ignored} (wont_fix)" if result.triage_state == "wont_fix"
         return "ignored: #{result.ignored}" if result.ignored
+        return "unconfirmed: #{result.unconfirmed_reason}" if result.unconfirmed?
         return "resource limit: #{result.reason}" if result.status == :resource_limit
         return result.crash_kill? ? "crash: #{result.kill_cause}" : result.kill_cause if result.kill_cause
 

data/lib/saboteur/reporter/survivor_classification.rb

@@ -55,6 +55,21 @@ module Saboteur
           nil
         end
 
+        # The label of one statement's surviving mutants as a whole, the
+        # same whichever mutant sorts first. Any gap kind decides, by
+        # GapKinds::PRECEDENCE, so one equivalent, logging or input_gap
+        # sibling never hides a real survivor. With no gap kind, the first
+        # label present stands: likely_equivalent, arid or tool_limit then
+        # describe the statement when all its mutants carry it.
+        #
+        # Among gap kinds, result unchecked wins because it is the strictly
+        # larger gap: nothing pins the statement's value at all, which
+        # subsumes a missing branch, boundary or effect check on it.
+        def group_label(results)
+          labels = results.map { |result| label(result) }.compact
+          GapKinds::PRECEDENCE.find { |kind| labels.include?(kind) } || labels.first
+        end
+
         private
 
         # The mutant's own def text, compiled side by side with the

data/lib/saboteur/reporter/survivor_list.rb

@@ -77,8 +77,8 @@ module Saboteur
         line = MutationText.line(first)
         path = relative(first.subject.file, report.root)
         @out.puts "\n#{@palette.paint(:head, "#{rank}. #{path}:#{line}")} #{first.subject.id} " \
-                  "#{@palette.paint(:dim, label_suffix(first, rank, total))}".rstrip
-        reason = SurvivorHints.label_reason(@classifier.label(first))
+                  "#{@palette.paint(:dim, label_suffix(group, rank, total))}".rstrip
+        reason = SurvivorHints.label_reason(@classifier.group_label(group))
         @out.puts "  #{@palette.paint(:dim, reason)}" if reason
         fence(first, line)
         @out.puts "  #{@palette.paint(:dim, "why it matters: #{why(first.mutation.operator)}")}"
@@ -105,10 +105,10 @@ module Saboteur
       # siblings read "#3 of 14 blocks", never blocks against mutants (#1147).
       # Label and rank stay on the header line; the reason moves to its own
       # dim line so the fixed text cannot push the header past 80 columns.
-      def label_suffix(result, rank, total)
-        label = @classifier.label(result)
+      def label_suffix(group, rank, total)
+        label = @classifier.group_label(group)
         unit = total == 1 ? "block" : "blocks"
-        parts = [("coarse" if result.mutation.coarse), label, "##{rank} of #{total} #{unit}"].compact
+        parts = [("coarse" if group.first.mutation.coarse), label, "##{rank} of #{total} #{unit}"].compact
         "[#{parts.join(' · ')}]"
       end
 
@@ -226,7 +226,7 @@ module Saboteur
 
         whole = @whole_suite_map&.tests_for(group.first.subject)
         if whole.nil?
-          @out.puts @palette.paint(:dim, "  covering tests unknown - run a full coverage build")
+          @out.puts @palette.paint(:dim, "  covering list is this run's selection only; run `saboteur run --all`")
           return
         end
 

data/lib/saboteur/reporter/terminal.rb

@@ -6,6 +6,7 @@ require_relative "wording"
 require_relative "baseline_tail"
 require_relative "next_steps"
 require_relative "kill_causes"
+require_relative "unconfirmed_section"
 require_relative "survivor_ranking"
 require_relative "palette"
 require_relative "notice"
@@ -62,6 +63,7 @@ module Saboteur
         return quiet_call(report) if @quiet
 
         findings(report)
+        UnconfirmedSection.new(@out, @palette).call(report)
         errors(report)
         skipped_subjects(report) if @verbose
         test_command_gaps(report)
@@ -94,7 +96,7 @@ module Saboteur
 
       def quiet_survivor_line(path, line, subject_id, results)
         first, *rest = results
-        label = SurvivorClassification.label(first)
+        label = SurvivorClassification.group_label(results)
         text = "#{path}:#{line} #{subject_id}#{" [#{label}]" if label} - #{first.digest}"
         rest.any? ? "#{text} (+#{rest.size} more)" : text
       end
@@ -346,7 +348,7 @@ module Saboteur
         untested = untested_subjects(report).size
         [counted(counts[:killed], "killed", plural: false, color: :pass),
          counted(live, "survived", plural: false, color: :fail),
-         counted(report.survivors.size - live, "accepted", plural: false),
+         counted(report.survivors.size - live, "accepted", plural: false), UnconfirmedSection.part(report),
          counted(counts[:timeout], "timeout", color: :warn), counted(counts[:resource_limit], "resource limit"),
          error_part(report, counts[:error]), counted(counts[:unverified], "unverified", plural: false),
          ("#{untested} subject#{'s' unless untested == 1} untested" if untested.positive?)].compact

data/lib/saboteur/reporter/unconfirmed_section.rb

@@ -0,0 +1,44 @@
+# SPDX-License-Identifier: MIT
+# frozen_string_literal: true
+
+require_relative "mutation_text"
+require_relative "survivor_ranking"
+
+module Saboteur
+  module Reporter
+    # Survivors whose covering tests were not all run (#1286): outside the
+    # score and the exit code, so they get their own short list instead of
+    # a place among the findings.
+    class UnconfirmedSection
+      LIMIT = 10
+
+      # The count for the one-line (--quiet) tally, nil when there is none.
+      def self.part(report)
+        count = report.unconfirmed_survivors.size
+        "#{count} unconfirmed" if count.positive?
+      end
+
+      def initialize(out, palette)
+        @out = out
+        @palette = palette
+      end
+
+      def call(report)
+        pending = report.unconfirmed_survivors
+        return if pending.empty?
+
+        @out.puts @palette.paint(:head, "unconfirmed (covering tests not run) - #{pending.size}, not in the score")
+        SurvivorRanking.ordered(report, pending).first(LIMIT).each { |result| @out.puts line(report, result) }
+        @out.puts "  ... #{pending.size - LIMIT} more" if pending.size > LIMIT
+        @out.puts
+      end
+
+      private
+
+      def line(report, result)
+        path = result.subject.file.delete_prefix("#{report.root}/")
+        "  #{path}:#{MutationText.line(result)} #{result.subject.id} - #{result.digest} (#{result.unconfirmed_reason})"
+      end
+    end
+  end
+end

data/lib/saboteur/result.rb

@@ -112,8 +112,37 @@ module Saboteur
       @digest ||= mutation.digest(subject_start: subject.start_byte, original_source: subject.file_source)
     end
 
+    # A survivor whose confirmation run could not happen (budget spent, or the
+    # run failed) is outside the score and the exit code until it is
+    # confirmed (#1286). A survivor the whole-suite map cannot speak for
+    # ("unconfirmed_map(...)") stays counted: a stale map never lowers the gate.
     def survived?
-      status == :survived
+      status == :survived && !unconfirmed?
+    end
+
+    def unconfirmed?
+      status == :survived && @confirmation.to_s.start_with?("unconfirmed(")
+    end
+
+    # The additive report field: confirmed, killed_by_confirmation or
+    # unconfirmed; nil when confirmation never applied (#1286).
+    def confirmation_state
+      return nil if @confirmation.nil?
+
+      label = @confirmation.to_s
+      label.start_with?("unconfirmed") ? "unconfirmed" : label
+    end
+
+    # The test a confirmation kill is credited to.
+    def credited_test
+      killed_by.first if confirmation_state == "killed_by_confirmation"
+    end
+
+    # Why the survivor stayed unconfirmed: the text inside unconfirmed(...).
+    def unconfirmed_reason
+      return unless unconfirmed?
+
+      @confirmation.delete_prefix("unconfirmed(").delete_suffix(")")
     end
 
     # Killed because a test's error, not its check, fired: NoMethodError and
@@ -280,7 +309,9 @@ module Saboteur
     end
 
     def counts
-      STATUSES.to_h { |status| [status, results.count { |result| result.status == status }] }
+      STATUSES.to_h do |status|
+        [status, results.count { |result| status == :survived ? result.survived? : result.status == status }]
+      end
     end
 
     # (killed + timeout) / (killed + survived + timeout); nil with nothing tested.
@@ -308,6 +339,12 @@ module Saboteur
       results.select(&:survived?)
     end
 
+    # Survivors whose covering tests were not all run: outside the score and
+    # the exit code, listed on their own (#1286).
+    def unconfirmed_survivors
+      results.select(&:unconfirmed?)
+    end
+
     def unignored_survivors
       results.select(&:unignored_survivor?)
     end