saboteur 0.12.0 → 0.12.2

data/lib/saboteur/engine.rb

@@ -31,11 +31,6 @@ module Saboteur
     include GatedRun
     include SelectionAnnouncement
 
-    # Confirmation only guards a partial coverage map, so it is silent unless
-    # the user opted into partial_coverage = use (#1228).
-    CONFIRMATION_SKIPPED = "survivor confirmation skipped: partial_coverage = use applies only to a cached " \
-                           "whole-suite run (no --test, one coverage part)"
-
     # Everything besides sources and tests that can change a verdict.
     VERDICT_INPUTS = %w[integration require rspec_args rails reeval sources operators exclude_operators arid
                         timeout_factor timeout_min
@@ -46,10 +41,11 @@ module Saboteur
     attr_reader :guessed_tests
 
     def initialize(config, err: $stderr, digest: nil, capture_evidence: false, evidence_files: [], verbose: false,
-                   palette: nil, quiet_start: true)
+                   palette: nil, quiet_start: true, quiet: false)
       @config = config
       @hooks = Hooks.new
-      @diagnostics = Diagnostics.new(err, verbose: verbose, palette: palette, quiet_start: quiet_start)
+      @diagnostics = Diagnostics.new(err, verbose: verbose, palette: palette, quiet_start: quiet_start,
+                                          quiet: quiet)
       @completed_baseline_passes = 0
       @baseline_elapsed = 0.0
       @root = File.realpath(config.root)
@@ -129,6 +125,12 @@ module Saboteur
       @coverage_store&.whole_suite_map
     end
 
+    # True when this run did not build a whole-suite map: a diff, named
+    # subjects, an explicit --test or inferred specs (#1286).
+    def scoped_selection?
+      !whole_suite_run?
+    end
+
     # False when caching is off, where no coverage build can ever leave a
     # whole-suite map to consult, so advising one would be wrong (review C1).
     def whole_suite_map_lookup?
@@ -291,15 +293,21 @@ module Saboteur
       confirm_survivors(run_candidates(subjects, lists, scheduler, cache, ignore_list), scheduler, cache, ignore_list)
     end
 
+    # Confirm each live survivor against the whole-suite coverage map (#1286):
+    # the covering specs the run's own selection never loaded get one budgeted
+    # run. Scoped runs (--since, --subject, pr) are the point, so nothing here
+    # depends on partial_coverage or on the run being unscoped. Without a
+    # whole-suite map there is nothing to confirm against; the run says so
+    # elsewhere (survivors unconfirmed: no whole-suite coverage map).
     def confirm_survivors(results, scheduler, cache, ignore_list)
-      unless CoveragePartial.use?(config, config["coverage_parts"])
-        live = results.any? { |result| result.survived? && !ignore_list.accepts?(result) }
-        @diagnostics.inform(CONFIRMATION_SKIPPED) if live && config["partial_coverage"] == "use"
-        return results
-      end
+      return results if results.none? { |result| result.survived? && !ignore_list.accepts?(result) }
 
-      SurvivorConfirmation.new(config: config, map: scheduler.map, scheduler: scheduler, diagnostics: @diagnostics,
-                               budget_exceeded: method(:budget_exceeded?), cache: cache)
+      whole = whole_suite_coverage_map
+      return results if whole.nil?
+
+      SurvivorConfirmation.new(config: config, map: whole, scheduler: scheduler, diagnostics: @diagnostics,
+                               budget_exceeded: method(:budget_exceeded?), cache: cache,
+                               load_specs: ->(files) { scheduler.runner.integration.load_spec_files(files) })
                           .call(results, ignore_list: ignore_list, early_stop: @early_stop)
     end
 
@@ -349,10 +357,19 @@ module Saboteur
     # baseline's own chatter (demoted to --verbose) would start.
     def announce_quiet_start(store, files)
       @map_cached = store.reusable?(files)
-      @diagnostics.quiet_start_header(scope: config["since"] ? "--since #{config['since']}" : "whole project",
+      @diagnostics.quiet_start_header(scope: quiet_scope,
                                       seed: config["seed"], operators: config.operator_policy["effective"].size)
     end
 
+    # Names what the run covers: the diff, the --subject filter, or the
+    # whole project (P6) - a --subject run is not "whole project".
+    def quiet_scope
+      parts = []
+      parts << "--since #{config['since']}" if config["since"]
+      parts << "--subject #{config['subjects'].join(', ')}" if config["subjects"].any?
+      parts.empty? ? "whole project" : parts.join(", ")
+    end
+
     def inform_about_run(subjects, map)
       if @diagnostics.quiet_start && !@diagnostics.verbose
         @diagnostics.quiet_baseline(passes: @completed_baseline_passes, elapsed: @baseline_elapsed,
@@ -366,11 +383,13 @@ module Saboteur
     end
 
     # This run's map is the whole-suite map only when nothing scoped it:
-    # no --since, no subjects, and tests either unset or inferred over every
-    # included file (never an explicit --test list) (review C1).
+    # no --since, no subjects, and every spec file loaded (--all or
+    # c.test_selection = :all). Inferred specs are a selection, never the
+    # whole suite: a spec with no matching source file is not in the map
+    # (#1286).
     def whole_suite_run?
       config["since"].nil? && config["subjects"].empty? &&
-        (config["tests"].empty? || !@guessed_tests.nil?)
+        config["tests"].empty? && @guessed_tests.nil?
     end
 
     def coverage_store(integration, load_lines, load_methods = {})

data/lib/saboteur/execution/runner.rb

@@ -74,6 +74,9 @@ module Saboteur
       # sandbox: when given, a callable run first in every child (Landlock
       # makes the project tree read-only there, #152). It raises rather than
       # run half-restricted, which the child reports as an error.
+      # Survivor confirmation loads the spec files a scoped run left out (#1286).
+      attr_reader :integration
+
       def initialize(integration:, reevaluator:, log_dir:, hooks: Hooks.new, supervisor: Supervisor.new, seed: 1,
                      sandbox: nil, canary: nil, crash_log: nil)
         @integration = integration

data/lib/saboteur/gates.rb

@@ -205,7 +205,7 @@ module Saboteur
 
     # A subject fails a gate when at least one of its mutants survived.
     def failing_subjects(scored)
-      scored.select { |result| result.status == :survived }.map(&:subject).uniq.size
+      scored.select(&:survived?).map(&:subject).uniq.size
     end
 
     def over?(count, limit, executed)

data/lib/saboteur/help.rb

@@ -140,15 +140,15 @@ module Saboteur
         name: "survivors",
         summary: "Browse a report's survivors in the terminal, read-only",
         description: "Reads the Stryker JSON report without booting the project or rerunning tests. " \
-                     "Lists every survivor with rank, file, line, operator, digest and triage label; " \
-                     "filters with --file, --operator and --triage; --ranked uses the report's ranked " \
-                     "findings order. --show DIGEST prints the diff, the exact replay command and an " \
-                     "ignore-file entry template for that mutant. It never edits the ignore file.",
+                     "Lists every survivor with rank, file, line, operator, digest and triage label, " \
+                     "in the report's ranked findings order; filters with --file, --operator and " \
+                     "--triage; --ranked is kept for compatibility. --show DIGEST prints the diff, " \
+                     "the exact replay command and an ignore-file entry template for that mutant. " \
+                     "It never edits the ignore file.",
         takes_options: false,
         exit_codes: { 0 => "Survivors listed or shown (including none).",
                       2 => "Missing, malformed or unreadable report; bad arguments." },
         examples: [["List survivors from the last run", "saboteur survivors .saboteur/report.json"],
-                   ["Review-priority order", "saboteur survivors .saboteur/report.json --ranked"],
                    ["Diff, replay command and ignore template for one mutant",
                     "saboteur survivors .saboteur/report.json --show 1a2b3c4d5e6f7a8b"]]
       ),

data/lib/saboteur/integration/rspec.rb

@@ -124,6 +124,36 @@ module Saboteur
         world.example_groups.map { |group| File.expand_path(group.metadata[:file_path]) }.uniq
       end
 
+      # Loads spec files the run's own selection left out, in the parent, so a
+      # forked confirmation run can execute their examples (#1286). Files
+      # already loaded are skipped; a file that defines no group is an error,
+      # since its examples could never run.
+      def load_spec_files(paths)
+        loaded = test_files
+        absolute = paths.map { |path| File.expand_path(path, config.root) }.uniq
+        (absolute - loaded).each do |path|
+          require_inside_root(path)
+          before = world.example_groups.size
+          from_directory(config.root) { load(path) }
+          raise RefusedSpecPath, "#{path} defines no example group" if world.example_groups.size == before
+        end
+      end
+
+      # The ids come from a cached coverage map, which is untrusted input
+      # (#427): only a spec file that really lives under the project root,
+      # symlinks resolved, may be loaded.
+      def require_inside_root(path)
+        raise RefusedSpecPath, "refusing to load #{path}: not a *_spec.rb file" unless path.end_with?("_spec.rb")
+
+        root = File.realpath(config.root)
+        real = File.realpath(path)
+        return if real.start_with?("#{root}#{File::SEPARATOR}") && real.end_with?("_spec.rb")
+
+        raise RefusedSpecPath, "refusing to load #{path}: it resolves outside the project root"
+      rescue SystemCallError
+        raise RefusedSpecPath, "refusing to load #{path}: it does not exist under the project root"
+      end
+
       # Runs the tests in the order given: groups by their earliest example,
       # examples by position.
       def run(ids)

data/lib/saboteur/reporter/default_findings.rb

@@ -57,7 +57,7 @@ module Saboteur
       def list_lines(report, live)
         groups(report, live).map do |(path, line, subject_id), results|
           first, *rest = results
-          label = SurvivorClassification.label(first)
+          label = SurvivorClassification.group_label(results)
           text = "#{path}:#{line} #{subject_id}#{" [#{label}]" if label} - #{first.digest}"
           rest.any? ? "#{text} (+#{rest.size} more)" : text
         end

data/lib/saboteur/reporter/findings.rb

@@ -12,6 +12,8 @@ module Saboteur
     # Survivors accepted on the ignore list are listed apart, with their
     # reason. The full, unbounded mutant data stays in the report's files
     # section; survivorsTruncated says how many more findings live only there.
+    # The uncapped ranked digest list lets offline readers (the survivors
+    # command) print the terminal's ordering without recomputing it (#1150).
     module Findings
       VERSION = 1
 
@@ -30,7 +32,8 @@ module Saboteur
             survivors: listed(report, live),
             survivorsTruncated: truncated(live),
             accepted: listed(report, accepted),
-            acceptedTruncated: truncated(accepted)
+            acceptedTruncated: truncated(accepted),
+            order: full_order(report, live, accepted)
           }
         end
 
@@ -44,6 +47,12 @@ module Saboteur
           [results.size - LIMIT, 0].max
         end
 
+        # Every survivor digest in terminal order: live ranked first, then
+        # accepted - uncapped, unlike the findings lists themselves.
+        def full_order(report, live, accepted)
+          (ordered(report, live) + ordered(report, accepted)).map(&:digest)
+        end
+
         def ordered(report, results)
           SurvivorRanking.ordered(report, results)
         end

data/lib/saboteur/reporter/gap_kinds.rb

@@ -58,6 +58,12 @@ module Saboteur
 
       FALLBACK = "result unchecked"
 
+      # Which kind names a statement when its surviving mutants disagree
+      # (#P4): the first one listed that any of them carries. The broadest
+      # unchecked thing wins, so a branch condition is not called a
+      # boundary because one of its mutants happened to flip a comparison.
+      PRECEDENCE = ["result unchecked", "error path", "branch", "boundary", "effect", "number"].freeze
+
       # The deterministic per-kind reason printed next to a survivor's rank.
       REASONS = {
         "boundary" => "the covering specs pass without pinning this boundary",

data/lib/saboteur/reporter/html.rb

@@ -151,6 +151,8 @@ module Saboteur
       # the same contract as the terminal headline.
       def headline(report)
         breakdown = report.score_breakdown
+        return error_dominated_headline(report) if report.majority_errored?
+
         text = "<p><strong>Covered mutation score: #{score_text(report.score)}</strong>"
         if breakdown.tested.positive?
           text << " (#{breakdown.detected}/#{breakdown.tested} #{Wording.detected_word(breakdown)}, tested subjects)"
@@ -195,12 +197,21 @@ module Saboteur
         "#{Text.counted(report.subjects.size, 'subject')}, #{Text.counted(report.results.size, 'mutant')}"
       end
 
+      # An error-dominated run shows no bare score, like the terminal headline
+      # (#557, #1272): the share covers only the few mutants that were scored.
+      def error_dominated_headline(report)
+        breakdown = report.score_breakdown
+        "<p><strong>Covered mutation score: n/a</strong> (#{breakdown.tested} of #{report.executed.size} " \
+          "mutants scored, #{score_text(breakdown.value)} of scored) - most executed mutants errored, " \
+          "so the run tested almost nothing</p>"
+      end
+
       def summary(report, data)
         seed = data.dig(:config, :seed)
         seed_note = seed ? ", seed #{seed}" : ""
         counts = data[:files].values.flat_map { |entry| entry[:mutants] }.map { |mutant| mutant[:status] }.tally
         <<~HTML
-          <p><strong>Covered mutation score: #{score_text(report.score)}</strong>
+          <p><strong>Covered mutation score: #{score_text(report.majority_errored? ? nil : report.score)}</strong>
           (#{counts_note(report)}, #{format('%.1f', report.wall_time)}s#{seed_note})</p>
           <p class="note">Covered mutation score = #{SCORE_FORMULA}. NoCoverage, errors, resource limits and skipped
           mutants are not counted. Survivors accepted on the ignore list stay in the denominator. It covers only the subjects of this run, not the whole project. A tool that reads the

data/lib/saboteur/reporter/next_steps.rb

@@ -20,7 +20,9 @@ module Saboteur
       end
 
       def call(report)
-        survivor = report.unignored_survivors.first
+        # The replay example is the #1 survivor of the ranked list, not
+        # whichever one the results hold first.
+        survivor = SurvivorRanking.ordered(report, report.unignored_survivors).first
         return unless survivor
 
         @out.puts "\n#{@palette.paint(:head, 'Next steps:')}"

data/lib/saboteur/reporter/score_table.rb

@@ -15,7 +15,6 @@ module Saboteur
     # it. Untested subjects are never dropped silently, even at 100 %.
     class ScoreTable
       MAX_ROWS = 15
-      MAX_UNTESTED_NAMES = 3
       # The skip reason CandidateSelection gives a subject no test covers.
       NO_COVERAGE = "no coverage"
 
@@ -63,7 +62,7 @@ module Saboteur
           detected: breakdown.detected, tested: breakdown.tested, untested: untested }
       end
 
-      # "cart.rb  100.0% (2/2) · n/a - 2 subjects - no test executes them: ...": the
+      # "cart.rb  100.0% (2/2) · n/a - 2 subjects - no test executes them": the
       # tested score and the untested count stand next to each other; a
       # file with nothing tested gets no percentage at all.
       def line(row, width)
@@ -75,18 +74,14 @@ module Saboteur
         note ? "#{base} #{@palette.glyph(:middot)} #{note}" : base
       end
 
-      # "n/a - 2 subjects - no test executes them: Cart#total, Cart#biggest":
-      # names the subjects the score cannot speak for, capped so a long
-      # list does not swallow the row; the Not scored block lists them all.
-      # Wording owns the claim (#1145): a stubbed-away subject has tests,
-      # they just never execute it.
+      # "n/a - 2 subjects - no test executes them": only the count. The
+      # Not scored block below names every subject once (P6), so the table
+      # does not repeat the names. Wording owns the claim (#1145): a
+      # stubbed-away subject has tests, they just never execute it.
       def untested_note(untested)
         return if untested.empty?
 
-        names = untested.map(&:id)
-        shown = names.first(MAX_UNTESTED_NAMES)
-        shown << "+#{names.size - MAX_UNTESTED_NAMES} more" if names.size > MAX_UNTESTED_NAMES
-        @palette.paint(:warn, "n/a - #{Wording.untested_note(untested.size)}: #{shown.join(', ')}")
+        @palette.paint(:warn, "n/a - #{Wording.untested_note(untested.size)}")
       end
     end
   end

data/lib/saboteur/reporter/stryker.rb

@@ -98,7 +98,7 @@ module Saboteur
       end
 
       def self.test_index(results)
-        names = results.flat_map { |result| result.covered_by + result.killed_by + result.executed.to_a }.uniq
+        names = results.flat_map { |result| result.covered_by + result.killed_by + result.executed.to_a }.uniq.sort
         names.each_with_index.to_h { |name, index| [name, index.to_s] }
       end
 
@@ -139,6 +139,10 @@ module Saboteur
           # Why the survivor is worth a look, as a label only: scores and
           # the exit code never change (#192).
           classification: SurvivorClassification.label(result),
+          # Additive (#1286): confirmed, killed_by_confirmation or unconfirmed.
+          # An unconfirmed survivor reports Ignored: outside the score.
+          confirmation: result.confirmation_state,
+          creditedTest: result.credited_test&.then { |name| test_ids.fetch(name) },
           killedBy: result.killed_by.map { |name| test_ids.fetch(name) }.then { |ids| ids.empty? ? nil : ids },
           coveredBy: result.covered_by.map { |name| test_ids.fetch(name) }.then { |ids| ids.empty? ? nil : ids },
           # Which of the covering tests actually ran against the mutant; absent
@@ -151,7 +155,7 @@ module Saboteur
       def self.status(result)
         case result.status
         when :killed then "Killed"
-        when :survived then "Survived"
+        when :survived then result.unconfirmed? ? "Ignored" : "Survived"
         when :timeout then "Timeout"
         when :error, :resource_limit then "RuntimeError"
         when :skipped then result.reason&.start_with?("no coverage") ? "NoCoverage" : "Ignored"
@@ -169,6 +173,7 @@ module Saboteur
         # ignore, and only wont_fix adds a suffix after the reason.
         return "ignored: #{result.ignored} (wont_fix)" if result.triage_state == "wont_fix"
         return "ignored: #{result.ignored}" if result.ignored
+        return "unconfirmed: #{result.unconfirmed_reason}" if result.unconfirmed?
         return "resource limit: #{result.reason}" if result.status == :resource_limit
         return result.crash_kill? ? "crash: #{result.kill_cause}" : result.kill_cause if result.kill_cause
 

data/lib/saboteur/reporter/survivor_classification.rb

@@ -55,6 +55,21 @@ module Saboteur
           nil
         end
 
+        # The label of one statement's surviving mutants as a whole, the
+        # same whichever mutant sorts first. Any gap kind decides, by
+        # GapKinds::PRECEDENCE, so one equivalent, logging or input_gap
+        # sibling never hides a real survivor. With no gap kind, the first
+        # label present stands: likely_equivalent, arid or tool_limit then
+        # describe the statement when all its mutants carry it.
+        #
+        # Among gap kinds, result unchecked wins because it is the strictly
+        # larger gap: nothing pins the statement's value at all, which
+        # subsumes a missing branch, boundary or effect check on it.
+        def group_label(results)
+          labels = results.map { |result| label(result) }.compact
+          GapKinds::PRECEDENCE.find { |kind| labels.include?(kind) } || labels.first
+        end
+
         private
 
         # The mutant's own def text, compiled side by side with the

data/lib/saboteur/reporter/survivor_list.rb

@@ -77,8 +77,8 @@ module Saboteur
         line = MutationText.line(first)
         path = relative(first.subject.file, report.root)
         @out.puts "\n#{@palette.paint(:head, "#{rank}. #{path}:#{line}")} #{first.subject.id} " \
-                  "#{@palette.paint(:dim, label_suffix(first, rank, total))}".rstrip
-        reason = SurvivorHints.label_reason(@classifier.label(first))
+                  "#{@palette.paint(:dim, label_suffix(group, rank, total))}".rstrip
+        reason = SurvivorHints.label_reason(@classifier.group_label(group))
         @out.puts "  #{@palette.paint(:dim, reason)}" if reason
         fence(first, line)
         @out.puts "  #{@palette.paint(:dim, "why it matters: #{why(first.mutation.operator)}")}"
@@ -105,10 +105,10 @@ module Saboteur
       # siblings read "#3 of 14 blocks", never blocks against mutants (#1147).
       # Label and rank stay on the header line; the reason moves to its own
       # dim line so the fixed text cannot push the header past 80 columns.
-      def label_suffix(result, rank, total)
-        label = @classifier.label(result)
+      def label_suffix(group, rank, total)
+        label = @classifier.group_label(group)
         unit = total == 1 ? "block" : "blocks"
-        parts = [("coarse" if result.mutation.coarse), label, "##{rank} of #{total} #{unit}"].compact
+        parts = [("coarse" if group.first.mutation.coarse), label, "##{rank} of #{total} #{unit}"].compact
         "[#{parts.join(' · ')}]"
       end
 
@@ -226,7 +226,7 @@ module Saboteur
 
         whole = @whole_suite_map&.tests_for(group.first.subject)
         if whole.nil?
-          @out.puts @palette.paint(:dim, "  covering tests unknown - run a full coverage build")
+          @out.puts @palette.paint(:dim, "  covering list is this run's selection only; run `saboteur run --all`")
           return
         end
 

data/lib/saboteur/reporter/terminal.rb

@@ -6,6 +6,7 @@ require_relative "wording"
 require_relative "baseline_tail"
 require_relative "next_steps"
 require_relative "kill_causes"
+require_relative "unconfirmed_section"
 require_relative "survivor_ranking"
 require_relative "palette"
 require_relative "notice"
@@ -62,6 +63,7 @@ module Saboteur
         return quiet_call(report) if @quiet
 
         findings(report)
+        UnconfirmedSection.new(@out, @palette).call(report)
         errors(report)
         skipped_subjects(report) if @verbose
         test_command_gaps(report)
@@ -94,7 +96,7 @@ module Saboteur
 
       def quiet_survivor_line(path, line, subject_id, results)
         first, *rest = results
-        label = SurvivorClassification.label(first)
+        label = SurvivorClassification.group_label(results)
         text = "#{path}:#{line} #{subject_id}#{" [#{label}]" if label} - #{first.digest}"
         rest.any? ? "#{text} (+#{rest.size} more)" : text
       end
@@ -332,7 +334,8 @@ module Saboteur
         parts = state_parts(report)
         parts << "no mutants" if parts.empty?
         line = "saboteur: #{parts.join(', ')} in #{duration(report.wall_time)}"
-        line += " - score #{painted_score(report.score)}"
+        # The headline withholds the score on an error-dominated run (#557); so does this line (#1271).
+        line += " - score #{painted_score(report.majority_errored? ? nil : report.score)}"
         line += " (threshold #{format('%g', @threshold)})" if @threshold
         @out.puts line
       end
@@ -345,7 +348,7 @@ module Saboteur
         untested = untested_subjects(report).size
         [counted(counts[:killed], "killed", plural: false, color: :pass),
          counted(live, "survived", plural: false, color: :fail),
-         counted(report.survivors.size - live, "accepted", plural: false),
+         counted(report.survivors.size - live, "accepted", plural: false), UnconfirmedSection.part(report),
          counted(counts[:timeout], "timeout", color: :warn), counted(counts[:resource_limit], "resource limit"),
          error_part(report, counts[:error]), counted(counts[:unverified], "unverified", plural: false),
          ("#{untested} subject#{'s' unless untested == 1} untested" if untested.positive?)].compact

data/lib/saboteur/reporter/unconfirmed_section.rb

@@ -0,0 +1,44 @@
+# SPDX-License-Identifier: MIT
+# frozen_string_literal: true
+
+require_relative "mutation_text"
+require_relative "survivor_ranking"
+
+module Saboteur
+  module Reporter
+    # Survivors whose covering tests were not all run (#1286): outside the
+    # score and the exit code, so they get their own short list instead of
+    # a place among the findings.
+    class UnconfirmedSection
+      LIMIT = 10
+
+      # The count for the one-line (--quiet) tally, nil when there is none.
+      def self.part(report)
+        count = report.unconfirmed_survivors.size
+        "#{count} unconfirmed" if count.positive?
+      end
+
+      def initialize(out, palette)
+        @out = out
+        @palette = palette
+      end
+
+      def call(report)
+        pending = report.unconfirmed_survivors
+        return if pending.empty?
+
+        @out.puts @palette.paint(:head, "unconfirmed (covering tests not run) - #{pending.size}, not in the score")
+        SurvivorRanking.ordered(report, pending).first(LIMIT).each { |result| @out.puts line(report, result) }
+        @out.puts "  ... #{pending.size - LIMIT} more" if pending.size > LIMIT
+        @out.puts
+      end
+
+      private
+
+      def line(report, result)
+        path = result.subject.file.delete_prefix("#{report.root}/")
+        "  #{path}:#{MutationText.line(result)} #{result.subject.id} - #{result.digest} (#{result.unconfirmed_reason})"
+      end
+    end
+  end
+end

data/lib/saboteur/result.rb

@@ -112,8 +112,37 @@ module Saboteur
       @digest ||= mutation.digest(subject_start: subject.start_byte, original_source: subject.file_source)
     end
 
+    # A survivor whose confirmation run could not happen (budget spent, or the
+    # run failed) is outside the score and the exit code until it is
+    # confirmed (#1286). A survivor the whole-suite map cannot speak for
+    # ("unconfirmed_map(...)") stays counted: a stale map never lowers the gate.
     def survived?
-      status == :survived
+      status == :survived && !unconfirmed?
+    end
+
+    def unconfirmed?
+      status == :survived && @confirmation.to_s.start_with?("unconfirmed(")
+    end
+
+    # The additive report field: confirmed, killed_by_confirmation or
+    # unconfirmed; nil when confirmation never applied (#1286).
+    def confirmation_state
+      return nil if @confirmation.nil?
+
+      label = @confirmation.to_s
+      label.start_with?("unconfirmed") ? "unconfirmed" : label
+    end
+
+    # The test a confirmation kill is credited to.
+    def credited_test
+      killed_by.first if confirmation_state == "killed_by_confirmation"
+    end
+
+    # Why the survivor stayed unconfirmed: the text inside unconfirmed(...).
+    def unconfirmed_reason
+      return unless unconfirmed?
+
+      @confirmation.delete_prefix("unconfirmed(").delete_suffix(")")
     end
 
     # Killed because a test's error, not its check, fired: NoMethodError and
@@ -280,7 +309,9 @@ module Saboteur
     end
 
     def counts
-      STATUSES.to_h { |status| [status, results.count { |result| result.status == status }] }
+      STATUSES.to_h do |status|
+        [status, results.count { |result| status == :survived ? result.survived? : result.status == status }]
+      end
     end
 
     # (killed + timeout) / (killed + survived + timeout); nil with nothing tested.
@@ -308,6 +339,12 @@ module Saboteur
       results.select(&:survived?)
     end
 
+    # Survivors whose covering tests were not all run: outside the score and
+    # the exit code, listed on their own (#1286).
+    def unconfirmed_survivors
+      results.select(&:unconfirmed?)
+    end
+
     def unignored_survivors
       results.select(&:unignored_survivor?)
     end

data/lib/saboteur/selection/coverage_map.rb

@@ -443,9 +443,19 @@ module Saboteur
         [prefix.count("\n") + 1, offset - ((prefix.rindex("\n") || -1) + 1)]
       end
 
+      # Durations are compared to the millisecond. Two tests within a
+      # millisecond of each other swap places from one map build to the next,
+      # and with fail-fast the first failing test names a mutant's killer and
+      # kill cause, so sub-millisecond noise must not decide the order (#1274).
+      # Suite order breaks the tie instead.
       def sort_test_ids(ids)
         order = test_order
-        ids.sort_by { |id| [durations.fetch(id, Float::INFINITY), order.fetch(id, Float::INFINITY)] }
+        ids.sort_by { |id| [duration_bucket(id), order.fetch(id, Float::INFINITY)] }
+      end
+
+      def duration_bucket(id)
+        seconds = durations[id]
+        seconds ? (seconds * 1000).ceil : Float::INFINITY
       end
 
       def test_order

data/lib/saboteur/version.rb

@@ -2,5 +2,5 @@
 # frozen_string_literal: true
 
 module Saboteur
-  VERSION = "0.12.0"
+  VERSION = "0.12.2"
 end

data/lib/saboteur.rb

@@ -6,6 +6,10 @@ require_relative "saboteur/version"
 module Saboteur
   class Error < StandardError; end
 
+  # A spec path from the cached coverage map that is not a spec file inside
+  # the project: the map is untrusted input (#427, #1286).
+  class RefusedSpecPath < Error; end
+
   # A phase timeout that --max-duration shortened: budget exhaustion, not a
   # broken suite. The engine turns it into the budget partial-report path
   # instead of a phase error (#296).

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: saboteur
 version: !ruby/object:Gem::Version
-  version: 0.12.0
+  version: 0.12.2
 platform: ruby
 authors:
 - Vamsi Pavan Mahesh Gunturu
@@ -63,6 +63,7 @@ files:
 - lib/saboteur/cli.rb
 - lib/saboteur/cli/agent_provenance.rb
 - lib/saboteur/cli/baseline_gate.rb
+- lib/saboteur/cli/color_option.rb
 - lib/saboteur/cli/dirty_tree_notice.rb
 - lib/saboteur/cli/doctor_command.rb
 - lib/saboteur/cli/error_cause.rb
@@ -89,6 +90,7 @@ files:
 - lib/saboteur/cli/shard_options.rb
 - lib/saboteur/cli/source_set.rb
 - lib/saboteur/cli/survivors_command.rb
+- lib/saboteur/cli/whole_suite_map_lookup.rb
 - lib/saboteur/config.rb
 - lib/saboteur/config_file.rb
 - lib/saboteur/constant_reads.rb
@@ -253,6 +255,7 @@ files:
 - lib/saboteur/reporter/survivor_ranking.rb
 - lib/saboteur/reporter/terminal.rb
 - lib/saboteur/reporter/test_matrix.rb
+- lib/saboteur/reporter/unconfirmed_section.rb
 - lib/saboteur/reporter/wording.rb
 - lib/saboteur/result.rb
 - lib/saboteur/score.rb