saboteur 0.3.0 → 0.5.0

data/lib/saboteur/reporter/terminal.rb

@@ -1,33 +1,92 @@
 # SPDX-License-Identifier: MIT
 # frozen_string_literal: true
 
+require_relative "next_steps"
+require_relative "survivor_ranking"
+require_relative "palette"
+require_relative "score_table"
+require_relative "survivor_list"
+require_relative "test_matrix"
+
 module Saboteur
   module Reporter
-    # Plain-text summary: counts per status, score, timings, skip reasons,
-    # errors, then a unified diff for each survivor.
+    # Plain-text report: survivors first (grouped by file, method and line),
+    # then errors and notes, then a short summary last so it stays on screen.
+    # Debug statistics (score split, timings, kill causes) need verbose.
     class Terminal
       MAX_ERRORS = 10
+      MAX_PASSING_TESTS = SurvivorList::MAX_PASSING_TESTS
 
-      def initialize(out)
+      def initialize(out, verbose: false, color: :auto)
         @out = out
+        @verbose = verbose
+        @palette = Palette.for(out, mode: color)
+        @score_table = ScoreTable.new(out, @palette)
+        @next_steps = NextSteps.new(out, @palette)
       end
 
       def call(report)
-        summary(report)
+        survivors(report)
+        errors(report)
         skipped_subjects(report)
+        outside_the_score(report)
+        inline_suppressions(report)
         warnings(report)
-        errors(report)
-        survivors(report)
+        early_stop(report)
+        test_matrix(report)
+        summary(report)
       end
 
       private
 
+      def test_matrix(report)
+        return unless report.matrix?
+
+        matrix = TestMatrix.new(report.results)
+        @out.puts "test matrix: #{matrix.tests.size} tests ran against mutants"
+        incomplete_matrix_note(matrix)
+        matrix_list("blind (covered mutants, killed none)", matrix.blind) do |test|
+          "#{matrix.mutants_covered(test)} covered"
+        end
+        matrix_list("redundant (every kill had another killer)", matrix.redundant) do |test|
+          "#{matrix.mutants_killed(test)} kills"
+        end
+      end
+
+      def incomplete_matrix_note(matrix)
+        return if matrix.incomplete.zero?
+
+        @out.puts "  #{matrix.incomplete} mutants stopped before every covering test ran (incomplete): " \
+                  "their tests get no blind or redundant verdict"
+      end
+
+      def matrix_list(title, tests)
+        @out.puts "  #{title}: #{tests.size}"
+        tests.first(TestMatrix::MAX_LISTED).each { |test| @out.puts "    #{test}  #{yield(test)}" }
+        more = tests.size - TestMatrix::MAX_LISTED
+        @out.puts "    ... #{more} more" if more.positive?
+      end
+
+      def inline_suppressions(report)
+        return if report.inline_suppressions.empty?
+
+        @out.puts("Inline suppressions: #{report.inline_suppressions.size} (outside the score)")
+        report.inline_suppressions.each do |entry|
+          path = entry[:file].delete_prefix("#{report.root}/")
+          @out.puts("  #{path}:#{entry[:line]} #{entry[:subject]} #{entry[:operator]} " \
+                    "#{entry[:digest]}: #{entry[:reason]}")
+        end
+      end
+
       def summary(report)
         seed = report.seed && ", seed #{report.seed} (repeat with --seed #{report.seed})"
         @out.puts "saboteur #{VERSION}: #{report.subjects.size} subjects, #{report.results.size} mutants, " \
                   "wall #{format('%.2f', report.wall_time)} s#{seed}"
         @out.puts status_line(report)
         details(report)
+        @score_table.call(report)
+        @out.puts @score_table.legend
+        @next_steps.call(report)
       end
 
       def status_line(report)
@@ -35,26 +94,72 @@ module Saboteur
         survived = "survived #{c[:survived]}"
         ignored = report.survivors.size - report.unignored_survivors.size
         survived += " (#{ignored} ignored)" if ignored.positive?
-        "killed #{c[:killed]}  #{survived}  timeout #{c[:timeout]}  resource limit #{c[:resource_limit]}  " \
-          "error #{c[:error]}  skipped #{c[:skipped]}"
+        line = "#{@palette.paint(:green,
+                                 "killed #{c[:killed]}")}  #{@palette.paint(:red,
+                                                                            survived)}  timeout #{c[:timeout]}  " \
+               "resource limit #{c[:resource_limit]}  error #{c[:error]}  skipped #{c[:skipped]}"
+        line += "  ran at load #{c[:ran_at_load]}" if c[:ran_at_load].positive?
+        line += "  unreachable #{c[:unreachable]}" if c[:unreachable].positive?
+        line
+      end
+
+      def hazard_suppressions(report)
+        return unless report.hazard_suppressions.positive?
+
+        @out.puts "hazardous mutations suppressed: #{report.hazard_suppressions} " \
+                  "(--allow-hazardous runs them anyway; the canary still watches the tree)"
       end
 
       def details(report)
-        score = report.score
-        @out.puts "score #{score ? format('%.1f %%', score * 100) : 'n/a'} " \
-                  "(killed+timeout)/(killed+survived+timeout); errors, resource limits and skips excluded"
-        mean = report.mean_runtime
-        @out.puts "mean time per executed mutant #{format('%.3f', mean)} s" if mean
-        @out.puts "invalid mutations dropped (did not parse): #{report.invalid_mutations}" if
-          report.invalid_mutations.positive?
+        breakdown = report.score_breakdown
+        @out.puts "score #{percent(breakdown.value)}" \
+                  "#{if @verbose
+                       "#{score_split(breakdown)} (killed+timeout)/(killed+survived+timeout); " \
+                         'errors, resource limits and skips excluded'
+                     end}"
         report.skip_reasons.each { |reason, count| @out.puts "skipped: #{reason}: #{count}" }
         reused = report.results.count(&:reused?)
         @out.puts "reused #{reused} of #{report.results.size} verdicts from earlier runs (--incremental)" if
           reused.positive?
+        @out.puts "invalid mutations dropped (did not parse): #{report.invalid_mutations}" if
+          report.invalid_mutations.positive?
+        hazard_suppressions(report)
+        debug_details(report) if @verbose
+      end
+
+      def debug_details(report)
+        whole_body_score(report)
+        mean = report.mean_runtime
+        @out.puts "mean time per executed mutant #{format('%.3f', mean)} s" if mean
         @out.puts "kills rejected by unchanged-method check: #{report.rejected_kills}"
         kill_causes(report)
       end
 
+      # " (assertion 50.0 %, crash 50.0 %)": how much of the score is a failed
+      # check and how much is a crash. Timeouts and kills with no recorded
+      # cause are named only when there are some, so the parts add up.
+      def score_split(breakdown)
+        return "" unless breakdown.split_known?
+
+        " (#{breakdown.parts.map { |label, share| "#{label} #{percent(share)}" }.join(', ')})"
+      end
+
+      # Whole-body mutants (body_nil, body_true, body_false, body_raise) are
+      # killed by almost any test that runs the method, so they get their own
+      # line.
+      def whole_body_score(report)
+        whole = report.coarse_score_breakdown
+        fine = report.fine_score_breakdown
+        return if whole.tested.zero? || fine.tested.zero?
+
+        @out.puts "score without whole-body mutants #{percent(fine.value)}#{score_split(fine)} " \
+                  "(#{fine.tested} mutants); whole-body mutants alone #{percent(whole.value)} (#{whole.tested})"
+      end
+
+      def percent(share)
+        share ? format("%.1f %%", share * 100) : "n/a"
+      end
+
       # How many kills came from a failed check and how many from a crash, and
       # which operators mostly kill by crashing (their score says less).
       def kill_causes(report)
@@ -91,6 +196,37 @@ module Saboteur
         end
       end
 
+      # Mutants no test can judge, listed apart so they are not read as
+      # misses: code that only ran while the app loaded, and methods a
+      # captured Method object still calls in their original form.
+      def outside_the_score(report)
+        { ran_at_load: "Ran at load (out of the score):",
+          unreachable: "Unreachable (out of the score):" }.each do |status, heading|
+          results = report.results.select { |result| result.status == status }
+          next if results.empty?
+
+          @out.puts heading
+          results.each do |result|
+            location = "#{relative(result.subject.file, report.root)}:#{result.subject.start_line}"
+            @out.puts "  #{result.subject.id} #{result.mutation.operator} #{result.digest} #{location}"
+          end
+          @out.puts "  hint: #{results.first.reason}" if results.first.reason
+        end
+      end
+
+      # The verdict covers only the subjects that ran, so say it where the
+      # score is read.
+      def early_stop(report)
+        stop = report.early_stop
+        return unless stop
+
+        @out.puts @palette.paint(:yellow, "stopped early (--fail-fast): #{stop.unreached.size} selected " \
+                                          "subject#{'s' unless stop.unreached.one?} did not run; the score below " \
+                                          "covers only the subjects that ran")
+        stop.unreached.first(MAX_ERRORS).each { |id| @out.puts "  not run: #{id}" }
+        @out.puts "  ... #{stop.unreached.size - MAX_ERRORS} more" if stop.unreached.size > MAX_ERRORS
+      end
+
       def warnings(report)
         report.warnings.each { |warning| @out.puts "warning: #{warning}" }
       end
@@ -108,35 +244,14 @@ module Saboteur
       end
 
       def survivors(report)
-        return if report.survivors.empty?
-
-        @out.puts "\nSurvivors:"
-        shown(report).each do |result, hidden|
-          location = "#{relative(result.subject.file, report.root)}:#{result.subject.start_line}"
-          @out.puts "\n#{result.subject.id} #{result.mutation.operator} #{result.digest} #{location}"
-          @out.puts "ignored: #{result.ignored}" if result.ignored
-          @out.puts result.diff(root: report.root)
-          @out.puts "#{hidden} more survivor#{'s' if hidden > 1} on this line in the JSON report" if hidden.positive?
-        end
-      end
-
-      # [[result, how many more on the same line]]. In review mode survivors
-      # on one line usually share a fate, so one is shown: live before
-      # ignored, then coarse before fine, then the earliest edit.
-      def shown(report)
-        return report.survivors.map { |result| [result, 0] } unless report.review?
-
-        lines = report.survivors.group_by { |result| [result.subject.file, edit_line(result)] }
-        lines.values.map do |same_line|
-          best = same_line.min_by do |result|
-            [result.ignored ? 1 : 0, result.mutation.coarse ? 0 : 1, result.mutation.edits.first.start_byte]
-          end
-          [best, same_line.size - 1]
-        end
+        SurvivorList.new(@out, @palette).call(report, ranked_survivors(report))
+        @out.puts unless report.survivors.empty?
       end
 
-      def edit_line(result)
-        result.subject.line_of(result.mutation.edits.first.start_byte)
+      # Live survivors precede accepted ones; each group follows SurvivorRanking.
+      def ranked_survivors(report)
+        live, accepted = report.survivors.partition(&:unignored_survivor?)
+        [live, accepted].flat_map { |group| SurvivorRanking.ordered(report, group) }
       end
 
       def relative(file, root)

data/lib/saboteur/reporter/test_matrix.rb

@@ -0,0 +1,57 @@
+# SPDX-License-Identifier: MIT
+# frozen_string_literal: true
+
+module Saboteur
+  module Reporter
+    # What a --matrix run says about the tests themselves. Every covering test
+    # ran against every mutant, so each killed mutant lists all of its
+    # killers. A blind test covered mutants and killed none of them. A
+    # redundant test killed some, and every one of those had another killer.
+    #
+    # Only mutants whose every planned test is known to have run (see
+    # MutantResult#complete?) count. A run that stopped early or could not
+    # report what it ran is an incomplete mutant: the tests planned for it
+    # are unverified, and no blind or redundant verdict is given for them.
+    class TestMatrix
+      MAX_LISTED = 20
+
+      attr_reader :blind, :redundant, :tests, :incomplete
+
+      def initialize(results)
+        executed = results.select { |result| %i[killed survived].include?(result.status) }
+        complete, partial = executed.partition(&:complete?)
+        @incomplete = partial.size
+        @covered = Hash.new(0)
+        @killed = Hash.new { |hash, test| hash[test] = [] }
+        complete.each { |result| tally(result) }
+        @unverified = partial.flat_map(&:covered_by).uniq
+        @tests = @covered.keys.sort
+        judged = @tests - @unverified
+        @blind = judged.select { |test| @killed[test].empty? }
+        @redundant = judged.select { |test| redundant?(test) }
+      end
+
+      def mutants_covered(test)
+        @covered.fetch(test)
+      end
+
+      def mutants_killed(test)
+        @killed[test].size
+      end
+
+      private
+
+      def tally(result)
+        key = [result.subject.id, result.digest]
+        result.covered_by.each { |test| @covered[test] += 1 }
+        killers = result.killed_by & result.covered_by
+        killers.each { |test| @killed[test] << [key, killers.size - 1] }
+      end
+
+      def redundant?(test)
+        kills = @killed[test]
+        kills.any? && kills.all? { |_key, others| others.positive? }
+      end
+    end
+  end
+end

data/lib/saboteur/result.rb

@@ -2,22 +2,27 @@
 # frozen_string_literal: true
 
 require_relative "reporter/diff"
+require_relative "score"
 
 module Saboteur
   # resource_limit: the mutant's process tree hit the memory or process cap.
   # Like error it is outside the score: a mutant that eats all memory was
   # stopped by saboteur, not detected by a test.
-  STATUSES = %i[killed survived timeout resource_limit error skipped].freeze
+  # ran_at_load: the mutated code only ran while the app loaded; no forked
+  # test can observe a change there, so it is outside the score.
+  # unreachable: a Method object captured before re-evaluation still calls
+  # the original body, so no test can observe the mutant. Outside the score.
+  STATUSES = %i[killed survived timeout resource_limit error skipped ran_at_load unreachable].freeze
 
   # Outcome of one mutation. reason carries the skip reason, the error
   # message or the timeout limit. ignored is the ignore-list reason, if any.
   class MutantResult
     attr_reader :subject, :mutation, :status, :reason, :runtime, :killed_by, :covered_by, :mutated_source,
-                :kill_cause
+                :kill_cause, :executed
     attr_accessor :ignored
 
     def initialize(subject:, mutation:, status:, mutated_source:, reason: nil, runtime: nil, killed_by: [],
-                   covered_by: [], kill_cause: nil, reused: false, rejected_kill: false)
+                   covered_by: [], kill_cause: nil, reused: false, rejected_kill: false, executed: nil)
       raise ArgumentError, "unknown status #{status.inspect}" unless STATUSES.include?(status)
 
       @subject = subject
@@ -29,6 +34,7 @@ module Saboteur
       @covered_by = covered_by
       @mutated_source = mutated_source
       @kill_cause = kill_cause
+      @executed = executed
       @rejected_kill = rejected_kill
       @reused = reused
       @ignored = nil
@@ -38,7 +44,14 @@ module Saboteur
     def as_error(reason, rejected_kill: false)
       MutantResult.new(subject: subject, mutation: mutation, status: :error, mutated_source: mutated_source,
                        reason: reason, runtime: runtime, killed_by: killed_by, covered_by: covered_by,
-                       rejected_kill: rejected_kill)
+                       executed: executed, rejected_kill: rejected_kill)
+    end
+
+    # True when every test planned for this mutant is known to have run.
+    # A killed run that stopped early, a test that called exit, or a verdict
+    # whose executed tests were never recorded is not complete.
+    def complete?
+      !executed.nil? && (covered_by - executed).empty?
     end
 
     def rejected_kill?
@@ -59,6 +72,11 @@ module Saboteur
       status == :killed && !kill_cause.nil? && kill_cause != "assertion"
     end
 
+    # Killed because a test's check failed: the test looked at a value.
+    def assertion_kill?
+      status == :killed && kill_cause == "assertion"
+    end
+
     # Taken from an earlier run (--incremental) instead of run again.
     def reused?
       @reused
@@ -82,14 +100,25 @@ module Saboteur
     end
   end
 
+  # A run that stopped before every selected subject was mutated (--fail-fast).
+  # unreached lists the subject ids that never ran; the verdict covers only
+  # the subjects that did.
+  EarlyStop = Data.define(:reason, :unreached)
+
   # Everything one run produced.
   class Report
-    attr_reader :results, :subjects, :wall_time, :invalid_mutations, :warnings, :root, :seed, :skipped_subjects
+    # More than half of the executed mutants errored: the run tested
+    # almost nothing and must not pass a gate.
+    MAX_ERROR_FRACTION = 0.5
+
+    attr_reader :results, :subjects, :wall_time, :invalid_mutations, :warnings, :root, :seed, :skipped_subjects,
+                :generation_events, :omissions, :run_evidence, :hazard_suppressions, :early_stop, :inline_suppressions
 
     # REVIEW: diff mode; the terminal then shows one survivor per line.
     # seed: the random seed every test run started from (nil when unknown).
     def initialize(results:, subjects:, wall_time:, root:, invalid_mutations: 0, warnings: [], review: false,
-                   seed: nil, skipped_subjects: {})
+                   seed: nil, skipped_subjects: {}, diff_scope: nil, generation_events: [], omissions: [],
+                   run_evidence: nil, hazard_suppressions: 0, early_stop: nil, matrix: false, inline_suppressions: [])
       @results = results
       @subjects = subjects
       @wall_time = wall_time
@@ -99,12 +128,46 @@ module Saboteur
       @review = review
       @seed = seed
       @skipped_subjects = skipped_subjects
+      @diff_scope = diff_scope
+      @generation_events = generation_events.dup.freeze
+      @omissions = omissions.dup.freeze
+      @run_evidence = run_evidence
+      @hazard_suppressions = hazard_suppressions
+      @early_stop = early_stop
+      @matrix = matrix
+      @inline_suppressions = inline_suppressions.dup.freeze
+    end
+
+    # A --matrix run: every covering test ran per mutant.
+    def matrix?
+      @matrix
+    end
+
+    # In diff mode, the changed lines (sorted) that a mutant touches; empty
+    # outside diff mode, or when the mutant touches none (a whole-body mutant
+    # of a method whose only change is its signature).
+    def changed_lines_for(result)
+      @diff_scope ? @diff_scope.changed_lines_for(result.subject, result.mutation) : []
+    end
+
+    def changed_line_distance(result)
+      @diff_scope&.distance_for(result.subject, result.mutation)
     end
 
     def rejected_kills
       results.count(&:rejected_kill?)
     end
 
+    # Errored mutants (a crash, a neutral check that failed) are outside
+    # the score, so a run whose executed mutants mostly errored tested
+    # almost nothing. Skipped mutants never ran: they are neither the
+    # numerator nor the denominator.
+    def majority_errored?
+      return false if executed.empty?
+
+      executed.count { |result| result.status == :error }.fdiv(executed.size) > MAX_ERROR_FRACTION
+    end
+
     def review?
       @review
     end
@@ -115,10 +178,23 @@ module Saboteur
 
     # (killed + timeout) / (killed + survived + timeout); nil with nothing tested.
     def score
-      c = counts
-      detected = c[:killed] + c[:timeout]
-      tested = detected + c[:survived]
-      tested.zero? ? nil : detected.fdiv(tested)
+      score_breakdown.value
+    end
+
+    # The score with its split into assertion kills and crash kills.
+    def score_breakdown
+      @score_breakdown ||= Score.of(results)
+    end
+
+    # The same breakdown without the whole-body mutants (every
+    # mutation marked coarse: body_nil, body_true, body_false, body_raise). Almost any test that runs the method kills
+    # those, so they flatter the score.
+    def fine_score_breakdown
+      @fine_score_breakdown ||= Score.of(results.reject { |result| result.mutation.coarse })
+    end
+
+    def coarse_score_breakdown
+      @coarse_score_breakdown ||= Score.of(results.select { |result| result.mutation.coarse })
     end
 
     def survivors

data/lib/saboteur/score.rb

@@ -0,0 +1,89 @@
+# SPDX-License-Identifier: MIT
+# frozen_string_literal: true
+
+module Saboteur
+  # The mutation score of a group of results, with the share of it that came
+  # from failed assertions and the share that came from crashes.
+  #
+  # value is (killed + timeout) / (killed + survived + timeout). A crash kill
+  # shows the code ran and broke, not that a test checked a value, so the
+  # split lets a reader see how much of the score is checked behavior.
+  # Errors, resource limits and skips are outside every number here.
+  class Score
+    def self.of(results)
+      new(results)
+    end
+
+    attr_reader :assertion_kills, :crash_kills, :unclassified_kills, :timeouts, :survivors
+
+    def initialize(results)
+      killed = results.select { |result| result.status == :killed }
+      @assertion_kills = killed.count(&:assertion_kill?)
+      @crash_kills = killed.count(&:crash_kill?)
+      @unclassified_kills = killed.size - assertion_kills - crash_kills
+      @timeouts = results.count { |result| result.status == :timeout }
+      @survivors = results.count(&:survived?)
+    end
+
+    def detected
+      assertion_kills + crash_kills + unclassified_kills + timeouts
+    end
+
+    def tested
+      detected + survivors
+    end
+
+    # nil with nothing tested.
+    def value
+      share(detected)
+    end
+
+    def assertion_share
+      share(assertion_kills)
+    end
+
+    def crash_share
+      share(crash_kills)
+    end
+
+    def unclassified_share
+      share(unclassified_kills)
+    end
+
+    def timeout_share
+      share(timeouts)
+    end
+
+    # False when no kill has a recorded cause: a split would then say
+    # nothing. A cause is missing for verdicts reused from a run made before
+    # causes were recorded, for a test that called exit, and for a failure
+    # outside any example (suite level).
+    def split_known?
+      assertion_kills.positive? || crash_kills.positive?
+    end
+
+    # [[label, share]]: assertion and crash always, then timeout and
+    # "cause not recorded" only when there are some, so the parts add up to
+    # value.
+    def parts
+      list = [["assertion", assertion_share], ["crash", crash_share]]
+      list << ["timeout", timeout_share] if timeouts.positive?
+      list << ["cause not recorded", unclassified_share] if unclassified_kills.positive?
+      list
+    end
+
+    def to_h
+      {
+        score: value, tested: tested, assertionKills: assertion_kills, crashKills: crash_kills,
+        unclassifiedKills: unclassified_kills, timeouts: timeouts, survivors: survivors,
+        assertionShare: assertion_share, crashShare: crash_share
+      }
+    end
+
+    private
+
+    def share(count)
+      tested.zero? ? nil : count.fdiv(tested)
+    end
+  end
+end