saboteur 0.3.0 → 0.4.0

data/lib/saboteur/reporter/agent_summary.rb

@@ -0,0 +1,131 @@
+# SPDX-License-Identifier: MIT
+# frozen_string_literal: true
+
+require_relative "findings"
+require_relative "agent_incomplete"
+require_relative "agent_clipping"
+
+module Saboteur
+  module Reporter
+    module Agent
+      # Builds the payload hash for one run: status, verdict, exit code, counts,
+      # provenance, a few findings and the next commands.
+      module Summary
+        class << self
+          include Clipping
+
+          # Public entry first; the helpers below are private.
+          def build(report, provenance, shown, snippet_limit)
+            incomplete = Incomplete.reasons(report)
+            status = status_of(report, incomplete)
+            verdict = verdict_of(report, status)
+            {
+              version: VERSION, status: status, verdict: verdict, exitCode: exit_code_for(verdict),
+              provenance: provenance_hash(provenance),
+              counts: counts(report), score: score(report),
+              scoreDefinition: "(killed + timeout) / (killed + survived + timeout); null when nothing was tested",
+              incomplete: incomplete, timeoutCountedAsDetected: report.counts[:timeout],
+              findings: findings(report, shown, snippet_limit),
+              warnings: warnings(report),
+              next: next_steps(report, provenance)
+            }
+          end
+
+          private
+
+          def status_of(report, incomplete)
+            return "not_applicable" if report.results.empty? && report.skipped_subjects.empty?
+
+            incomplete.empty? ? "complete" : "partial"
+          end
+
+          def verdict_of(report, status)
+            return "survivors" if report.unignored_survivors.any?
+
+            { "complete" => "clean", "partial" => "incomplete", "not_applicable" => "not_applicable" }.fetch(status)
+          end
+
+          def exit_code_for(verdict)
+            case verdict
+            when "clean" then EXIT_CLEAN
+            when "survivors" then EXIT_REVIEW
+            else EXIT_INCOMPLETE
+            end
+          end
+
+          def counts(report)
+            report.counts.transform_keys { |status| camel(status) }.merge(executed: report.executed.size)
+          end
+
+          def camel(status)
+            status.to_s.gsub(/_(\w)/) { Regexp.last_match(1).upcase }.to_sym
+          end
+
+          def score(report)
+            report.score&.round(4)
+          end
+
+          # Display values are clipped; the commands under next use the exact values or are left out.
+          def provenance_hash(provenance)
+            { ref: clip_value(provenance.ref, 120), seed: bounded_number(provenance.seed),
+              head: clip_value(provenance.head, 64),
+              workingTree: provenance.working_tree, changedFiles: provenance.changed_files,
+              mode: provenance.mode, selection: selection_hash(provenance) }
+          end
+
+          def selection_hash(provenance)
+            %i[subjects tests sources].to_h do |key|
+              list = provenance.public_send(key)
+              [key, { count: list.size, first: list.first(3).map { |entry| clip_value(entry, 120) } }]
+            end.merge(config: clip_value(provenance.config_path, 120), digest: provenance.digest,
+                      settings: provenance.settings.transform_values { |value| clip_value(value, 120) })
+          end
+
+          def findings(report, shown, snippet_limit)
+            all = Findings.build(report)
+            live = report.unignored_survivors.size
+            {
+              survivors: live, shown: shown_findings(all[:survivors], shown, snippet_limit),
+              omitted: live - [shown, all[:survivors].size].min,
+              accepted: report.survivors.size - live, ordering: all[:ordering]
+            }
+          end
+
+          def shown_findings(listed, shown, snippet_limit)
+            listed.first(shown).map do |finding|
+              finding.slice(:digest, :line, :endLine, :operator, :coarse)
+                     .merge(path: clip_value(finding[:path], 200), subject: clip_value(finding[:subject], 200),
+                            original: clip(finding[:original], snippet_limit),
+                            replacement: clip(finding[:replacement], snippet_limit))
+            end
+          end
+
+          def warnings(report)
+            report.warnings.first(WARNING_LIMIT).map { |warning| warning.to_s[0, WARNING_LENGTH] }
+          end
+
+          def next_steps(report, provenance)
+            steps = {}
+            add_command(steps, :rerun, Commands.rerun(provenance))
+            first = Findings.build(report)[:survivors].first
+            add_command(steps, :replay, Commands.replay(provenance, first[:digest])) if first
+            steps[:fullReport] = provenance.json_report if short_path?(provenance.json_report)
+            steps
+          end
+
+          def short_path?(path)
+            path && path.bytesize <= 400
+          end
+
+          def add_command(steps, key, command)
+            steps[key] = command
+            return if command
+
+            steps[:"#{key}Unavailable"] =
+              "command longer than #{Commands::MAX_BYTES} bytes; rebuild it from provenance"
+          end
+        end
+      end
+    end
+  end
+end

data/lib/saboteur/reporter/findings.rb

@@ -76,9 +76,17 @@ module Saboteur
             edits: entries,
             coarse: result.mutation.coarse
           }
+          entry = with_changed_lines(report, result, entry)
           result.ignored ? entry.merge(reason: result.ignored) : entry
         end
 
+        # In diff mode, the changed lines the mutant touches. line above is where
+        # the mutated statement starts, which can be above the changed line.
+        def with_changed_lines(report, result, entry)
+          lines = report.changed_lines_for(result)
+          lines.empty? ? entry : entry.merge(changedLines: lines)
+        end
+
         # Subject#apply applies edits in start-byte order; findings use the
         # same order, so an unsorted edit list from a custom mutation source
         # still gets a sane range and one entry per edit.

data/lib/saboteur/reporter/html.rb

@@ -76,6 +76,7 @@ module Saboteur
           <p class="note">Covered mutation score = #{SCORE_FORMULA}. NoCoverage, errors, resource limits and skipped
           mutants are not counted. Survivors accepted on the ignore list stay in the denominator. It covers only the subjects of this run, not the whole project. A tool that reads the
           Stryker JSON may show a different score, because the schema counts NoCoverage as undetected.</p>
+          #{score_split(report)}
           <p class="note">Wide tables scroll sideways.</p>
           <p>#{counts.map { |status, count| "#{status_label(status)} #{count}" }.join(' ')}</p>
           <p>Survivors: #{report.unignored_survivors.size} to fix, #{report.survivors.size - report.unignored_survivors.size}
@@ -83,6 +84,37 @@ module Saboteur
         HTML
       end
 
+      # How the score splits into assertion and crash kills, and the score
+      # without the whole-body mutants, which almost any test that runs the
+      # method kills. Each part shows only when it has something to say.
+      def score_split(report)
+        breakdown = report.score_breakdown
+        lines = []
+        lines << kill_split_line(breakdown) if breakdown.split_known?
+        lines << whole_body_line(report)
+        lines.compact.join("\n")
+      end
+
+      def kill_split_line(breakdown)
+        "<p>Of the tested mutants: #{parts_text(breakdown)}. A crash shows the code ran and broke, " \
+          "not that a test checked a value.</p>"
+      end
+
+      def parts_text(breakdown)
+        breakdown.parts.map { |label, share| "#{score_text(share)} #{label}" }.join(", ")
+      end
+
+      def whole_body_line(report)
+        whole = report.coarse_score_breakdown
+        fine = report.fine_score_breakdown
+        return if whole.tested.zero? || fine.tested.zero?
+
+        split = fine.split_known? ? " (#{parts_text(fine)})" : ""
+        "<p>Without whole-body mutants (body_nil, body_true, body_false, body_raise): #{score_text(fine.value)}" \
+          "#{split} over #{fine.tested} mutants. Whole-body mutants alone: #{score_text(whole.value)} " \
+          "over #{whole.tested}.</p>"
+      end
+
       def score_text(score)
         score ? format("%.1f%%", score * 100) : "n/a"
       end
@@ -126,13 +158,29 @@ module Saboteur
       def mutant_row(mutant, names, result)
         start = mutant[:location][:start]
         tests = Array(mutant[:killedBy]).map { |id| names.fetch(id) }.join(", ")
-        notes = [mutant[:statusReason], tests.empty? ? nil : "killed by #{tests}"].compact.join("; ")
+        notes = [mutant[:statusReason], tests.empty? ? nil : "killed by #{tests}",
+                 passing_tests_note(mutant, names, result)].compact.join("; ")
         "<tr><td class=\"fit\">line #{start[:line]}, column #{start[:column]}</td><td>#{row_status(mutant,
                                                                                                    result)}</td>" \
           "<td>#{escape(mutant[:mutatorName])}</td><td class=\"wide\">#{change(mutant, result)}</td>" \
           "<td class=\"fit\"><code>#{escape(mutant[:id])}</code></td><td class=\"wide\">#{escape(notes)}</td></tr>"
       end
 
+      # A survivor's covering tests all passed with the mutant applied, or, for
+      # a reused verdict, in the earlier run it came from.
+      def passing_tests_note(mutant, names, result)
+        return unless mutant[:status] == "Survived"
+
+        passed = Array(mutant[:coveredBy]).map { |id| names.fetch(id) }
+        return if passed.empty?
+
+        shown = passed.first(Terminal::MAX_PASSING_TESTS).join(", ")
+        more = passed.size - Terminal::MAX_PASSING_TESTS
+        extra = more.positive? ? " (+#{more} more)" : ""
+        when_passed = result.reused? ? "passed in an earlier run" : "passed"
+        "covered by #{passed.size} #{passed.one? ? 'test' : 'tests'} that #{when_passed}: #{shown}#{extra}"
+      end
+
       def row_status(mutant, result)
         badge = status_label(mutant[:status])
         # Only survivors are ever put on the ignore list.

data/lib/saboteur/reporter/stryker.rb

@@ -16,14 +16,14 @@ module Saboteur
         @path = path
       end
 
-      def call(report)
+      def call(report, exit_cause: nil)
         FileUtils.mkdir_p(File.dirname(@path))
-        File.write(@path, JSON.generate(self.class.build(report)))
+        File.write(@path, JSON.generate(self.class.build(report, exit_cause: exit_cause)))
       end
 
       # Tests are listed once under testFiles with short ids; mutants refer to
       # them by id in killedBy/coveredBy instead of repeating full test names.
-      def self.build(report)
+      def self.build(report, exit_cause: nil)
         test_ids = test_index(report.results)
         files = report.results.group_by { |result| result.subject.file }.to_h do |file, results|
           [relative(file, report.root), file_entry(results, test_ids)]
@@ -38,10 +38,17 @@ module Saboteur
           rejectedKills: report.rejected_kills,
           warnings: report.warnings,
           framework: { name: "saboteur", version: VERSION },
-          saboteur: { findings: Findings.build(report) }
+          saboteur: { findings: Findings.build(report), scores: scores(report) }
+                     .merge(exit_cause ? { exitCause: exit_cause } : {})
         }.merge(report.seed ? { config: { seed: report.seed } } : {})
       end
 
+      # Score with its assertion and crash shares, for all mutants and without
+      # the whole-body ones (body_nil, body_true, body_false, body_raise).
+      def self.scores(report)
+        { all: report.score_breakdown.to_h, withoutWholeBodyMutants: report.fine_score_breakdown.to_h }
+      end
+
       def self.skipped_subjects(report)
         report.skipped_subjects.map do |subject, reason|
           { subject: subject.id, file: relative(subject.file, report.root), line: subject.start_line, reason: reason }

data/lib/saboteur/reporter/terminal.rb

@@ -7,6 +7,8 @@ module Saboteur
     # errors, then a unified diff for each survivor.
     class Terminal
       MAX_ERRORS = 10
+      MAX_PASSING_TESTS = 5
+      MAX_CHANGED_LINES = 3
 
       def initialize(out)
         @out = out
@@ -40,9 +42,10 @@ module Saboteur
       end
 
       def details(report)
-        score = report.score
-        @out.puts "score #{score ? format('%.1f %%', score * 100) : 'n/a'} " \
+        breakdown = report.score_breakdown
+        @out.puts "score #{percent(breakdown.value)}#{score_split(breakdown)} " \
                   "(killed+timeout)/(killed+survived+timeout); errors, resource limits and skips excluded"
+        whole_body_score(report)
         mean = report.mean_runtime
         @out.puts "mean time per executed mutant #{format('%.3f', mean)} s" if mean
         @out.puts "invalid mutations dropped (did not parse): #{report.invalid_mutations}" if
@@ -55,6 +58,31 @@ module Saboteur
         kill_causes(report)
       end
 
+      # " (assertion 50.0 %, crash 50.0 %)": how much of the score is a failed
+      # check and how much is a crash. Timeouts and kills with no recorded
+      # cause are named only when there are some, so the parts add up.
+      def score_split(breakdown)
+        return "" unless breakdown.split_known?
+
+        " (#{breakdown.parts.map { |label, share| "#{label} #{percent(share)}" }.join(', ')})"
+      end
+
+      # Whole-body mutants (body_nil, body_true, body_false, body_raise) are
+      # killed by almost any test that runs the method, so they get their own
+      # line.
+      def whole_body_score(report)
+        whole = report.coarse_score_breakdown
+        fine = report.fine_score_breakdown
+        return if whole.tested.zero? || fine.tested.zero?
+
+        @out.puts "score without whole-body mutants #{percent(fine.value)}#{score_split(fine)} " \
+                  "(#{fine.tested} mutants); whole-body mutants alone #{percent(whole.value)} (#{whole.tested})"
+      end
+
+      def percent(share)
+        share ? format("%.1f %%", share * 100) : "n/a"
+      end
+
       # How many kills came from a failed check and how many from a crash, and
       # which operators mostly kill by crashing (their score says less).
       def kill_causes(report)
@@ -114,12 +142,47 @@ module Saboteur
         shown(report).each do |result, hidden|
           location = "#{relative(result.subject.file, report.root)}:#{result.subject.start_line}"
           @out.puts "\n#{result.subject.id} #{result.mutation.operator} #{result.digest} #{location}"
+          changed_lines(report, result)
           @out.puts "ignored: #{result.ignored}" if result.ignored
           @out.puts result.diff(root: report.root)
+          passing_tests(result)
           @out.puts "#{hidden} more survivor#{'s' if hidden > 1} on this line in the JSON report" if hidden.positive?
         end
       end
 
+      # The tests that ran with the mutant applied and still passed: the
+      # missing assertion belongs in one of them. They are listed likeliest
+      # killers first, the order the run used. Line coverage can overstate which
+      # tests exercise the line, so the list is a lead, not proof.
+      def passing_tests(result)
+        tests = result.covered_by
+        return if tests.empty?
+
+        @out.puts "#{tests.size} #{tests.one? ? 'test' : 'tests'} covered this line and #{passed_phrase(result)} " \
+                  "(the missing check may belong in one of them):"
+        tests.first(MAX_PASSING_TESTS).each { |name| @out.puts "  #{name}" }
+        more = tests.size - MAX_PASSING_TESTS
+        @out.puts "  ... #{more} more in the JSON report (coveredBy)" if more.positive?
+      end
+
+      # A reused verdict was not run again: its tests passed in the earlier run.
+      def passed_phrase(result)
+        result.reused? ? "passed in an earlier run (--incremental)" : "still passed"
+      end
+
+      # In diff mode: the changed lines this mutant touches. The location above
+      # is where the mutated statement starts, which can be a line or two above
+      # the line that changed.
+      def changed_lines(report, result)
+        lines = report.changed_lines_for(result)
+        return if lines.empty?
+
+        path = relative(result.subject.file, report.root)
+        shown = lines.first(MAX_CHANGED_LINES).map { |line| "#{path}:#{line}" }.join(", ")
+        more = lines.size - MAX_CHANGED_LINES
+        @out.puts "changed #{lines.one? ? 'line' : 'lines'}: #{shown}#{" (+#{more} more)" if more.positive?}"
+      end
+
       # [[result, how many more on the same line]]. In review mode survivors
       # on one line usually share a fate, so one is shown: live before
       # ignored, then coarse before fine, then the earliest edit.

data/lib/saboteur/result.rb

@@ -2,6 +2,7 @@
 # frozen_string_literal: true
 
 require_relative "reporter/diff"
+require_relative "score"
 
 module Saboteur
   # resource_limit: the mutant's process tree hit the memory or process cap.
@@ -59,6 +60,11 @@ module Saboteur
       status == :killed && !kill_cause.nil? && kill_cause != "assertion"
     end
 
+    # Killed because a test's check failed: the test looked at a value.
+    def assertion_kill?
+      status == :killed && kill_cause == "assertion"
+    end
+
     # Taken from an earlier run (--incremental) instead of run again.
     def reused?
       @reused
@@ -84,12 +90,16 @@ module Saboteur
 
   # Everything one run produced.
   class Report
+    # More than half of the executed mutants errored: the run tested
+    # almost nothing and must not pass a gate.
+    MAX_ERROR_FRACTION = 0.5
+
     attr_reader :results, :subjects, :wall_time, :invalid_mutations, :warnings, :root, :seed, :skipped_subjects
 
     # REVIEW: diff mode; the terminal then shows one survivor per line.
     # seed: the random seed every test run started from (nil when unknown).
     def initialize(results:, subjects:, wall_time:, root:, invalid_mutations: 0, warnings: [], review: false,
-                   seed: nil, skipped_subjects: {})
+                   seed: nil, skipped_subjects: {}, diff_scope: nil)
       @results = results
       @subjects = subjects
       @wall_time = wall_time
@@ -99,12 +109,30 @@ module Saboteur
       @review = review
       @seed = seed
       @skipped_subjects = skipped_subjects
+      @diff_scope = diff_scope
+    end
+
+    # In diff mode, the changed lines (sorted) that a mutant touches; empty
+    # outside diff mode, or when the mutant touches none (a whole-body mutant
+    # of a method whose only change is its signature).
+    def changed_lines_for(result)
+      @diff_scope ? @diff_scope.changed_lines_for(result.subject, result.mutation) : []
     end
 
     def rejected_kills
       results.count(&:rejected_kill?)
     end
 
+    # Errored mutants (a crash, a neutral check that failed) are outside
+    # the score, so a run whose executed mutants mostly errored tested
+    # almost nothing. Skipped mutants never ran: they are neither the
+    # numerator nor the denominator.
+    def majority_errored?
+      return false if executed.empty?
+
+      executed.count { |result| result.status == :error }.fdiv(executed.size) > MAX_ERROR_FRACTION
+    end
+
     def review?
       @review
     end
@@ -115,10 +143,23 @@ module Saboteur
 
     # (killed + timeout) / (killed + survived + timeout); nil with nothing tested.
     def score
-      c = counts
-      detected = c[:killed] + c[:timeout]
-      tested = detected + c[:survived]
-      tested.zero? ? nil : detected.fdiv(tested)
+      score_breakdown.value
+    end
+
+    # The score with its split into assertion kills and crash kills.
+    def score_breakdown
+      @score_breakdown ||= Score.of(results)
+    end
+
+    # The same breakdown without the whole-body mutants (every
+    # mutation marked coarse: body_nil, body_true, body_false, body_raise). Almost any test that runs the method kills
+    # those, so they flatter the score.
+    def fine_score_breakdown
+      @fine_score_breakdown ||= Score.of(results.reject { |result| result.mutation.coarse })
+    end
+
+    def coarse_score_breakdown
+      @coarse_score_breakdown ||= Score.of(results.select { |result| result.mutation.coarse })
     end
 
     def survivors

data/lib/saboteur/score.rb

@@ -0,0 +1,89 @@
+# SPDX-License-Identifier: MIT
+# frozen_string_literal: true
+
+module Saboteur
+  # The mutation score of a group of results, with the share of it that came
+  # from failed assertions and the share that came from crashes.
+  #
+  # value is (killed + timeout) / (killed + survived + timeout). A crash kill
+  # shows the code ran and broke, not that a test checked a value, so the
+  # split lets a reader see how much of the score is checked behavior.
+  # Errors, resource limits and skips are outside every number here.
+  class Score
+    def self.of(results)
+      new(results)
+    end
+
+    attr_reader :assertion_kills, :crash_kills, :unclassified_kills, :timeouts, :survivors
+
+    def initialize(results)
+      killed = results.select { |result| result.status == :killed }
+      @assertion_kills = killed.count(&:assertion_kill?)
+      @crash_kills = killed.count(&:crash_kill?)
+      @unclassified_kills = killed.size - assertion_kills - crash_kills
+      @timeouts = results.count { |result| result.status == :timeout }
+      @survivors = results.count(&:survived?)
+    end
+
+    def detected
+      assertion_kills + crash_kills + unclassified_kills + timeouts
+    end
+
+    def tested
+      detected + survivors
+    end
+
+    # nil with nothing tested.
+    def value
+      share(detected)
+    end
+
+    def assertion_share
+      share(assertion_kills)
+    end
+
+    def crash_share
+      share(crash_kills)
+    end
+
+    def unclassified_share
+      share(unclassified_kills)
+    end
+
+    def timeout_share
+      share(timeouts)
+    end
+
+    # False when no kill has a recorded cause: a split would then say
+    # nothing. A cause is missing for verdicts reused from a run made before
+    # causes were recorded, for a test that called exit, and for a failure
+    # outside any example (suite level).
+    def split_known?
+      assertion_kills.positive? || crash_kills.positive?
+    end
+
+    # [[label, share]]: assertion and crash always, then timeout and
+    # "cause not recorded" only when there are some, so the parts add up to
+    # value.
+    def parts
+      list = [["assertion", assertion_share], ["crash", crash_share]]
+      list << ["timeout", timeout_share] if timeouts.positive?
+      list << ["cause not recorded", unclassified_share] if unclassified_kills.positive?
+      list
+    end
+
+    def to_h
+      {
+        score: value, tested: tested, assertionKills: assertion_kills, crashKills: crash_kills,
+        unclassifiedKills: unclassified_kills, timeouts: timeouts, survivors: survivors,
+        assertionShare: assertion_share, crashShare: crash_share
+      }
+    end
+
+    private
+
+    def share(count)
+      tested.zero? ? nil : count.fdiv(tested)
+    end
+  end
+end

data/lib/saboteur/version.rb

@@ -2,5 +2,5 @@
 # frozen_string_literal: true
 
 module Saboteur
-  VERSION = "0.3.0"
+  VERSION = "0.4.0"
 end

data/lib/saboteur/working_tree.rb

@@ -0,0 +1,48 @@
+# SPDX-License-Identifier: MIT
+# frozen_string_literal: true
+
+require "open3"
+
+module Saboteur
+  # The state of the working source a run looked at: the commit it was on and
+  # (the full SHA) and whether files differed from it, tracked changes and each
+  # untracked file alike.
+  # Saboteur's own output (cache and reports) is passed in as ignored paths, so a
+  # report from an earlier run does not make the tree look dirty. Without git, or
+  # outside a repository, the state is unknown.
+  class WorkingTree
+    State = Data.define(:head, :dirty, :changed_files) do
+      def label
+        return "unknown" if dirty.nil?
+
+        dirty ? "dirty" : "clean"
+      end
+    end
+
+    UNKNOWN = State.new(head: nil, dirty: nil, changed_files: nil)
+
+    # ignored: paths relative to root that are not source (saboteur's cache and reports).
+    def initialize(root, ignored: [])
+      @root = root
+      @ignored = ignored
+    end
+
+    def state
+      head, head_status = Open3.capture2e("git", "rev-parse", "HEAD", chdir: @root)
+      changes, changes_status = Open3.capture2e(*status_command, chdir: @root)
+      return UNKNOWN unless head_status.success? && changes_status.success?
+
+      count = changes.lines.size
+      State.new(head: head.strip, dirty: count.positive?, changed_files: count)
+    rescue SystemCallError
+      UNKNOWN
+    end
+
+    private
+
+    def status_command
+      excluded = @ignored.map { |path| ":(exclude)#{path}" }
+      ["git", "status", "--porcelain", "--untracked-files=all", "--", ".", *excluded]
+    end
+  end
+end

data/lib/saboteur.rb

@@ -43,11 +43,15 @@ require_relative "saboteur/selection/coverage_map"
 require_relative "saboteur/selection/recorder"
 require_relative "saboteur/selection/git_diff"
 require_relative "saboteur/result"
+require_relative "saboteur/custom_map"
 require_relative "saboteur/config"
 require_relative "saboteur/ignore_list"
 require_relative "saboteur/frameworks/rails"
 require_relative "saboteur/reporter/diff"
 require_relative "saboteur/reporter/findings"
+require_relative "saboteur/reporter/agent"
+require_relative "saboteur/working_tree"
+require_relative "saboteur/agent_skill"
 require_relative "saboteur/reporter/terminal"
 require_relative "saboteur/reporter/stryker"
 require_relative "saboteur/reporter/html"