saboteur 0.3.0 → 0.5.0

data/lib/saboteur/reporter/agent_summary.rb

@@ -0,0 +1,137 @@
+# SPDX-License-Identifier: MIT
+# frozen_string_literal: true
+
+require_relative "findings"
+require_relative "survivor_hints"
+require_relative "agent_incomplete"
+require_relative "agent_clipping"
+
+module Saboteur
+  module Reporter
+    module Agent
+      # Builds the payload hash for one run: status, verdict, exit code, counts,
+      # provenance, a few findings and the next commands.
+      module Summary
+        class << self
+          include Clipping
+
+          # Public entry first; the helpers below are private.
+          def build(report, provenance, shown, snippet_limit)
+            incomplete = Incomplete.reasons(report)
+            status = status_of(report, incomplete)
+            verdict = verdict_of(report, status)
+            {
+              version: VERSION, status: status, verdict: verdict, exitCode: exit_code_for(verdict),
+              provenance: provenance_hash(provenance),
+              counts: counts(report), score: score(report),
+              scoreDefinition: "(killed + timeout) / (killed + survived + timeout); null when nothing was tested",
+              incomplete: incomplete, timeoutCountedAsDetected: report.counts[:timeout],
+              findings: findings(report, shown, snippet_limit),
+              warnings: report.warnings.first(WARNING_LIMIT).map { |warning| warning.to_s[0, WARNING_LENGTH] },
+              next: next_steps(report, provenance)
+            }
+          end
+
+          private
+
+          def status_of(report, incomplete)
+            return "not_applicable" if report.results.empty? && incomplete.empty?
+
+            incomplete.empty? ? "complete" : "partial"
+          end
+
+          def verdict_of(report, status)
+            return "survivors" if report.unignored_survivors.any?
+
+            { "complete" => "clean", "partial" => "incomplete", "not_applicable" => "not_applicable" }.fetch(status)
+          end
+
+          def exit_code_for(verdict)
+            case verdict
+            when "clean" then EXIT_CLEAN
+            when "survivors" then EXIT_REVIEW
+            else EXIT_INCOMPLETE
+            end
+          end
+
+          def counts(report)
+            report.counts.transform_keys { |status| camel(status) }.merge(executed: report.executed.size)
+          end
+
+          def camel(status)
+            status.to_s.gsub(/_(\w)/) { Regexp.last_match(1).upcase }.to_sym
+          end
+
+          def score(report)
+            report.score&.round(4)
+          end
+
+          # Display values are clipped; the commands under next use the exact values or are left out.
+          def provenance_hash(provenance)
+            { ref: clip_value(provenance.ref, 120), seed: bounded_number(provenance.seed),
+              head: clip_value(provenance.head, 64),
+              workingTree: provenance.working_tree, changedFiles: provenance.changed_files,
+              mode: provenance.mode, selection: selection_hash(provenance) }
+          end
+
+          def selection_hash(provenance)
+            %i[subjects tests sources].to_h do |key|
+              list = provenance.public_send(key)
+              [key, { count: list.size, first: list.first(3).map { |entry| clip_value(entry, 120) } }]
+            end.merge(config: clip_value(provenance.config_path, 120), digest: provenance.digest,
+                      settings: provenance.settings.transform_values { |value| clip_value(value, 120) })
+          end
+
+          def findings(report, shown, snippet_limit)
+            all = Findings.build(report)
+            live = report.unignored_survivors.size
+            {
+              survivors: live, shown: shown_findings(all[:survivors], shown, snippet_limit),
+              omitted: live - [shown, all[:survivors].size].min,
+              accepted: report.survivors.size - live, ordering: all[:ordering],
+              hintCaveat: SurvivorHints::CAVEAT
+            }
+          end
+
+          def shown_findings(listed, shown, snippet_limit)
+            listed.first(shown).map do |finding|
+              finding.slice(:digest, :line, :endLine, :operator, :coarse)
+                     .merge(path: clip_value(finding[:path], 200), subject: clip_value(finding[:subject], 200),
+                            original: clip(finding[:original], snippet_limit),
+                            replacement: clip(finding[:replacement], snippet_limit))
+                     .merge(finding_hints(finding))
+            end
+          end
+
+          def finding_hints(finding)
+            structured = SurvivorHints.structured(operator: finding[:operator], original: finding[:original],
+                                                  replacement: finding[:replacement], coarse: finding[:coarse])
+            { hint: SurvivorHints.for(finding[:operator], coarse: finding[:coarse]) }.compact
+                                                                                     .merge(testHint: structured)
+          end
+
+          def next_steps(report, provenance)
+            steps = {}
+            add_command(steps, :rerun, Commands.rerun(provenance))
+            first = Findings.build(report)[:survivors].first
+            add_command(steps, :replay, Commands.replay(provenance, first[:digest])) if first
+            steps[:fullReport] = provenance.json_report if short_path?(provenance.json_report)
+            steps
+          end
+
+          def short_path?(path)
+            path && path.bytesize <= 400
+          end
+
+          def add_command(steps, key, command)
+            steps[key] = command
+            return if command
+
+            steps[:"#{key}Unavailable"] =
+              "command longer than #{Commands::MAX_BYTES} bytes; rebuild it from provenance"
+          end
+        end
+      end
+    end
+  end
+end

data/lib/saboteur/reporter/findings.rb

@@ -1,6 +1,8 @@
 # SPDX-License-Identifier: MIT
 # frozen_string_literal: true
 
+require_relative "survivor_ranking"
+
 module Saboteur
   module Reporter
     # Findings for a code-review surface such as the saboteur GitHub Action:
@@ -15,12 +17,7 @@ module Saboteur
       # GitHub shows at most 10 annotations of one kind per step.
       LIMIT = 10
 
-      # Published ordering, coarse survivors first: a surviving coarse mutant
-      # replaced the whole method body, which hints that no test pins the
-      # method's behavior - though it can also be a re-evaluation limit or an
-      # equivalent mutant, so this is a heuristic, not a guarantee. Ties break
-      # by file, line and digest, so one report always reads the same way.
-      ORDERING = "coarse survivors first, then file, line and digest"
+      ORDERING = SurvivorRanking::ORDERING
 
       class << self
         def build(report)
@@ -46,20 +43,8 @@ module Saboteur
           [results.size - LIMIT, 0].max
         end
 
-        # partition keeps the coarse group ahead; inside a group the key is
-        # total (no two mutants share a digest), so the order is stable.
         def ordered(report, results)
-          results.partition { |result| result.mutation.coarse }
-                 .map { |group| group.sort_by { |result| tie_break(report, result) } }
-                 .flatten
-        end
-
-        def tie_break(report, result)
-          [path(report, result), first_line(result), result.digest]
-        end
-
-        def first_line(result)
-          result.subject.line_of(result.mutation.edits.map(&:start_byte).min)
+          SurvivorRanking.ordered(report, results)
         end
 
         def finding(report, result)
@@ -76,9 +61,17 @@ module Saboteur
             edits: entries,
             coarse: result.mutation.coarse
           }
+          entry = with_changed_lines(report, result, entry)
           result.ignored ? entry.merge(reason: result.ignored) : entry
         end
 
+        # In diff mode, the changed lines the mutant touches. line above is where
+        # the mutated statement starts, which can be above the changed line.
+        def with_changed_lines(report, result, entry)
+          lines = report.changed_lines_for(result)
+          lines.empty? ? entry : entry.merge(changedLines: lines)
+        end
+
         # Subject#apply applies edits in start-byte order; findings use the
         # same order, so an unsorted edit list from a custom mutation source
         # still gets a sane range and one entry per edit.

data/lib/saboteur/reporter/history_paths.rb

@@ -0,0 +1,38 @@
+# SPDX-License-Identifier: MIT
+# frozen_string_literal: true
+
+module Saboteur
+  module Reporter
+    # Refuses linked or non-regular history/report destinations before writes.
+    module HistoryPaths
+      module_function
+
+      def validate(paths)
+        paths = paths.compact.map { |path| File.expand_path(path) }
+        identities = paths.map do |path|
+          reject_links(path)
+          next unless File.exist?(path)
+
+          stat = File.stat(path)
+          raise Error, "score history/report destination must be a regular file" unless stat.file?
+
+          [stat.dev, stat.ino]
+        end.compact
+        raise Error, "score history/report destinations alias the same file" unless identities.uniq == identities
+      rescue SystemCallError => e
+        raise Error, "score history/report paths cannot be verified: #{e.message}"
+      end
+
+      def reject_links(path)
+        loop do
+          raise Error, "score history/report paths must not contain symlinks" if File.symlink?(path)
+
+          parent = File.dirname(path)
+          break if parent == path
+
+          path = parent
+        end
+      end
+    end
+  end
+end

data/lib/saboteur/reporter/history_record.rb

@@ -0,0 +1,132 @@
+# SPDX-License-Identifier: MIT
+# frozen_string_literal: true
+
+require "json"
+require "time"
+
+module Saboteur
+  module Reporter
+    # Strict version-one observation schema. Never repairs imported evidence.
+    module HistoryRecord
+      FIELDS = %w[version date commit selection inventory score coveredScore counts breakdown incomplete].freeze
+      BREAKDOWN = %w[score tested assertionKills crashKills unclassifiedKills timeouts survivors
+                     assertionShare crashShare].freeze
+      COUNTS = STATUSES.map(&:to_s).freeze
+
+      class UniqueObject < Hash
+        def []=(key, value)
+          raise Error, "score history contains duplicate keys" if key?(key)
+
+          super
+        end
+      end
+
+      module_function
+
+      def parse(line)
+        record = JSON.parse(line, object_class: UniqueObject)
+        validate(record)
+        record
+      rescue JSON::ParserError, ArgumentError, TypeError => e
+        raise Error, "score history contains an invalid record: #{e.message}"
+      end
+
+      def validate(record)
+        fields(record, FIELDS)
+        unless record.fetch("version").is_a?(Integer) && record.fetch("version") == 1
+          raise Error,
+                "score history contains an invalid version"
+        end
+
+        date(record.fetch("date"))
+        commit = record.fetch("commit")
+        raise Error, "score history contains an invalid commit" unless commit.nil? || sha?(commit, 40)
+
+        %w[selection inventory].each do |key|
+          raise Error, "score history contains an invalid digest" unless sha?(record.fetch(key), 64)
+        end
+        %w[score coveredScore].each { |key| fraction(record.fetch(key)) }
+        counts = record.fetch("counts")
+        fields(counts, COUNTS)
+        counts.each_value { |value| count(value) }
+        breakdown(record.fetch("breakdown"), counts, record.fetch("coveredScore"))
+        score_consistency(record)
+        reasons(record.fetch("incomplete"))
+      end
+
+      def date(value)
+        return if Time.iso8601(value).utc.iso8601 == value
+
+        raise Error,
+              "score history contains an invalid date"
+      rescue ArgumentError, TypeError, NoMethodError => e
+        raise Error, "score history contains an invalid date: #{e.message}"
+      end
+
+      def reasons(value)
+        unless value.is_a?(Array) && value.all? do |reason|
+          reason.is_a?(String) && !reason.empty?
+        end && value.uniq == value
+          raise Error, "score history contains invalid incomplete reasons"
+        end
+      end
+
+      def score_consistency(record)
+        value = record.fetch("breakdown")
+        detected = value.values_at("assertionKills", "crashKills", "unclassifiedKills", "timeouts").sum
+        tested = value.fetch("tested")
+        expected = tested.zero? ? nil : detected.fdiv(tested)
+        raise Error, "score history contains an inconsistent score" unless record.fetch("coveredScore") == expected
+
+        %w[assertion crash].each do |kind|
+          expected_share = tested.zero? ? nil : value.fetch("#{kind}Kills").fdiv(tested)
+          unless value.fetch("#{kind}Share") == expected_share
+            raise Error,
+                  "score history contains an inconsistent share"
+          end
+        end
+        overall = record.fetch("score")
+        return unless overall && ((record.fetch("coveredScore").nil? && !overall.zero?) ||
+                          (record.fetch("coveredScore") && overall > record.fetch("coveredScore")))
+
+        raise Error, "score history contains an inconsistent overall score"
+      end
+
+      def fields(object, expected)
+        return if object.is_a?(Hash) && object.keys.sort == expected.sort
+
+        raise Error,
+              "score history contains an invalid record schema"
+      end
+
+      def sha?(value, length)
+        value.is_a?(String) && value.match?(/\A[0-9a-f]{#{length}}\z/)
+      end
+
+      def fraction(value)
+        return if value.nil? || (0..1).cover?(value)
+
+        raise Error,
+              "score history contains an invalid score"
+      end
+
+      def count(value)
+        raise Error, "score history contains an invalid count" unless value.is_a?(Integer) && value >= 0
+      end
+
+      def breakdown(value, counts, covered)
+        fields(value, BREAKDOWN)
+        %w[score assertionShare crashShare].each { |key| fraction(value.fetch(key)) }
+        (BREAKDOWN - %w[score assertionShare crashShare]).each { |key| count(value.fetch(key)) }
+        kills = value.values_at("assertionKills", "crashKills", "unclassifiedKills").sum
+        tested = counts.values_at("killed", "survived", "timeout").sum
+        unless kills == counts.fetch("killed") && value.fetch("tested") == tested &&
+               value.fetch("timeouts") == counts.fetch("timeout") &&
+               value.fetch("survivors") == counts.fetch("survived") &&
+               value.fetch("score") == covered
+          raise Error, "score history contains inconsistent counts"
+        end
+      end
+    end
+  end
+end

data/lib/saboteur/reporter/html.rb

@@ -57,7 +57,7 @@ module Saboteur
           <title>saboteur report</title>
           <style>#{STYLE}</style></head><body>
           <h1>saboteur report</h1>
-          #{summary(report, data)}#{warnings(data[:warnings])}#{skipped_subjects(data[:skippedSubjects])}
+          #{summary(report, data)}#{inline_suppressions(report)}#{warnings(data[:warnings])}#{skipped_subjects(data[:skippedSubjects])}
           <h2>Files</h2>
           #{data[:files].map { |file, entry| file_section(file, entry, names, results) }.join("\n")}
           </body></html>
@@ -66,6 +66,17 @@ module Saboteur
 
       private
 
+      def inline_suppressions(report)
+        return "" if report.inline_suppressions.empty?
+
+        items = report.inline_suppressions.map do |entry|
+          path = entry[:file].delete_prefix("#{report.root}/")
+          "<li>#{escape(path)}:#{entry[:line]} #{escape(entry[:subject])} " \
+            "#{escape(entry[:operator])} <code>#{escape(entry[:digest])}</code>: #{escape(entry[:reason])}</li>"
+        end
+        "<h2>Inline suppressions (#{items.size}, outside the score)</h2><ul>#{items.join}</ul>"
+      end
+
       def summary(report, data)
         seed = data.dig(:config, :seed)
         seed_note = seed ? ", seed #{seed}" : ""
@@ -76,6 +87,7 @@ module Saboteur
           <p class="note">Covered mutation score = #{SCORE_FORMULA}. NoCoverage, errors, resource limits and skipped
           mutants are not counted. Survivors accepted on the ignore list stay in the denominator. It covers only the subjects of this run, not the whole project. A tool that reads the
           Stryker JSON may show a different score, because the schema counts NoCoverage as undetected.</p>
+          #{score_split(report)}
           <p class="note">Wide tables scroll sideways.</p>
           <p>#{counts.map { |status, count| "#{status_label(status)} #{count}" }.join(' ')}</p>
           <p>Survivors: #{report.unignored_survivors.size} to fix, #{report.survivors.size - report.unignored_survivors.size}
@@ -83,6 +95,37 @@ module Saboteur
         HTML
       end
 
+      # How the score splits into assertion and crash kills, and the score
+      # without the whole-body mutants, which almost any test that runs the
+      # method kills. Each part shows only when it has something to say.
+      def score_split(report)
+        breakdown = report.score_breakdown
+        lines = []
+        lines << kill_split_line(breakdown) if breakdown.split_known?
+        lines << whole_body_line(report)
+        lines.compact.join("\n")
+      end
+
+      def kill_split_line(breakdown)
+        "<p>Of the tested mutants: #{parts_text(breakdown)}. A crash shows the code ran and broke, " \
+          "not that a test checked a value.</p>"
+      end
+
+      def parts_text(breakdown)
+        breakdown.parts.map { |label, share| "#{score_text(share)} #{label}" }.join(", ")
+      end
+
+      def whole_body_line(report)
+        whole = report.coarse_score_breakdown
+        fine = report.fine_score_breakdown
+        return if whole.tested.zero? || fine.tested.zero?
+
+        split = fine.split_known? ? " (#{parts_text(fine)})" : ""
+        "<p>Without whole-body mutants (body_nil, body_true, body_false, body_raise): #{score_text(fine.value)}" \
+          "#{split} over #{fine.tested} mutants. Whole-body mutants alone: #{score_text(whole.value)} " \
+          "over #{whole.tested}.</p>"
+      end
+
       def score_text(score)
         score ? format("%.1f%%", score * 100) : "n/a"
       end
@@ -126,13 +169,29 @@ module Saboteur
       def mutant_row(mutant, names, result)
         start = mutant[:location][:start]
         tests = Array(mutant[:killedBy]).map { |id| names.fetch(id) }.join(", ")
-        notes = [mutant[:statusReason], tests.empty? ? nil : "killed by #{tests}"].compact.join("; ")
+        notes = [mutant[:statusReason], tests.empty? ? nil : "killed by #{tests}",
+                 passing_tests_note(mutant, names, result)].compact.join("; ")
         "<tr><td class=\"fit\">line #{start[:line]}, column #{start[:column]}</td><td>#{row_status(mutant,
                                                                                                    result)}</td>" \
           "<td>#{escape(mutant[:mutatorName])}</td><td class=\"wide\">#{change(mutant, result)}</td>" \
           "<td class=\"fit\"><code>#{escape(mutant[:id])}</code></td><td class=\"wide\">#{escape(notes)}</td></tr>"
       end
 
+      # A survivor's covering tests all passed with the mutant applied, or, for
+      # a reused verdict, in the earlier run it came from.
+      def passing_tests_note(mutant, names, result)
+        return unless mutant[:status] == "Survived"
+
+        passed = Array(mutant[:coveredBy]).map { |id| names.fetch(id) }
+        return if passed.empty?
+
+        shown = passed.first(Terminal::MAX_PASSING_TESTS).join(", ")
+        more = passed.size - Terminal::MAX_PASSING_TESTS
+        extra = more.positive? ? " (+#{more} more)" : ""
+        when_passed = result.reused? ? "passed in an earlier run" : "passed"
+        "covered by #{passed.size} #{passed.one? ? 'test' : 'tests'} that #{when_passed}: #{shown}#{extra}"
+      end
+
       def row_status(mutant, result)
         badge = status_label(mutant[:status])
         # Only survivors are ever put on the ignore list.

data/lib/saboteur/reporter/mutation_text.rb

@@ -0,0 +1,36 @@
+# SPDX-License-Identifier: MIT
+# frozen_string_literal: true
+
+module Saboteur
+  module Reporter
+    # The original and replacement text of one mutant, and the line it
+    # changes, for compact one-line output.
+    module MutationText
+      module_function
+
+      def line(result)
+        result.subject.line_of(sorted_edits(result).first.start_byte)
+      end
+
+      def original(result)
+        join(result) { |edit| result.subject.file_source.byteslice(edit.start_byte, edit.end_byte - edit.start_byte) }
+      end
+
+      def replacement(result)
+        join(result, &:replacement)
+      end
+
+      def sorted_edits(result)
+        result.mutation.edits.sort_by(&:start_byte)
+      end
+
+      # Whole-body mutants carry a long text; one line keeps the output compact.
+      def join(result, &pick)
+        text = sorted_edits(result).map { |edit| pick.call(edit).to_s.dup.force_encoding(Encoding::UTF_8).scrub }
+                                   .join(" | ")
+        flat = text.gsub(/\s*\n\s*/, " ").strip
+        flat.length > 60 ? "#{flat[0, 57]}..." : flat
+      end
+    end
+  end
+end

data/lib/saboteur/reporter/next_steps.rb

@@ -0,0 +1,30 @@
+# SPDX-License-Identifier: MIT
+# frozen_string_literal: true
+
+module Saboteur
+  module Reporter
+    # The footer under a run that left survivors: how to look at one, how to
+    # accept one, and how to hand the run to an agent. Silent when nothing
+    # survived, so a clean run stays short.
+    class NextSteps
+      def initialize(out, palette)
+        @out = out
+        @palette = palette
+      end
+
+      def call(report)
+        survivor = report.unignored_survivors.first
+        return unless survivor
+
+        @out.puts "\n#{@palette.paint(:bold, 'Next steps:')}"
+        seed = " --seed #{report.seed}" if report.seed
+        @out.puts "  replay one survivor: saboteur run --id #{survivor.digest}#{seed}"
+        @out.puts "    (add this run's own --config, --since, --subject, --test and source options)"
+        @out.puts "  accept an equivalent one: add to your ignore file (c.ignore_file, default #{Config::IGNORE_FILE}):"
+        entry = %(i.mutation "#{survivor.digest}", subject: "#{survivor.subject.id}", reason: "why it is equivalent")
+        @out.puts "    #{entry}"
+        @out.puts "  let an agent close the gaps: saboteur run --agent, with the same options"
+      end
+    end
+  end
+end

data/lib/saboteur/reporter/palette.rb

@@ -0,0 +1,37 @@
+# SPDX-License-Identifier: MIT
+# frozen_string_literal: true
+
+module Saboteur
+  module Reporter
+    # Wraps text in ANSI colour codes when colour is on, and returns it
+    # unchanged when it is off. Colour is on only when asked for or when the
+    # output is a terminal and NO_COLOR is not set (https://no-color.org).
+    class Palette
+      CODES = { red: 31, green: 32, yellow: 33, magenta: 35, cyan: 36, dim: 2, bold: 1 }.freeze
+
+      # mode: :auto, :always or :never.
+      def self.for(out, mode: :auto, environment: ENV)
+        enabled = case mode
+                  when :always then true
+                  when :never then false
+                  else out.respond_to?(:tty?) && out.tty? && environment["NO_COLOR"].to_s.empty?
+                  end
+        new(enabled: enabled)
+      end
+
+      def initialize(enabled:)
+        @enabled = enabled
+      end
+
+      def enabled?
+        @enabled
+      end
+
+      def paint(color, text)
+        return text unless @enabled
+
+        "\e[#{CODES.fetch(color)}m#{text}\e[0m"
+      end
+    end
+  end
+end