saboteur 0.3.0 → 0.5.0

data/lib/saboteur/reporter/score_history.rb

@@ -0,0 +1,123 @@
+# SPDX-License-Identifier: MIT
+# frozen_string_literal: true
+
+require "json"
+require "time"
+require "digest"
+require "fileutils"
+require_relative "agent_incomplete"
+require_relative "history_record"
+
+module Saboteur
+  module Reporter
+    # Opt-in append-only score observations, not a baseline or a CI gate.
+    class ScoreHistory
+      DIFFERENT_RUN = "score history: change unavailable (different selection/inventory or incomplete run)"
+      MISSING_SCORE = "score history: change unavailable (no tested mutants)"
+      MAX_RECORD_BYTES = 64 * 1024
+      OPEN_FLAGS = File::RDWR | File::CREAT | File::NONBLOCK | File::NOFOLLOW
+
+      def initialize(path)
+        @path = path
+      end
+
+      def append(report, head:, scope:)
+        record = build(report, head, scope)
+        HistoryRecord.validate(record)
+        line = "#{JSON.generate(record)}\n"
+        raise Error, "score history record exceeds byte limit" if line.bytesize > MAX_RECORD_BYTES
+
+        FileUtils.mkdir_p(File.dirname(@path))
+        File.open(@path, OPEN_FLAGS, 0o600) do |file|
+          regular_file(file)
+          lock(file, File::LOCK_EX)
+
+          previous = previous_record(file)
+          file.seek(0, IO::SEEK_END)
+          file.write(line)
+          file.flush
+          change(previous, record)
+        end
+      rescue SystemCallError, IOError, JSON::ParserError, KeyError, TypeError => e
+        raise Error, "score history was not recorded: #{e.message}"
+      end
+
+      def validate_existing
+        return unless File.exist?(@path) || File.symlink?(@path)
+
+        File.open(@path, File::RDONLY | File::NONBLOCK | File::NOFOLLOW) do |file|
+          regular_file(file)
+          lock(file, File::LOCK_SH)
+
+          previous_record(file)
+        end
+      rescue SystemCallError, IOError => e
+        raise Error, "score history cannot be read: #{e.message}"
+      end
+
+      private
+
+      def build(report, head, scope)
+        { "version" => 1, "date" => Time.now.utc.iso8601, "commit" => head,
+          "selection" => Digest::SHA256.hexdigest(JSON.generate(scope.sort_by { |key, _| key.to_s }.to_h)),
+          "inventory" => Digest::SHA256.hexdigest(JSON.generate(report.results.map(&:digest).sort)),
+          "score" => overall_score(report), "coveredScore" => report.score,
+          "counts" => report.counts.transform_keys(&:to_s),
+          "breakdown" => report.score_breakdown.to_h.transform_keys(&:to_s),
+          "incomplete" => incompleteness(report) }
+      end
+
+      def overall_score(report)
+        score = report.score_breakdown
+        missing = report.results.count { |result| result.status == :skipped && result.reason == "no coverage" }
+        denominator = score.tested + missing
+        denominator.zero? ? nil : score.detected.fdiv(denominator)
+      end
+
+      def incompleteness(report)
+        reasons = Agent::Incomplete.reasons(report).map { |reason| reason[:reason] }
+        reasons << "baseline_incomplete" if report.run_evidence && !report.run_evidence["complete"]
+        reasons << "omissions" unless report.omissions.empty?
+        reasons.uniq
+      end
+
+      def lock(file, mode)
+        deadline = Process.clock_gettime(Process::CLOCK_MONOTONIC) + 2
+        until file.flock(mode | File::LOCK_NB)
+          if Process.clock_gettime(Process::CLOCK_MONOTONIC) >= deadline
+            raise Error,
+                  "score history lock was not acquired"
+          end
+
+          sleep 0.01
+        end
+      end
+
+      def regular_file(file)
+        raise Error, "score history must be a regular file" unless file.stat.file?
+      end
+
+      def previous_record(file)
+        while (line = file.gets(MAX_RECORD_BYTES + 1))
+          raise Error, "score history record exceeds byte limit" if line.bytesize > MAX_RECORD_BYTES
+          raise Error, "score history contains a truncated record" unless line.end_with?("\n")
+
+          previous = HistoryRecord.parse(line)
+        end
+        previous
+      end
+
+      def change(previous, current)
+        return "score history: first record" unless previous
+        return DIFFERENT_RUN unless
+          previous["selection"] == current["selection"] && previous["inventory"] == current["inventory"] &&
+          previous["incomplete"].empty? && current["incomplete"].empty?
+        return MISSING_SCORE if
+          previous["coveredScore"].nil? || current["coveredScore"].nil?
+
+        delta = (current["coveredScore"] - previous["coveredScore"]) * 100
+        format("score history: covered score change %+.2f percentage points (same configured selection)", delta)
+      end
+    end
+  end
+end

data/lib/saboteur/reporter/score_table.rb

@@ -0,0 +1,57 @@
+# SPDX-License-Identifier: MIT
+# frozen_string_literal: true
+
+require_relative "../score"
+
+module Saboteur
+  module Reporter
+    # One row per source file, worst score first, with survivor counts, and a
+    # one-line legend for the statuses in the summary.
+    class ScoreTable
+      MAX_ROWS = 15
+      LEGEND = ["killed: a test failed  survived: every selected test passed",
+                "timeout: exceeded its deadline",
+                "error, resource limit, skipped and the other statuses: not scored"].freeze
+
+      def initialize(out, palette)
+        @out = out
+        @palette = palette
+      end
+
+      def call(report)
+        rows = rows(report)
+        return if rows.size < 2
+
+        @out.puts "\nScore by file (worst first):"
+        width = rows.first(MAX_ROWS).map { |row| row[:path].length }.max
+        rows.first(MAX_ROWS).each { |row| @out.puts line(row, width) }
+        @out.puts "  ... #{rows.size - MAX_ROWS} more files" if rows.size > MAX_ROWS
+      end
+
+      def legend
+        LEGEND.each_with_index.map do |text, index|
+          @palette.paint(:dim, "#{index.zero? ? 'legend: ' : '        '}#{text}")
+        end.join("\n")
+      end
+
+      private
+
+      def rows(report)
+        by_file = report.results.group_by { |result| result.subject.file }
+        rows = by_file.map { |file, results| row(report, file, results) }
+        rows.reject { |row| row[:tested].zero? }.sort_by { |row| [row[:value], -row[:survivors], row[:path]] }
+      end
+
+      def row(report, file, results)
+        breakdown = Score.of(results)
+        { path: file.delete_prefix("#{report.root}/"), value: breakdown.value,
+          survivors: results.count(&:unignored_survivor?), tested: breakdown.tested }
+      end
+
+      def line(row, width)
+        score = format("%5.1f %%", row[:value] * 100)
+        "  #{row[:path].ljust(width)}  #{score}  #{row[:survivors]} survivor#{'s' unless row[:survivors] == 1}"
+      end
+    end
+  end
+end

data/lib/saboteur/reporter/stryker.rb

@@ -16,14 +16,14 @@ module Saboteur
         @path = path
       end
 
-      def call(report)
+      def call(report, exit_cause: nil)
         FileUtils.mkdir_p(File.dirname(@path))
-        File.write(@path, JSON.generate(self.class.build(report)))
+        File.write(@path, JSON.generate(self.class.build(report, exit_cause: exit_cause)))
       end
 
       # Tests are listed once under testFiles with short ids; mutants refer to
       # them by id in killedBy/coveredBy instead of repeating full test names.
-      def self.build(report)
+      def self.build(report, exit_cause: nil)
         test_ids = test_index(report.results)
         files = report.results.group_by { |result| result.subject.file }.to_h do |file, results|
           [relative(file, report.root), file_entry(results, test_ids)]
@@ -35,13 +35,30 @@ module Saboteur
           files: files,
           testFiles: test_files(test_ids),
           skippedSubjects: skipped_subjects(report),
+          earlyStop: early_stop(report),
           rejectedKills: report.rejected_kills,
           warnings: report.warnings,
           framework: { name: "saboteur", version: VERSION },
-          saboteur: { findings: Findings.build(report) }
+          saboteur: { findings: Findings.build(report), scores: scores(report),
+                      inlineSuppressions: report.inline_suppressions.map do |entry|
+                        entry.merge(file: relative(entry[:file], report.root))
+                      end }
+                     .merge(exit_cause ? { exitCause: exit_cause } : {})
         }.merge(report.seed ? { config: { seed: report.seed } } : {})
       end
 
+      # Score with its assertion and crash shares, for all mutants and without
+      # the whole-body ones (body_nil, body_true, body_false, body_raise).
+      def self.scores(report)
+        { all: report.score_breakdown.to_h, withoutWholeBodyMutants: report.fine_score_breakdown.to_h }
+      end
+
+      # nil for a run that covered its whole selection.
+      def self.early_stop(report)
+        stop = report.early_stop
+        stop && { reason: stop.reason, unreachedSubjects: stop.unreached }
+      end
+
       def self.skipped_subjects(report)
         report.skipped_subjects.map do |subject, reason|
           { subject: subject.id, file: relative(subject.file, report.root), line: subject.start_line, reason: reason }
@@ -94,7 +111,10 @@ module Saboteur
         when :survived then "Survived"
         when :timeout then "Timeout"
         when :error, :resource_limit then "RuntimeError"
-        when :skipped then result.reason == "no coverage" ? "NoCoverage" : "Ignored"
+        when :skipped then result.reason&.start_with?("no coverage") ? "NoCoverage" : "Ignored"
+        # The schema has no status for "ran, but no test can observe a
+        # change here". Ignored with the reason is the honest mapping.
+        when :ran_at_load, :unreachable then "Ignored"
         end
       end
 

data/lib/saboteur/reporter/survivor_hints.rb

@@ -0,0 +1,68 @@
+# SPDX-License-Identifier: MIT
+# frozen_string_literal: true
+
+module Saboteur
+  module Reporter
+    # Suggests a check that might kill a surviving mutant, by operator. It does
+    # not say what is missing: the survivor can also be an equivalent mutant.
+    module SurvivorHints
+      BY_OPERATOR = {
+        "comparison" => "try an example at the boundary: equal values, or values one apart",
+        "arithmetic" => "try asserting the exact computed number",
+        "boolean_logic" => "try an example where each side of the condition alone decides the result",
+        "negation_removal" => "try an example where the negated value changes the result",
+        "literal_number" => "try asserting something that depends on this number",
+        "literal_boolean" => "try an example that depends on this true/false value",
+        "literal_string" => "try asserting on this text",
+        "statement_removal" => "try asserting the effect of this statement",
+        "or_assign" => "try an example where the variable is already set (||= vs =)",
+        "safe_navigation" => "try an example where the receiver is nil",
+        "argument_removal" => "try have_received(...).with(...), or assert a result that depends on this argument",
+        "pair_removal" => "try have_received(...).with(...), or assert the hash content under this key",
+        "endpoint_replacement" => "try a collection with 2 or more elements, to tell first from last",
+        "filter_replacement" => "try items that match and items that do not, to tell select from reject"
+      }.freeze
+
+      COARSE = "try asserting this method's result or its effect"
+
+      # Printed once after the survivors: a hint is a place to look, not a diagnosis.
+      CAVEAT = "Hints are suggestions, not diagnoses: a survivor can also be an equivalent mutant, or a change " \
+               "the selected tests cannot observe."
+
+      # Comparison operator swaps where the two operators give the same answer
+      # except in one named case, so a test that hits that case can tell them
+      # apart. Anything else gets the plain text hint.
+      EQUAL_CASE = [%w[< <=], %w[> >=]].map(&:to_set).freeze
+      UNEQUAL_CASE = [%w[< >], %w[<= >=]].map(&:to_set).freeze
+
+      module_function
+
+      def for(operator, coarse: false)
+        BY_OPERATOR.fetch(operator) { coarse ? COARSE : nil }
+      end
+
+      # The hint as data for `--agent`: { kind:, text: } and, for a comparison
+      # swap, case: the relation between the two operands that separates the
+      # original from the mutant ("left == right"). It is where to look, not a
+      # promise that a test there will kill the mutant. nil when there is none.
+      def structured(operator:, original:, replacement:, coarse: false)
+        text = self.for(operator, coarse: coarse)
+        return unless text
+
+        operands = boundary_case(operator, original, replacement)
+        return { kind: "boundary", case: operands, text: text } if operands
+
+        { kind: coarse ? "method_result" : "assertion", text: text }
+      end
+
+      def boundary_case(operator, original, replacement)
+        return unless operator == "comparison"
+
+        pair = Set[original, replacement]
+        return "left == right" if EQUAL_CASE.include?(pair)
+
+        "left != right" if UNEQUAL_CASE.include?(pair)
+      end
+    end
+  end
+end

data/lib/saboteur/reporter/survivor_list.rb

@@ -0,0 +1,126 @@
+# SPDX-License-Identifier: MIT
+# frozen_string_literal: true
+
+require_relative "mutation_text"
+require_relative "survivor_hints"
+
+module Saboteur
+  module Reporter
+    # Survivors grouped by file, then method, then line. The source line and
+    # the tests that covered it are printed once; each mutation is one compact
+    # line: operator, original -> replacement, digest.
+    class SurvivorList
+      MAX_PASSING_TESTS = 5
+
+      def initialize(out, palette)
+        @out = out
+        @palette = palette
+      end
+
+      # Every survivor, in report order; the grouping keeps them readable.
+      def call(report, survivors)
+        return if survivors.empty?
+
+        @out.puts "Survivors (#{survivors.size}):"
+        survivors.group_by { |result| result.subject.file }.sort_by { |file, _| file }.each do |file, in_file|
+          file_section(report, relative(file, report.root), in_file)
+        end
+        @out.puts "\n#{@palette.paint(:dim, SurvivorHints::CAVEAT)}" if survivors.any? { |result| hint_for(result) }
+      end
+
+      private
+
+      def file_section(report, path, in_file)
+        @out.puts "\n#{@palette.paint(:bold, path)}"
+        in_file.group_by { |result| result.subject.id }.each do |subject_id, in_subject|
+          @out.puts "  #{subject_id}"
+          lines(report, path, in_subject)
+        end
+      end
+
+      def lines(report, path, in_subject)
+        in_subject.group_by { |result| MutationText.line(result) }.sort.each do |line, on_line|
+          first = on_line.first
+          @out.puts "    #{path}:#{line} in #{first.subject.id}"
+          @out.puts "      #{@palette.paint(:dim, "#{line}:")} #{source_line(first, line)}"
+          evidence_groups(report, on_line).each { |group| evidence_group(report, path, group) }
+        end
+      end
+
+      # Mutants on one line share the printed evidence only when it is the
+      # same for all of them: covering tests (in order), reuse and changed
+      # lines can each differ per mutant, so each run of neighbours with the
+      # same tuple is one group and shares evidence above its own mutations.
+      # Evidence that repeats after another group is printed again, which
+      # keeps the ranked order.
+      def evidence_groups(report, on_line)
+        on_line.chunk { |result| [result.covered_by, result.reused?, report.changed_lines_for(result)] }
+               .map(&:last)
+      end
+
+      def evidence_group(report, path, group)
+        first = group.first
+        changed_lines(report, path, first)
+        passing_tests(first)
+        group.each { |result| mutation(result) }
+      end
+
+      def source_line(result, line)
+        result.subject.file_source.lines[line - 1].to_s.strip
+      end
+
+      def mutation(result)
+        text = "      #{@palette.paint(:red, '-')} #{result.mutation.operator} " \
+               "#{MutationText.original(result)} -> #{@palette.paint(:green, MutationText.replacement(result))} " \
+               "#{result.digest}"
+        text += " (ignored: #{result.ignored})" if result.ignored
+        @out.puts text
+        hint(result)
+      end
+
+      def hint(result)
+        text = hint_for(result)
+        @out.puts "        #{@palette.paint(:cyan, 'hint:')} #{text}" if text
+      end
+
+      def hint_for(result)
+        return nil if result.ignored
+
+        SurvivorHints.for(result.mutation.operator, coarse: result.mutation.coarse)
+      end
+
+      # The tests that ran with the mutant applied and still passed: the
+      # missing assertion belongs in one of them. They are listed likeliest
+      # killers first, the order the run used. Line coverage can overstate which
+      # tests exercise the line, so the list is a lead, not proof.
+      def passing_tests(result)
+        tests = result.covered_by
+        return if tests.empty?
+
+        @out.puts "      #{tests.size} #{tests.one? ? 'test' : 'tests'} covered this line and " \
+                  "#{passed_phrase(result)} (the missing check may belong in one of them):"
+        tests.first(MAX_PASSING_TESTS).each { |name| @out.puts "        #{name}" }
+        more = tests.size - MAX_PASSING_TESTS
+        @out.puts "        ... #{more} more in the JSON report (coveredBy)" if more.positive?
+      end
+
+      def passed_phrase(result)
+        result.reused? ? "passed in an earlier run (--incremental)" : "still passed"
+      end
+
+      # In diff mode: the changed lines this mutant touches.
+      def changed_lines(report, path, result)
+        lines = report.changed_lines_for(result)
+        return if lines.empty?
+
+        shown = lines.first(3).map { |line| "#{path}:#{line}" }.join(", ")
+        more = lines.size - 3
+        @out.puts "      changed #{lines.one? ? 'line' : 'lines'}: #{shown}#{" (+#{more} more)" if more.positive?}"
+      end
+
+      def relative(file, root)
+        file.delete_prefix("#{root}/")
+      end
+    end
+  end
+end

data/lib/saboteur/reporter/survivor_ranking.rb

@@ -0,0 +1,34 @@
+# SPDX-License-Identifier: MIT
+# frozen_string_literal: true
+
+module Saboteur
+  module Reporter
+    # Review priority, not a claim that a survivor is a real defect.
+    module SurvivorRanking
+      ORDERING = "coarse first, then more covering tests, larger edited-line span, " \
+                 "nearer changed lines, then file, line and digest"
+
+      def self.ordered(report, results)
+        results.sort_by { |result| key(report, result) }
+      end
+
+      def self.key(report, result)
+        [result.mutation.coarse ? 0 : 1, -result.covered_by.uniq.size, -edited_lines(result).size,
+         report.changed_line_distance(result) || Float::INFINITY,
+         result.subject.file.delete_prefix("#{report.root}/"), first_line(result), result.digest]
+      end
+
+      def self.first_line(result)
+        result.subject.line_of(result.mutation.edits.map(&:start_byte).min)
+      end
+
+      def self.edited_lines(result)
+        result.mutation.edits.flat_map do |edit|
+          first = result.subject.line_of(edit.start_byte)
+          last = result.subject.line_of([edit.end_byte - 1, edit.start_byte].max)
+          (first..last).to_a
+        end.uniq
+      end
+    end
+  end
+end