saboteur 0.7.0 → 0.8.0

data/lib/saboteur/cli/exit_status.rb

@@ -28,6 +28,8 @@ module Saboteur
 
         return EXIT_TOOL if untested_partial?(report)
 
+        return EXIT_TOOL if unverifiable_scope?(report)
+
         return EXIT_TOOL if mostly_errored?(report)
 
         return EXIT_TOOL if incomplete_under_threshold?(report)
@@ -53,6 +55,32 @@ module Saboteur
         true
       end
 
+      # A run that verified none of the change under review - every
+      # subject skipped (refinements, no coverage, ..., #574) or every
+      # mutant unreachable (a suite re-loading mutated files, #597) -
+      # must not exit 0 with score n/a and green-light an untested change.
+      def unverifiable_scope?(report)
+        if unverifiable_review?(report)
+          report.skipped_subjects.each do |subject, reason|
+            @err.puts("saboteur: #{subject.id} was skipped (#{reason}), so the change under test was not verified")
+          end
+          return true
+        end
+        return false unless unverifiable_unreachable?(report)
+
+        @err.puts("saboteur: all mutants ran unreachable, so the change under test was not verified")
+        true
+      end
+
+      def unverifiable_review?(report)
+        report.review? && report.subjects.any? &&
+          report.subjects.all? { |subject| report.skipped_subjects.key?(subject) }
+      end
+
+      def unverifiable_unreachable?(report)
+        report.executed.any? && report.executed.all? { |result| result.status == :unreachable }
+      end
+
       def gates = (@gates ||= Gates.new(config))
 
       def mostly_errored?(report)
@@ -89,8 +117,11 @@ module Saboteur
         else
           if report.majority_errored?
             "errorMajority"
+          elsif unverifiable_review?(report) || unverifiable_unreachable?(report) ||
+                (report.early_stop && report.score_breakdown.tested.zero?)
+            "incomplete"
           else
-            report.early_stop && report.score_breakdown.tested.zero? ? "incomplete" : "incompleteUnderThreshold"
+            "incompleteUnderThreshold"
           end
         end
       end

data/lib/saboteur/cli/help_options.rb

@@ -9,7 +9,8 @@ module Saboteur
       def option_lines
         run = parser.summarize
         { "run" => run, "pr" => run, "audit" => run, "doctor" => doctor_parser.summarize,
-          "baseline" => baseline_parser.summarize }
+          "baseline" => baseline_parser.summarize, "badge" => badge_parser.summarize,
+          "trend" => trend_parser.summarize }
       end
     end
   end

data/lib/saboteur/cli/history_command.rb

@@ -0,0 +1,135 @@
+# SPDX-License-Identifier: MIT
+# frozen_string_literal: true
+
+require_relative "../reporter/history_reader"
+
+module Saboteur
+  class CLI
+    # `saboteur badge` and `saboteur trend` (#474): a local README badge and
+    # a trend read-out over the opt-in score history, no hosted service.
+    module HistoryCommand
+      DEFAULT_HIGH = 80.0
+      DEFAULT_LOW = 60.0
+
+      def badge_parser
+        OptionParser.new do |opts|
+          opts.banner = "Usage: saboteur badge [--history PATH] [--output PATH] [--mark-partial] " \
+                        "[--high PERCENT] [--low PERCENT]"
+          opts.on("--history PATH", "Score history JSONL (default: c.score_history)") { |v| @history_path = v }
+          opts.on("--output PATH", "Badge JSON destination (default: stdout)") { |v| @output = v }
+          opts.on("--mark-partial", "Badge an incomplete run, labeled as partial") { @mark_partial = true }
+          opts.on("--high PERCENT", Float, "brightgreen at or above (default #{DEFAULT_HIGH.to_i})") { |v| @high = v }
+          opts.on("--low PERCENT", Float, "yellow at or above (default #{DEFAULT_LOW.to_i})") { |v| @low = v }
+        end
+      end
+
+      def trend_parser
+        OptionParser.new do |opts|
+          opts.banner = "Usage: saboteur trend [--history PATH]"
+          opts.on("--history PATH", "Score history JSONL (default: c.score_history)") { |v| @history_path = v }
+        end
+      end
+
+      private
+
+      def history_command(command) = command == "badge" ? badge : trend
+
+      def badge
+        badge_parser.parse!(@argv)
+        raise UsageError, "unexpected arguments: #{@argv.join(' ')}" if @argv.any?
+
+        write_badge(badge_payload(read_history.last))
+        EXIT_OK
+      rescue ConfigError, Error, SystemCallError => e
+        history_failure(e)
+      end
+
+      def trend
+        trend_parser.parse!(@argv)
+        raise UsageError, "unexpected arguments: #{@argv.join(' ')}" if @argv.any?
+
+        print_trend(latest_pair)
+        EXIT_OK
+      rescue ConfigError, Error, SystemCallError => e
+        history_failure(e)
+      end
+
+      def history_failure(error)
+        @err.puts("saboteur: #{error.message}")
+        error.is_a?(ConfigError) ? EXIT_USAGE : EXIT_TOOL
+      end
+
+      def latest_pair
+        records = read_history
+        current = records.last
+        [records[0...-1].reverse.find { |record| Reporter::HistoryReader.comparable?(record, current) }, current]
+      end
+
+      def read_history
+        records = Reporter::HistoryReader.read(history_path)
+        if records.empty?
+          raise Error,
+                "score history is empty - record it first by running saboteur with c.score_history set"
+        end
+
+        records
+      end
+
+      def history_path
+        @history_path ||= config.path("score_history")
+        @history_path or raise UsageError, "no score history: pass --history PATH or set c.score_history"
+      rescue ConfigError
+        raise UsageError, "no score history: pass --history PATH or set c.score_history"
+      end
+
+      def badge_payload(record)
+        partial = record["incomplete"].any?
+        if partial && !@mark_partial
+          raise Error, "the latest score history record is an incomplete run " \
+                       "(#{record['incomplete'].map(&:inspect).join(', ')}) - " \
+                       "badge evidence needs a complete run, or pass --mark-partial to label it"
+        end
+        score = record["coveredScore"]
+        raise Error, "the latest score history record has no tested mutants" if score.nil?
+
+        { schemaVersion: 1, label: partial ? "mutation score (partial)" : "mutation score",
+          message: format("%.1f%%", score * 100), color: badge_color(score) }
+      end
+
+      def write_badge(payload)
+        json = "#{JSON.generate(payload)}\n"
+        return @out.print(json) unless @output
+
+        File.write(@output, json)
+        @out.puts(@output)
+      end
+
+      def badge_color(score)
+        percent = score * 100
+        return "brightgreen" if percent >= (@high || DEFAULT_HIGH)
+        return "yellow" if percent >= (@low || DEFAULT_LOW)
+
+        "red"
+      end
+
+      def print_trend((previous, current))
+        ([previous].compact + [current]).each { |record| @out.puts(trend_line(record)) }
+        unless previous && previous["coveredScore"] && current["coveredScore"]
+          return @out.puts("score history: no comparable earlier record (different selection/inventory, " \
+                           "operator policy or an incomplete run)")
+        end
+
+        delta = (current["coveredScore"] - previous["coveredScore"]) * 100
+        @out.puts(format("covered score change %<delta>+.2f percentage points since %<since>s " \
+                         "(same configured selection)", delta: delta, since: previous["date"]))
+      end
+
+      def trend_line(record)
+        suffix = record["incomplete"].empty? ? "" : " (incomplete)"
+        score = record["coveredScore"]
+        "#{record['date']} #{record['commit']&.slice(0, 8) || '-'} " \
+          "#{score.nil? ? 'no tested mutants' : format('%.1f%%', score * 100)}#{suffix}"
+      end
+    end
+  end
+end

data/lib/saboteur/cli/presets.rb

@@ -4,18 +4,22 @@
 module Saboteur
   class CLI
     # Opinionated bundles (#244): pr is the fast changed-lines loop (one
-    # baseline check, a 10-minute budget that does not fail the build),
+    # baseline check, a 10-minute budget that fails an incomplete run),
     # audit is the whole project with both reports. Preset flags go first,
     # so an explicit flag after them wins on the usual last-flag rule.
     module Presets
       PRESETS = {
-        "pr" => %w[--baseline-passes 1 --max-duration 10m --max-duration-ok],
+        "pr" => %w[--baseline-passes 1 --max-duration 10m],
         "audit" => %w[--all --operators all --baseline-passes 2 --json .saboteur/audit-report.json
                       --html .saboteur/audit-report.html]
       }.freeze
 
       private
 
+      def preset_or_usage_error(command)
+        PRESETS.key?(command) ? preset(command) : usage_error
+      end
+
       def preset(name)
         flags = PRESETS.fetch(name).dup
         if name == "pr"

data/lib/saboteur/cli/selection_options.rb

@@ -0,0 +1,31 @@
+# SPDX-License-Identifier: MIT
+# frozen_string_literal: true
+
+module Saboteur
+  class CLI
+    # The run's subject-selection flags: operator sets, single-mutant replay,
+    # --since scoping, --dry-run, the flaky-check confirmation rerun (#476)
+    # and explicit --subject filters. Grouped so the CLI stays under the
+    # house size caps.
+    module SelectionOptions
+      private
+
+      def define_selection_options(opts)
+        opts.on("--operators NAMES", Array, "Choose operator names (default: all)") do |v|
+          set(:operators, v)
+        end
+        opts.on("--exclude-operators NAMES", Array, "Exclude operator names from the selected set") do |v|
+          set(:exclude_operators, v)
+        end
+        opts.on("--list-operators", "List operator names and descriptions") { set(:list_operators, true) }
+        opts.on("--dry-run", "Preview mutants without boot, tests, forks or report writes") { set(:dry_run, true) }
+        opts.on("--id DIGEST", "Replay exactly one current mutant by its full digest") { |v| set(:id, v) }
+        opts.on("--shard K/N", "Run one of N deterministic candidate shards (for example 1/4)") { |v| set(:shard, v) }
+        opts.on("--since REF", "Only mutate subjects changed since git REF") { |v| options[:since] = v }
+        flaky_desc = "Confirm a fresh verdict (survivors, decisive) with one reordered rerun"
+        opts.on("--flaky-check MODE", %w[off survivors decisive], flaky_desc) { |v| set(:flaky_check, v) }
+        opts.on("--subject EXPR", "Klass#meth, Klass.meth or a namespace; repeatable") { |v| add_subject(v) }
+      end
+    end
+  end
+end

data/lib/saboteur/cli/shard_options.rb

@@ -0,0 +1,17 @@
+# SPDX-License-Identifier: MIT
+# frozen_string_literal: true
+
+module Saboteur
+  class CLI
+    # Validates the deterministic candidate-shard selection before the target boots.
+    module ShardOptions
+      private
+
+      def validate_shard_options
+        raise UsageError, "--id cannot be combined with --shard" if options[:id] && config["shard"]
+
+        Engine::Shard.parse(config["shard"]) if config["shard"]
+      end
+    end
+  end
+end

data/lib/saboteur/cli/source_set.rb

@@ -0,0 +1,53 @@
+# SPDX-License-Identifier: MIT
+# frozen_string_literal: true
+
+module Saboteur
+  class CLI
+    # The effective source set a run will mutate, refused when it is empty.
+    module SourceSet
+      private
+
+      # The Rails-shaped footgun (#447): the default include targets plain
+      # gems, so a first run on an app/ tree selects nothing and would exit 0
+      # over 0 subjects - a green gate that mutates nothing, every CI run.
+      # Refuse like a typo'd source: exit 2, name the stale pattern and the
+      # way out. --dry-run still previews: it gates nothing.
+      def refuse_empty_source_set
+        return unless config.included_files.empty?
+
+        raise UsageError, empty_source_set_message
+      end
+
+      # A --since ref whose resolved range diffs empty yields 0 subjects
+      # and a green run over nothing: `saboteur pr <merge_sha>` on a
+      # checkout AT the merge diffs the merge against itself (#643).
+      # Refuse like a typo'd ref: exit 2, name the base and the range.
+      def refuse_empty_since_range
+        ref = config["since"]
+        return unless ref
+
+        paths = Selection::GitDiff.changed_paths(ref, root: config.root)
+        return unless paths.values.all?(&:empty?)
+
+        base = Selection::GitDiff.merge_base(ref, root: config.root)
+        raise UsageError, "--since #{ref}: the resolved range #{base}..the working tree has no changes - " \
+                          "the run would pass with 0 subjects. A checkout AT the ref (the CI merge-sha " \
+                          "checkout) diffs it against itself: fetch the merge's parent, or point --since " \
+                          "at the base of the change under test"
+      end
+
+      def empty_source_set_message
+        include_patterns = config["include"]
+        matched = include_patterns.flat_map { |pattern| Dir.glob(pattern, base: config.root) }
+                                  .select { |file| file.end_with?(".rb") }
+        cause = if matched.empty?
+                  "include: #{include_patterns.join(', ')} selects no Ruby file"
+                else
+                  "exclude: #{config['exclude'].join(', ')} removes every included file"
+                end
+        "#{cause} - the run would pass with 0 subjects. Point include at your code " \
+          "(Rails apps: bundle exec rails generate saboteur:install writes the config) or fix the patterns"
+      end
+    end
+  end
+end

data/lib/saboteur/cli/survivors_command.rb

@@ -0,0 +1,157 @@
+# SPDX-License-Identifier: MIT
+# frozen_string_literal: true
+
+require "json"
+
+module Saboteur
+  class CLI
+    # saboteur survivors <report.json> (#475): a read-only, line-oriented
+    # terminal browser for the Stryker JSON report. It never boots the
+    # project or reruns tests; it lists survivors, filters them, and for
+    # one digest prints the diff, the exact replay command and the
+    # ignore-file template. It never edits the ignore file.
+    module SurvivorsCommand
+      private
+
+      def survivors_command
+        survivors_command_inner
+      rescue UsageError => e
+        @err.puts("saboteur: #{e.message}")
+        EXIT_USAGE
+      end
+
+      def survivors_command_inner
+        parser = OptionParser.new do |opts|
+          opts.on("--ranked", "Review-priority order: the report's ranked findings first")
+          opts.on("--file SUBSTR", "Only survivors whose path contains SUBSTR")
+          opts.on("--operator NAME", "Only survivors of operator NAME")
+          opts.on("--triage LABEL", "Only survivors classified LABEL (for example missing_test)")
+          opts.on("--show DIGEST", "Diff, replay command and ignore template for one survivor")
+        end
+        options = {}
+        args = parser.parse!(@argv, into: options)
+        path = args.shift or raise UsageError,
+                                   "survivors expects a report path: saboteur survivors .saboteur/report.json"
+        raise UsageError, "unexpected arguments: #{args.join(' ')}" if args.any?
+
+        report = load_survivor_report(path)
+        browser = SurvivorsBrowser.new(report)
+        if options[:show]
+          @out.puts(browser.show(options[:show]))
+        else
+          @out.puts(browser.list(**options.slice(:ranked, :file, :operator, :triage)))
+        end
+        EXIT_OK
+      end
+
+      def load_survivor_report(path)
+        raise UsageError, "report not found: #{path}" unless File.file?(path)
+
+        JSON.parse(File.read(path))
+      rescue JSON::ParserError => e
+        raise UsageError, "#{path} is not a readable saboteur report: #{e.message}"
+      end
+    end
+
+    # Reads the report's files section (the complete mutant list), not the
+    # capped findings, so survivors beyond the findings limit still list.
+    class SurvivorsBrowser
+      def initialize(report)
+        @report = report
+        @findings = report.dig("saboteur", "findings") || {}
+      end
+
+      def list(ranked: false, file: nil, operator: nil, triage: nil)
+        survivors = filtered(file, operator, triage)
+        survivors = ranked_order(survivors) if ranked
+        return "no survivors#{filters_note(file, operator, triage)}" if survivors.empty?
+
+        [header(survivors.size, file, operator, triage), *rows(survivors)]
+      end
+
+      def show(digest)
+        survivor = all_survivors.find { |s| s[:digest].start_with?(digest) } ||
+                   raise(UsageError, "no survivor with digest #{digest} in this report")
+        finding = findings_by_digest[survivor[:digest]] || {}
+        subject = finding["subject"]
+        [
+          "#{survivor[:path]}:#{survivor[:line]}  #{survivor[:operator]}  #{survivor[:digest]}",
+          "",
+          "- #{finding['original'] || original_line(survivor)}",
+          "+ #{finding['replacement'] || survivor[:replacement]}",
+          "",
+          "Replay exactly this mutant:",
+          "  bundle exec saboteur run --id #{survivor[:digest]}",
+          "",
+          "Ignore-file entry (config/saboteur_ignore.rb):",
+          *ignore_entry(survivor, subject)
+        ].join("\n")
+      end
+
+      private
+
+      def filtered(file, operator, triage)
+        survivors = all_survivors
+        survivors = survivors.select { |s| s[:path].include?(file) } if file
+        survivors = survivors.select { |s| s[:operator] == operator } if operator
+        triage ? survivors.select { |s| s[:classification] == triage } : survivors
+      end
+
+      def header(size, file, operator, triage)
+        text = "#{size} survivor#{'s' unless size == 1}#{filters_note(file, operator, triage)}"
+        truncated = @findings["survivorsTruncated"].to_i
+        return text unless truncated.positive?
+
+        "#{text} (the report's ranked findings name only the first #{@findings['limit'] || 10})"
+      end
+
+      def rows(survivors)
+        survivors.each_with_index.map do |s, i|
+          format("%3<i>d. %<path>s:%<line>d  %-28<operator>s %<digest>s  %<classification>s",
+                 i: i + 1, path: s[:path], line: s[:line], operator: s[:operator], digest: s[:digest],
+                 classification: s[:classification] || "-")
+        end
+      end
+
+      def ignore_entry(survivor, subject)
+        note = subject ? subject.inspect : '"Klass#meth" # TODO: capped findings omit this digest - fill it in'
+        [%(  i.mutation "#{survivor[:digest]}",), "      subject: #{note},", %(      reason: "...")]
+      end
+
+      def all_survivors
+        @all_survivors ||= begin
+          survivors = @report.fetch("files", {}).flat_map do |path, entry|
+            entry.fetch("mutants", []).select { |m| m["status"] == "Survived" }.map do |m|
+              { path: path, line: m.dig("location", "start", "line").to_i, operator: m["mutatorName"],
+                digest: m["id"], replacement: m["replacement"], classification: m["classification"],
+                covered: m["coveredBy"] || [], source: entry["source"] }
+            end
+          end
+          survivors.sort_by { |s| [s[:path], s[:line], s[:digest]] }
+        end
+      end
+
+      def ranked_order(survivors)
+        rank = @findings.fetch("survivors", []).each_with_index.to_h { |f, i| [f["digest"], i] }
+        survivors.sort_by { |s| [rank[s[:digest]] || Float::INFINITY, -s[:covered].size, s[:path], s[:line], s[:digest]] }
+      end
+
+      def findings_by_digest
+        @findings_by_digest ||=
+          (@findings.fetch("survivors", []) + @findings.fetch("accepted", [])).to_h { |f| [f["digest"], f] }
+      end
+
+      def original_line(survivor)
+        survivor[:source].to_s.lines[survivor[:line] - 1].to_s.strip
+      end
+
+      def filters_note(file, operator, triage)
+        parts = []
+        parts << "file~#{file}" if file
+        parts << "operator=#{operator}" if operator
+        parts << "triage=#{triage}" if triage
+        parts.empty? ? "" : " (#{parts.join(', ')})"
+      end
+    end
+  end
+end