saboteur 0.3.0 → 0.4.0

data/lib/saboteur/install.rb

@@ -0,0 +1,132 @@
+# SPDX-License-Identifier: MIT
+# frozen_string_literal: true
+
+require "fileutils"
+
+module Saboteur
+  # Writes the Rails starter files: config/saboteur.rb, bin/saboteur and a
+  # .gitignore entry. Plain Ruby so it runs and tests without Rails; the
+  # `rails generate saboteur:install` generator is a thin wrapper over it.
+  # Existing files are never overwritten unless force is set.
+  class Install
+    CONFIG = <<~RUBY
+      # saboteur configuration for a Rails + RSpec app. CLI flags override this file.
+      Saboteur.configure do |c|
+        c.include = ["app/**/*.rb"]
+        # Empty base classes: nothing useful to mutate.
+        c.exclude = [
+          "app/models/application_record.rb",
+          "app/controllers/application_controller.rb",
+          "app/jobs/application_job.rb",
+          "app/mailers/application_mailer.rb",
+        ]
+        c.require = ["./config/environment"] # boots the app before specs load
+        c.rails = true                       # reconnect ActiveRecord after fork, per-worker databases
+        c.jobs = 4                           # parallel workers; use 1 if your specs share external state
+      end
+    RUBY
+
+    BIN = <<~RUBY
+      #!/usr/bin/env ruby
+      # frozen_string_literal: true
+
+      ENV["RAILS_ENV"] ||= "test"
+      require "bundler/setup"
+      load Gem.bin_path("saboteur", "saboteur")
+    RUBY
+
+    IGNORE_ENTRY = ".saboteur/"
+
+    GEMFILE_HINT = <<~TEXT
+      Add saboteur to your Gemfile if it is not there yet:
+
+        group :test do
+          gem "saboteur", require: false
+        end
+
+      Then run it with SimpleCov off for the run, for example:
+
+        bin/saboteur run --since origin/main
+    TEXT
+
+    Result = Data.define(:status, :path)
+
+    def initialize(root: Dir.pwd, force: false)
+      @root = File.expand_path(root)
+      @force = force
+    end
+
+    # Returns the list of Results (:created, :overwritten, :skipped, :appended, :unchanged).
+    def call
+      [write("config/saboteur.rb", CONFIG), write("bin/saboteur", BIN, mode: 0o755), gitignore]
+    end
+
+    # Printed after install; the Gemfile is never edited.
+    def gemfile_hint
+      gemfile = File.join(@root, "Gemfile")
+      return nil if File.file?(gemfile) && File.read(gemfile).match?(/^\s*gem\s+["']saboteur["']/)
+
+      GEMFILE_HINT
+    end
+
+    private
+
+    # Never writes or chmods through a symlink: not the file itself, and not a
+    # symlinked directory above it (bin -> elsewhere). Without force the file is
+    # skipped; with force the link is removed and replaced by a real file or
+    # directory inside the project, and the old target is left alone.
+    def write(relative, content, mode: nil)
+      path = File.join(@root, relative)
+      link = symlink_component(relative)
+      existed = present?(path)
+      return Result.new(:skipped, relative) if (existed || link) && !@force
+
+      File.delete(link) if link
+      FileUtils.mkdir_p(File.dirname(path))
+      File.write(path, content)
+      File.chmod(mode, path) if mode
+      Result.new(existed ? :overwritten : :created, relative)
+    end
+
+    # The first symlink on the way from the project root to the file, if any.
+    def symlink_component(relative)
+      current = @root
+      relative.split("/").each do |part|
+        current = File.join(current, part)
+        return current if File.symlink?(current)
+      end
+      nil
+    end
+
+    # A dangling symlink does not "exist" but is not free to write either.
+    def present?(path)
+      File.exist?(path) || File.symlink?(path)
+    end
+
+    def gitignore
+      path = File.join(@root, ".gitignore")
+      link = symlink_component(".gitignore")
+      return Result.new(:skipped, ".gitignore") if link && !@force
+
+      current = File.file?(path) ? File.read(path) : ""
+      listed = listed?(current)
+      return Result.new(:unchanged, ".gitignore") if listed && !link
+
+      File.delete(link) if link
+      File.write(path, "#{current}#{missing_entry(current, listed)}")
+      Result.new(link ? :overwritten : :appended, ".gitignore")
+    end
+
+    def listed?(text)
+      text.lines.map(&:strip).include?(IGNORE_ENTRY)
+    end
+
+    # What to append so .saboteur/ is on its own line; nothing if already there.
+    def missing_entry(current, listed)
+      return "" if listed
+
+      separator = current.empty? || current.end_with?("\n") ? "" : "\n"
+      "#{separator}#{IGNORE_ENTRY}\n"
+    end
+  end
+end

data/lib/saboteur/integration/rspec.rb

@@ -59,14 +59,14 @@ module Saboteur
       def run_all(recorder)
         configuration.order = "rand:#{config.fetch('seed', 1)}"
         configuration.fail_fast = false
-        execute(Listener.new(recorder))
+        execute(Listener.new(recorder), log_phase: true)
       end
 
       def run_baseline(seed:)
         configuration.order = "rand:#{seed}"
         configuration.fail_fast = false
         listener = Listener.new
-        passed = execute(listener)
+        passed = execute(listener, log_phase: true)
         RunResult.new(passed: passed && listener.failed.empty?, failed_ids: listener.failed,
                       executed_ids: listener.executed, runtime: listener.runtime,
                       failure_classes: listener.failure_classes, failure_messages: listener.failure_messages)
@@ -104,11 +104,19 @@ module Saboteur
         world.example_groups.flat_map(&:descendants).each(&:reset_memoized)
       end
 
-      def execute(listener)
+      # log_phase is for the baseline and coverage children, whose $stdout is
+      # redirected to the phase's log: progress and failure detail land in
+      # that log, so a failed phase can be diagnosed from it. Mutant runs
+      # stay quiet.
+      def execute(listener, log_phase: false)
         reporter = ::RSpec::Core::Reporter.new(configuration)
         reporter.register_listener(listener, *Listener::EVENTS)
         groups = world.ordered_example_groups.select { |group| group.descendant_filtered_examples.any? }
         count = world.example_count(groups)
+        if log_phase
+          logger = PhaseLogger.new($stdout, total: count)
+          reporter.register_listener(logger, *PhaseLogger::EVENTS)
+        end
         started = Process.clock_gettime(Process::CLOCK_MONOTONIC)
         passed = reporter.report(count) do |r|
           configuration.with_suite_hooks { groups.map { |group| group.run(r) }.all? }
@@ -117,6 +125,42 @@ module Saboteur
         passed && !world.non_example_failure
       end
 
+      # Progress and failure detail for a phase log. Every 25 finished
+      # examples is enough to see movement on a tail; failures print RSpec's
+      # full formatted detail, and the run ends with RSpec's totals line.
+      class PhaseLogger
+        EVENTS = %i[example_finished message dump_failures dump_summary].freeze
+
+        PROGRESS_EVERY = 25
+
+        def initialize(out, total:)
+          @out = out
+          @total = total
+          @executed = 0
+        end
+
+        def example_finished(_notification)
+          @executed += 1
+          return unless @executed == @total || (@executed % PROGRESS_EVERY).zero?
+
+          @out.puts("#{@executed}/#{@total} examples finished")
+        end
+
+        # A failure outside any example (a suite hook, a load error) arrives
+        # as a message, never in dump_failures.
+        def message(notification)
+          @out.puts(notification.message)
+        end
+
+        def dump_failures(notification)
+          @out.puts(notification.fully_formatted_failed_examples)
+        end
+
+        def dump_summary(notification)
+          @out.puts(notification.totals_line)
+        end
+      end
+
       class Listener
         EVENTS = %i[example_started example_finished example_failed].freeze
 

data/lib/saboteur/mutation_source/argument_removal.rb

@@ -0,0 +1,86 @@
+# SPDX-License-Identifier: MIT
+# frozen_string_literal: true
+
+require "prism"
+require_relative "base"
+
+module Saboteur
+  module MutationSource
+    # Deletes one ordinary argument from a parenthesized method call.
+    class ArgumentRemoval < Base
+      BOUNDARIES = [Prism::DefNode, Prism::ClassNode, Prism::ModuleNode, Prism::SingletonClassNode,
+                    Prism::ParametersNode, Prism::BlockParametersNode].freeze
+      EXCLUDED = [Prism::KeywordHashNode, Prism::SplatNode, Prism::ForwardingArgumentsNode].freeze
+
+      def initialize(arid: []) # rubocop:disable Lint/MissingSuper
+        @arid = arid.map { |pattern| Regexp.new(pattern) }
+      end
+
+      def name
+        "argument_removal"
+      end
+
+      def mutations_for(subject)
+        found = []
+        tokens = Prism.lex(subject.file_source).value.map(&:first)
+        visit(subject.node.body, subject, found, tokens)
+        found
+      end
+
+      private
+
+      def visit(node, subject, found, tokens)
+        return if node.nil? || BOUNDARIES.any? { |type| node.is_a?(type) }
+
+        if node.is_a?(Prism::StatementsNode)
+          node.body.each do |child|
+            visit(child, subject, found, tokens) unless @arid.any? do |p|
+              p.match?(child.slice)
+            end
+          end
+          return
+        end
+        remove_arguments(node, subject, found, tokens) if node.is_a?(Prism::CallNode)
+        node.compact_child_nodes.each { |child| visit(child, subject, found, tokens) }
+      end
+
+      def remove_arguments(node, subject, found, tokens)
+        return unless node.opening_loc&.slice == "(" && node.arguments
+
+        arguments = node.arguments.arguments
+        arguments.each_with_index do |argument, index|
+          next if EXCLUDED.any? { |type| argument.is_a?(type) } || heredoc?(argument)
+
+          separator = separator_for(node, arguments, index, tokens)
+          edits = [delete(subject, argument.location)]
+          edits << delete(subject, separator) if separator
+          found << Mutation.new(subject_id: subject.id, operator: name, edits: edits, source: name)
+        end
+      end
+
+      def separator_for(node, arguments, index, tokens)
+        argument = arguments[index]
+        after = comma_between(argument.location.end_offset, node.closing_loc.start_offset, tokens)
+        return after if after
+
+        previous = arguments[0...index].last
+        comma_between(previous.location.end_offset, argument.location.start_offset, tokens) if previous
+      end
+
+      def comma_between(first, last, tokens)
+        tokens.find do |token|
+          token.type == :COMMA && token.location.start_offset >= first && token.location.end_offset <= last
+        end&.location
+      end
+
+      def heredoc?(node)
+        (node.respond_to?(:opening_loc) && node.opening_loc&.slice&.start_with?("<<")) ||
+          node.compact_child_nodes.any? { |child| heredoc?(child) }
+      end
+
+      def delete(subject, location)
+        Edit.new(file: subject.file, start_byte: location.start_offset, end_byte: location.end_offset, replacement: "")
+      end
+    end
+  end
+end

data/lib/saboteur/mutation_source/builtin.rb

@@ -5,6 +5,8 @@ require "prism"
 require_relative "base"
 require_relative "filter_replacement"
 require_relative "literal_replacement"
+require_relative "argument_removal"
+require_relative "pair_removal"
 
 module Saboteur
   module MutationSource
@@ -50,6 +52,8 @@ module Saboteur
         @arid = arid.map { |pattern| pattern.is_a?(Regexp) ? pattern : Regexp.new(pattern) }
         @filter_replacement = FilterReplacement.new(arid: arid)
         @literal_replacement = LiteralReplacement.new(arid: arid)
+        @argument_removal = ArgumentRemoval.new(arid: arid)
+        @pair_removal = PairRemoval.new(arid: arid)
       end
 
       def name
@@ -64,6 +68,8 @@ module Saboteur
         visit(body)
         @found.concat(@filter_replacement.mutations_for(subject))
         @found.concat(@literal_replacement.mutations_for(subject))
+        @found.concat(@argument_removal.mutations_for(subject))
+        @found.concat(@pair_removal.mutations_for(subject))
       ensure
         @subject = nil
       end
@@ -91,8 +97,8 @@ module Saboteur
       end
 
       # A raise that the method's own rescue catches is absorbed by design. It
-      # would survive, and a surviving coarse mutant stops every fine mutant
-      # of the method, so it is not generated.
+      # is not generated: rejecting that exception is not the rescue contract.
+      # Fine mutants still run regardless of the coarse verdicts.
       def rescues_everything?(body)
         clause = body.is_a?(Prism::BeginNode) ? body.rescue_clause : nil
         while clause

data/lib/saboteur/mutation_source/pair_removal.rb

@@ -0,0 +1,101 @@
+# SPDX-License-Identifier: MIT
+# frozen_string_literal: true
+
+require "prism"
+require_relative "base"
+
+module Saboteur
+  module MutationSource
+    # Deletes one association from a hash literal or keyword argument list.
+    class PairRemoval < Base
+      BOUNDARIES = [Prism::DefNode, Prism::ClassNode, Prism::ModuleNode, Prism::SingletonClassNode,
+                    Prism::ParametersNode, Prism::BlockParametersNode].freeze
+
+      def initialize(arid: []) # rubocop:disable Lint/MissingSuper
+        @arid = arid.map { |pattern| Regexp.new(pattern) }
+      end
+
+      def name
+        "pair_removal"
+      end
+
+      def mutations_for(subject)
+        found = []
+        tokens = Prism.lex(subject.file_source).value.map(&:first)
+        visit(subject.node.body, subject, found, tokens)
+        found
+      end
+
+      private
+
+      def visit(node, subject, found, tokens)
+        return if node.nil? || BOUNDARIES.any? { |type| node.is_a?(type) }
+
+        if node.is_a?(Prism::StatementsNode)
+          node.body.each do |child|
+            visit(child, subject, found, tokens) unless @arid.any? { |p| p.match?(child.slice) }
+          end
+          return
+        end
+        remove_pairs(node, subject, found, tokens) if node.is_a?(Prism::HashNode)
+        remove_keywords(node, subject, found, tokens) if node.is_a?(Prism::CallNode)
+        node.compact_child_nodes.each { |child| visit(child, subject, found, tokens) }
+      end
+
+      def remove_keywords(node, subject, found, tokens)
+        return unless node.arguments
+
+        node.arguments.arguments.grep(Prism::KeywordHashNode).each do |hash|
+          remove_pairs(hash, subject, found, tokens, call: node)
+        end
+      end
+
+      def remove_pairs(node, subject, found, tokens, call: nil)
+        pairs = node.elements
+        pairs.each_with_index do |pair, index|
+          next unless pair.is_a?(Prism::AssocNode)
+          next if heredoc?(pair)
+
+          edits = [delete(subject, pair.location)]
+          separator = separator_for(node, pairs, index, tokens, call)
+          edits << delete(subject, separator) if separator
+          found << Mutation.new(subject_id: subject.id, operator: name, edits: edits, source: name)
+        end
+      end
+
+      def separator_for(node, pairs, index, tokens, call)
+        pair = pairs[index]
+        ending = ending_for(node, call)
+        after = comma_between(pair.location.end_offset, ending, tokens)
+        return after if after
+
+        previous = pairs[0...index].last
+        previous ||= call.arguments.arguments[0...-1].last if call
+        comma_between(previous.location.end_offset, pair.location.start_offset, tokens) if previous
+      end
+
+      def ending_for(node, call)
+        return node.closing_loc.start_offset if node.is_a?(Prism::HashNode)
+        return call.closing_loc.start_offset if call.closing_loc
+        return call.block.location.start_offset if call.block
+
+        node.location.end_offset
+      end
+
+      def comma_between(first, last, tokens)
+        tokens.find do |token|
+          token.type == :COMMA && token.location.start_offset >= first && token.location.end_offset <= last
+        end&.location
+      end
+
+      def heredoc?(node)
+        (node.respond_to?(:opening_loc) && node.opening_loc&.slice&.start_with?("<<")) ||
+          node.compact_child_nodes.any? { |child| heredoc?(child) }
+      end
+
+      def delete(subject, location)
+        Edit.new(file: subject.file, start_byte: location.start_offset, end_byte: location.end_offset, replacement: "")
+      end
+    end
+  end
+end

data/lib/saboteur/replay.rb

@@ -2,7 +2,7 @@
 # frozen_string_literal: true
 
 module Saboteur
-  # Resolves one exact digest from current source, before caps or coarse gating.
+  # Resolves one exact digest from current source, before caps, ignore suppression or incremental reuse.
   class Replay
     def initialize(digest:)
       unless digest.match?(/\A[0-9a-f]{16}\z/)

data/lib/saboteur/reporter/agent.rb

@@ -0,0 +1,87 @@
+# SPDX-License-Identifier: MIT
+# frozen_string_literal: true
+
+require "json"
+require_relative "agent_summary"
+require_relative "agent_commands"
+
+module Saboteur
+  module Reporter
+    # A small JSON summary for an AI agent that runs saboteur (`saboteur run --agent`).
+    #
+    # The payload stays under BYTE_BUDGET: it never embeds file sources or the full
+    # Stryker report. It names the verdict, says plainly what was not covered, shows
+    # the first few findings with short edit snippets, counts the rest, and points
+    # at the commands and files that hold the full detail.
+    #
+    # Exit codes in this mode are stricter than the classic ones: 0 means the run
+    # was complete and clean, nothing less.
+    module Agent
+      VERSION = 1
+      BYTE_BUDGET = 8 * 1024
+      FINDING_LIMIT = 3
+      SNIPPET_LIMITS = [200, 80, 0].freeze
+      MAX_NUMBER_DIGITS = 20
+      WARNING_LIMIT = 3
+      WARNING_LENGTH = 160
+
+      EXIT_CLEAN = 0
+      EXIT_REVIEW = 1
+      EXIT_INVOCATION = 2
+      EXIT_INCOMPLETE = 3
+
+      # Where a verdict comes from: the exact selection and the state of the working
+      # source before the run started. subjects, tests and sources are arrays.
+      Provenance = Data.define(:ref, :seed, :head, :working_tree, :changed_files, :mode, :json_report, :subjects,
+                               :tests, :sources, :config_path, :digest, :cache, :incremental, :settings)
+
+      class << self
+        def call(report, provenance)
+          fit(report, provenance)
+        end
+
+        def render(payload)
+          "#{JSON.pretty_generate(payload)}\n"
+        end
+
+        def exit_code(payload)
+          payload.fetch(:exitCode)
+        end
+
+        # The payload for a run that never started or could not finish.
+        def failure(message, exit_code)
+          verdict = exit_code == EXIT_INVOCATION ? "invalid_invocation" : "tool_failure"
+          { version: VERSION, status: "error", verdict: verdict, exitCode: exit_code,
+            message: message.to_s[0, 500], next: { help: "saboteur help run" } }
+        end
+
+        private
+
+        def fit(report, provenance)
+          FINDING_LIMIT.downto(0) do |shown|
+            SNIPPET_LIMITS.each do |snippet_limit|
+              payload = Summary.build(report, provenance, shown, snippet_limit)
+              return payload if render(payload).bytesize <= BYTE_BUDGET
+            end
+          end
+          bounded_last_resort(report, provenance)
+        end
+
+        # The minimal payload, or four fixed keys when even that is over the budget.
+        def bounded_last_resort(report, provenance)
+          small = minimal(report, provenance)
+          return small if render(small).bytesize <= BYTE_BUDGET
+
+          small.slice(:version, :status, :verdict, :exitCode).merge(truncated: true)
+        end
+
+        # The last resort: fixed keys and short clipped values, so it always fits.
+        def minimal(report, provenance)
+          small = Summary.build(report, provenance, 0, SNIPPET_LIMITS.last)
+          small.merge(warnings: [], next: {}, provenance: small[:provenance].slice(:seed, :workingTree, :mode),
+                      incomplete: small[:incomplete].map { |reason| reason.slice(:reason, :count) })
+        end
+      end
+    end
+  end
+end

data/lib/saboteur/reporter/agent_clipping.rb

@@ -0,0 +1,29 @@
+# SPDX-License-Identifier: MIT
+# frozen_string_literal: true
+
+module Saboteur
+  module Reporter
+    module Agent
+      # Keeps display values short so the agent payload stays inside its byte budget.
+      module Clipping
+        # A seed or setting stays a number while it is short; a huge value cannot take the budget.
+        def bounded_number(value)
+          value.to_s.length <= MAX_NUMBER_DIGITS ? value : clip_value(value, MAX_NUMBER_DIGITS)
+        end
+
+        def clip_value(value, limit)
+          return nil if value.nil?
+
+          text = value.to_s
+          text.length > limit ? "#{text[0, limit]}..." : text
+        end
+
+        def clip(text, limit)
+          return "" if limit.zero?
+
+          text.length > limit ? "#{text[0, limit]}..." : text
+        end
+      end
+    end
+  end
+end

data/lib/saboteur/reporter/agent_commands.rb

@@ -0,0 +1,61 @@
+# SPDX-License-Identifier: MIT
+# frozen_string_literal: true
+
+require "shellwords"
+
+module Saboteur
+  module Reporter
+    module Agent
+      # The shell commands an agent is told to run next. Every word is escaped with
+      # Shellwords, so a ref, path or subject holding shell syntax stays one literal
+      # argument. A command is never cut short: when it is longer than MAX_BYTES it
+      # is left out, because half a command could run something else.
+      module Commands
+        MAX_BYTES = 1024
+
+        # Options typed on the command line that change how a run behaves.
+        SETTING_FLAGS = { reeval: "--reeval", integration: "--integration", jobs: "--jobs",
+                          timeout_factor: "--timeout-factor", json_report: "--json", html_report: "--html" }.freeze
+
+        class << self
+          # Run the same selection again: ref, subjects, sources, tests, config,
+          # cache mode and replay digest, with the recorded seed.
+          def rerun(provenance)
+            words = ["saboteur", "run", "--agent", *selection(provenance), "--seed", provenance.seed.to_s]
+            finish(words, provenance)
+          end
+
+          # Replay one survivor by digest in the same selection.
+          def replay(provenance, digest)
+            words = ["saboteur", "run", "--id", digest, *selection(provenance, replay: true), "--seed",
+                     provenance.seed.to_s]
+            finish(words, provenance)
+          end
+
+          private
+
+          def finish(words, provenance)
+            words += ["--", *provenance.sources] unless provenance.sources.empty?
+            command = Shellwords.join(words)
+            command.bytesize <= MAX_BYTES ? command : nil
+          end
+
+          def selection(provenance, replay: false)
+            words = []
+            words.push("--config", provenance.config_path) if provenance.config_path
+            words.push("--since", provenance.ref) if provenance.ref
+            provenance.subjects.each { |subject| words.push("--subject", subject) }
+            provenance.tests.each { |test| words.push("--test", test) }
+            SETTING_FLAGS.each do |key, flag|
+              words.push(flag, provenance.settings[key].to_s) if provenance.settings.key?(key)
+            end
+            words << "--no-cache" if provenance.cache == false
+            words << "--incremental" if provenance.incremental
+            words.push("--id", provenance.digest) if provenance.digest && !replay
+            words
+          end
+        end
+      end
+    end
+  end
+end

data/lib/saboteur/reporter/agent_incomplete.rb

@@ -0,0 +1,37 @@
+# SPDX-License-Identifier: MIT
+# frozen_string_literal: true
+
+module Saboteur
+  module Reporter
+    module Agent
+      # What makes a verdict cover less than the whole selection. Each reason has a
+      # stable code, a count and a plain meaning.
+      module Incomplete
+        class << self
+          def reasons(report)
+            reasons = []
+            add(reasons, "no_coverage", skipped_for_coverage(report), "subjects or mutants had no covering test")
+            add(reasons, "mutant_cap", omitted_by_cap(report), "mutants omitted by max_mutants_per_method")
+            add(reasons, "errors", report.counts[:error], "mutants errored; their verdict did not hold up")
+            add(reasons, "resource_limit", report.counts[:resource_limit], "mutants hit the memory or process cap")
+            reasons
+          end
+
+          private
+
+          def add(reasons, code, count, meaning)
+            reasons << { reason: code, count: count, meaning: meaning } if count.positive?
+          end
+
+          def skipped_for_coverage(report)
+            report.skipped_subjects.size + report.counts[:skipped]
+          end
+
+          def omitted_by_cap(report)
+            report.warnings.sum { |warning| warning[/omitted (\d+) mutant/, 1].to_i }
+          end
+        end
+      end
+    end
+  end
+end