saboteur 0.2.0

data/lib/saboteur/engine/coverage_store.rb

@@ -0,0 +1,99 @@
+# SPDX-License-Identifier: MIT
+# frozen_string_literal: true
+
+require "coverage"
+require "fileutils"
+require "json"
+
+module Saboteur
+  class Engine
+    # Returns the coverage map for a run: from the cache when every input is
+    # unchanged, otherwise built by running the whole suite once in a
+    # supervised child and saved.
+    class CoverageStore
+      CACHE_INPUTS = %w[integration require rspec_args rails].freeze
+      LOCKFILES = %w[Gemfile.lock gems.locked].freeze
+      LOG = "coverage-build.log"
+
+      def initialize(config:, integration:, hooks:, supervisor:, diagnostics:)
+        @config = config
+        @integration = integration
+        @hooks = hooks
+        @supervisor = supervisor
+        @diagnostics = diagnostics
+        @root = File.realpath(config.root)
+      end
+
+      def fetch(files)
+        path = File.join(@config.path("cache_dir"), "coverage-#{cache_key(files)}.json")
+        cached = @config["cache"] ? Selection::CoverageMap.load(path) : nil
+        return cached if cached
+
+        map = build(files)
+        map.save(path)
+        map
+      end
+
+      private
+
+      def cache_key(files)
+        inputs = JSON.generate(CACHE_INPUTS.to_h { |key| [key, @config[key]] })
+        all = files + @integration.test_files + SuiteFiles.new(@config).files + lockfiles
+        Selection::CoverageMap.cache_key(files: all.uniq, extra: inputs)
+      end
+
+      def lockfiles
+        LOCKFILES.map { |file| File.join(@root, file) }.select { |file| File.file?(file) }
+      end
+
+      def build(files)
+        unless Coverage.respond_to?(:state) && Coverage.state == :suspended
+          raise Error, "Coverage is #{Coverage.state}; saboteur must set it up before the app loads. " \
+                       "Run through the saboteur executable and disable SimpleCov for the run."
+        end
+
+        @diagnostics.inform("building coverage map (runs the whole suite once)")
+        result = @supervisor.run(timeout: @config["coverage_timeout"]) { |writer| record(writer, files) }
+        raise Error, "coverage map build stopped: #{result.detail}" if result.stopped_by
+        raise Error, "coverage map build failed: #{died(result.exit_status)}" if result.payload.empty?
+
+        data = JSON.parse(result.payload)
+        raise Error, "coverage map build failed: #{data['error']}" if data["error"]
+
+        Selection::CoverageMap.from_json(result.payload)
+      end
+
+      def died(status)
+        how = if status&.signaled? then "was killed by signal #{Signal.signame(status.termsig)}"
+              else "exited with status #{status&.exitstatus}"
+              end
+        "the suite #{how} before reporting; its output is in #{log_path.delete_prefix("#{@root}/")}"
+      end
+
+      def log_path
+        File.join(@config.path("cache_dir"), LOG)
+      end
+
+      # Runs in the supervised child. The suite's output goes to a log, so a
+      # child that dies without reporting can still be diagnosed.
+      def record(writer, files)
+        FileUtils.mkdir_p(File.dirname(log_path))
+        $stdout.reopen(log_path, "w")
+        $stdout.sync = true
+        $stderr.reopen($stdout)
+        @hooks.run(:after_fork)
+        started = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+        recorder = Selection::Recorder.new(files)
+        recorder.start
+        Kernel.srand(@config["seed"])
+        @integration.run_all(recorder)
+        build_seconds = Process.clock_gettime(Process::CLOCK_MONOTONIC) - started
+        writer.write(recorder.result(build_seconds: build_seconds).to_json)
+        writer.close
+      rescue Exception => e # rubocop:disable Lint/RescueException
+        writer.write(JSON.generate(error: "#{e.class}: #{e.message}"))
+        exit!(1)
+      end
+    end
+  end
+end

data/lib/saboteur/engine/diagnostics.rb

@@ -0,0 +1,37 @@
+# SPDX-License-Identifier: MIT
+# frozen_string_literal: true
+
+module Saboteur
+  class Engine
+    # What a run has to say besides results: warnings for the report, the
+    # count of mutations dropped for not parsing, and progress marks.
+    class Diagnostics
+      MARKS = { killed: ".", survived: "S", timeout: "T", resource_limit: "L", error: "E" }.freeze
+
+      attr_reader :warnings, :invalid_mutations
+
+      def initialize(err)
+        @err = err
+        @warnings = []
+        @invalid_mutations = 0
+      end
+
+      def warn(message)
+        @warnings << message
+      end
+
+      def count_invalid_mutation
+        @invalid_mutations += 1
+      end
+
+      def inform(message)
+        @err.puts("saboteur: #{message}")
+      end
+
+      def progress(result)
+        @err.print(MARKS.fetch(result.status, "?"))
+        @err.flush
+      end
+    end
+  end
+end

data/lib/saboteur/engine/diff_scope.rb

@@ -0,0 +1,32 @@
+# SPDX-License-Identifier: MIT
+# frozen_string_literal: true
+
+module Saboteur
+  class Engine
+    # Review-shaped diff mode (--since). A reviewer asks "do the tests check
+    # what this change did?", so only mutants on changed lines answer it.
+    # The coarse mutants of each changed method still run: they ask whether
+    # the method is tested at all.
+    class DiffScope
+      # changed_lines: { "/abs/file.rb" => Set[line numbers] }
+      def initialize(changed_lines)
+        @changed_lines = changed_lines
+      end
+
+      def restrict(subject, candidates)
+        changed = @changed_lines.fetch(subject.file, Set.new)
+        candidates.select do |mutation, _|
+          mutation.coarse || mutation.edits.any? { |edit| edited_lines(subject, edit).any? { changed.include?(_1) } }
+        end
+      end
+
+      private
+
+      def edited_lines(subject, edit)
+        first = subject.line_of(edit.start_byte)
+        last = subject.line_of([edit.end_byte - 1, edit.start_byte].max)
+        (first..last)
+      end
+    end
+  end
+end

data/lib/saboteur/engine/distribution.rb

@@ -0,0 +1,74 @@
+# SPDX-License-Identifier: MIT
+# frozen_string_literal: true
+
+module Saboteur
+  class Engine
+    # Runs each subject's candidates through the scheduler, on one process or
+    # on a pool of workers (--jobs N). A worker sends its results back as
+    # plain data; the parent rebuilds them against its own copy of the same
+    # candidate list, so results look the same either way.
+    class Distribution
+      def initialize(jobs:, scheduler:, diagnostics:, cache:, on_start: nil)
+        @jobs = jobs
+        @on_start = on_start
+        @scheduler = scheduler
+        @diagnostics = diagnostics
+        @cache = cache
+      end
+
+      # lists: each subject's [[mutation, text]] candidates, in subject order.
+      def run(subjects, lists)
+        return subjects.zip(lists).flat_map { |subject, list| @scheduler.call(subject, list) } if @jobs <= 1
+
+        pool = Execution::WorkerPool.new(jobs: @jobs, on_start: @on_start)
+        rows = pool.map(subjects.each_index.to_a, weights: lists.map(&:size)) do |i|
+          work(subjects[i], lists[i])
+        end
+        rows.each_with_index.flat_map { |row, i| rebuild(row, subjects[i], lists[i]) }
+      end
+
+      private
+
+      # Runs in a worker.
+      def work(subject, list)
+        before = @diagnostics.warnings.size
+        results = @scheduler.call(subject, list)
+        { "warnings" => @diagnostics.warnings.drop(before),
+          "results" => results.map { |result| encode(result, list) } }
+      end
+
+      def encode(result, list)
+        {
+          "position" => list.index { |mutation, _| mutation.equal?(result.mutation) },
+          "status" => result.status.to_s, "reason" => result.reason, "runtime" => result.runtime,
+          "killed_by" => result.killed_by, "covered_by" => result.covered_by,
+          "kill_cause" => result.kill_cause, "reused" => result.reused?
+        }
+      end
+
+      def rebuild(row, subject, list)
+        return failed(row["error"], subject, list) if row["error"]
+
+        @diagnostics.warnings.concat(row["warnings"])
+        row["results"].map do |data|
+          mutation, text = list.fetch(data["position"])
+          result = MutantResult.new(subject: subject, mutation: mutation, status: data["status"].to_sym,
+                                    mutated_source: text, reason: data["reason"], runtime: data["runtime"],
+                                    killed_by: data["killed_by"], covered_by: data["covered_by"],
+                                    kill_cause: data["kill_cause"], reused: data["reused"])
+          @cache&.record(result) unless result.reused?
+          result
+        end
+      end
+
+      # The worker died (or raised) while running this subject.
+      def failed(message, subject, list)
+        @diagnostics.warn("#{subject.id}: #{message}")
+        list.map do |mutation, text|
+          MutantResult.new(subject: subject, mutation: mutation, status: :error, mutated_source: text,
+                           reason: message)
+        end
+      end
+    end
+  end
+end

data/lib/saboteur/engine/result_cache.rb

@@ -0,0 +1,93 @@
+# SPDX-License-Identifier: MIT
+# frozen_string_literal: true
+
+require "digest"
+require "fileutils"
+require "json"
+
+module Saboteur
+  class Engine
+    # Opt-in reuse of earlier verdicts (--incremental), like PIT's and
+    # Stryker's history files.
+    #
+    # A verdict is reused only while everything it was computed from is the
+    # same: the method's source, the mutant, the set of covering tests, the
+    # content of those tests' files, the lockfile, the run's inputs (config,
+    # saboteur version). Only killed, survived and timeout are stored;
+    # errors and skips always run again.
+    #
+    # Known hole, the reason this is opt-in: a change in a method this one
+    # calls is not seen, so a stored verdict may be stale after it.
+    class ResultCache
+      FORMAT = 1
+      STORED = %i[killed survived timeout].freeze
+      LOCKFILES = %w[Gemfile.lock gems.locked].freeze
+
+      # suite_digest: SuiteFiles#digest, so an edit to a spec helper or
+      # support file invalidates every verdict.
+      def initialize(path:, root:, inputs:, suite_digest: "")
+        @path = path
+        @root = root
+        @inputs = inputs
+        @suite_digest = suite_digest
+        @entries = load
+        @file_digests = {}
+      end
+
+      # The stored verdict as a hash (status, reason, killed_by, kill_cause,
+      # runtime), or nil when it may no longer hold.
+      def lookup(subject, mutation, tests)
+        entry = @entries[key(subject, mutation)]
+        return nil unless entry && entry["fingerprint"] == fingerprint(subject, tests)
+
+        { status: entry["status"].to_sym, reason: entry["reason"], killed_by: entry["killed_by"],
+          kill_cause: entry["kill_cause"], runtime: entry["runtime"] }
+      end
+
+      def record(result)
+        return unless STORED.include?(result.status)
+
+        @entries[key(result.subject, result.mutation)] = {
+          "fingerprint" => fingerprint(result.subject, result.covered_by), "status" => result.status.to_s,
+          "reason" => result.reason, "killed_by" => result.killed_by, "kill_cause" => result.kill_cause,
+          "runtime" => result.runtime
+        }
+      end
+
+      def save
+        FileUtils.mkdir_p(File.dirname(@path))
+        File.write(@path, JSON.generate(format: FORMAT, entries: @entries))
+      end
+
+      private
+
+      def load
+        data = JSON.parse(File.read(@path))
+        data["format"] == FORMAT ? data["entries"] : {}
+      rescue SystemCallError, JSON::ParserError
+        {}
+      end
+
+      def key(subject, mutation)
+        "#{subject.id} #{mutation.digest(subject_start: subject.start_byte, original_source: subject.file_source)}"
+      end
+
+      def fingerprint(subject, tests)
+        digest = Digest::SHA256.new
+        digest << VERSION << "\0" << @inputs << "\0" << @suite_digest << "\0" << subject.source << "\0"
+        tests.sort.each { |test| digest << test << "\0" << file_digest(test_file(test)) << "\0" }
+        LOCKFILES.each { |name| digest << file_digest(File.join(@root, name)) << "\0" }
+        digest.hexdigest
+      end
+
+      # RSpec ids name their file: "./spec/a_spec.rb[1:2]".
+      def test_file(test)
+        File.expand_path(test[/\A(.+?\.rb)(?:\[|:|\z)/, 1] || test, @root)
+      end
+
+      def file_digest(path)
+        @file_digests[path] ||= File.file?(path) ? Digest::SHA256.file(path).hexdigest : "missing"
+      end
+    end
+  end
+end

data/lib/saboteur/engine/scheduler.rb

@@ -0,0 +1,215 @@
+# SPDX-License-Identifier: MIT
+# frozen_string_literal: true
+
+module Saboteur
+  class Engine
+    # Runs one subject's candidates and returns their results. This is the
+    # unit of work parallel workers will share out.
+    #
+    # Order of business: skip what cannot be run (refinements, not loaded,
+    # shadowed, no covering test); coarse mutants; fine mutants only if no
+    # coarse mutant survived; then check every kill and timeout against the
+    # unchanged method (VerdictCheck). Each mutant runs only the tests that
+    # executed its edit, likeliest killers first, with a timeout from those
+    # tests' recorded durations.
+    class Scheduler
+      # A subject to run, with what its mutants need. killers counts the
+      # tests that killed earlier mutants of this subject; slowdown is how
+      # much slower than recorded its tests ran in a child (see calibrate).
+      Plan = Struct.new(:subject, :tests, :killers, :slowdown, keyword_init: true)
+
+      # Tests that killed more earlier mutants first; otherwise keep the order.
+      def self.prioritize(tests, killers)
+        tests.sort_by.with_index { |id, index| [-killers.fetch(id, 0), index] }
+      end
+
+      def initialize(config:, runner:, map:, reevaluator:, diagnostics:, cache: nil)
+        @config = config
+        @runner = runner
+        @map = map
+        @reevaluator = reevaluator
+        @diagnostics = diagnostics
+        @cache = cache
+      end
+
+      def call(subject, candidates)
+        return [] if candidates.empty?
+
+        reason = subject.skip_reason || load_problem(subject)
+        return skip_all(candidates, subject, reason) if reason
+
+        tests = @map.tests_for(subject)
+        return skip_all(candidates, subject, "no coverage") if tests.empty?
+
+        plan = Plan.new(subject:, tests:, killers: Hash.new(0), slowdown: 1.0)
+        reused = all_reused(plan, candidates)
+        return reused if reused
+
+        results = run_candidates(plan, candidates) do |mutation, text|
+          reuse(plan, mutation, text) || execute(plan, mutation, text)
+        end
+        checked(subject, results)
+      end
+
+      # The per-mutant timeout: timeout_factor times what the tests took
+      # while the map was built (times the subject's measured slowdown), and
+      # never less than timeout_min.
+      def timeout_for(tests, slowdown = 1.0)
+        recorded = @map.recorded_runtime(tests)
+        return @config["baseline_timeout"] unless recorded
+
+        deadline(recorded * slowdown)
+      end
+
+      private
+
+      # Verdicts that did not hold up are warned about once per subject;
+      # only checked verdicts are stored for --incremental.
+      def checked(subject, results)
+        run_unchanged = lambda do |tests, timeout|
+          @runner.run(subject: subject, text: subject.source, test_ids: tests, timeout: timeout)
+                 .tap { |outcome| note_leftovers(subject, "verdict check", outcome) }
+        end
+        check = VerdictCheck.new(run_unchanged: run_unchanged, baseline_timeout: @config["baseline_timeout"])
+        final = check.call(results)
+        final.reject(&:reused?).each { |result| @cache&.record(result) }
+        overturned = final.zip(results).count { |after, before| !after.equal?(before) }
+        if overturned.positive?
+          @diagnostics.warn("#{subject.id}: #{overturned} verdict(s) failed with the method unchanged")
+        end
+        final
+      end
+
+      # When every verdict can be reused, nothing forks: not even the neutral
+      # check. Returns the results, or nil at the first one that must run.
+      def all_reused(plan, candidates)
+        return nil unless @cache
+
+        catch(:must_run) do
+          run_candidates(plan, candidates) { |mutation, text| reuse(plan, mutation, text) || throw(:must_run) }
+        end
+      end
+
+      def run_candidates(plan, candidates, &run_one)
+        coarse, fine = candidates.partition { |mutation, _| mutation.coarse }
+        results = coarse.map { |mutation, text| run_one.call(mutation, text) }
+        # A surviving coarse mutant means nothing checks the body, so the fine
+        # mutants would only add noise. Not so when a rescue or else can
+        # return a value of its own: fine mutants that make the body raise
+        # can still be caught.
+        if results.any?(&:survived?) && !plan.subject.recovers?
+          return results + skip_all(fine, plan.subject, "coarse mutant survived", covered_by: plan.tests)
+        end
+
+        results + fine.map { |mutation, text| run_one.call(mutation, text) }
+      end
+
+      # A stored verdict that still holds, as a result; nil when it must run.
+      def reuse(plan, mutation, text)
+        return nil unless @cache
+
+        tests = tests_for(plan, mutation)
+        return skip_all([[mutation, text]], plan.subject, "no coverage").first if tests.empty?
+
+        hit = @cache.lookup(plan.subject, mutation, tests)
+        return nil unless hit
+
+        MutantResult.new(subject: plan.subject, mutation: mutation, status: hit[:status], mutated_source: text,
+                         reason: hit[:reason], runtime: hit[:runtime], killed_by: hit[:killed_by],
+                         covered_by: tests, kill_cause: hit[:kill_cause], reused: true)
+      end
+
+      def execute(plan, mutation, text)
+        tests = tests_for(plan, mutation)
+        return skip_all([[mutation, text]], plan.subject, "no coverage").first if tests.empty?
+
+        timeout = timeout_for(tests, plan.slowdown)
+        outcome = @runner.run(subject: plan.subject, text: text, test_ids: tests, timeout: timeout, mutation: mutation)
+        outcome = calibrate(plan, mutation, text, tests, timeout) if outcome.status == :timeout
+        result = MutantResult.new(subject: plan.subject, mutation: mutation, status: outcome.status,
+                                  mutated_source: text, reason: outcome.message, runtime: outcome.runtime,
+                                  killed_by: outcome.failed_ids, covered_by: tests, kill_cause: outcome.kill_cause)
+        outcome.failed_ids.each { |id| plan.killers[id] += 1 }
+        note_leftovers(plan.subject, "#{mutation.operator} #{result.digest}", outcome)
+        @diagnostics.progress(result)
+        result
+      end
+
+      # The tests that ran the edited code (a coarse edit spans the whole
+      # body, so it gets every test of the method), prior killers first, then
+      # fastest-first.
+      def tests_for(plan, mutation)
+        line_numbers = mutation.edits.flat_map { |edit| plan.subject.coverage_lines_for(edit) }.uniq
+        tests = plan.tests & @map.tests_for_lines(plan.subject.file, line_numbers)
+        self.class.prioritize(tests, plan.killers)
+      end
+
+      def deadline(runtime)
+        [@config["timeout_factor"] * runtime, @config["timeout_min"]].max
+      end
+
+      # Recorded durations come from one warm process; a fresh child can be
+      # slower (lazy loading, first queries). So a timeout is checked at
+      # once: the unchanged method runs the same tests with
+      # baseline_timeout. If the deadline from that real runtime is longer,
+      # the mutant runs again with it, and later mutants of the subject get
+      # the measured slowdown. A timeout that remains is the mutant's own.
+      def calibrate(plan, mutation, text, tests, timeout)
+        unchanged = @runner.run(subject: plan.subject, text: plan.subject.source, test_ids: tests,
+                                timeout: @config["baseline_timeout"])
+        note_leftovers(plan.subject, "timeout check", unchanged)
+        return unchanged_failed(unchanged) unless unchanged.status == :survived
+
+        recorded = @map.recorded_runtime(tests)
+        plan.slowdown = [plan.slowdown, unchanged.runtime / recorded].max if recorded&.positive?
+        longer = deadline(unchanged.runtime)
+        return Execution::Runner::Outcome.new(status: :timeout, runtime: timeout) if longer <= timeout
+
+        @runner.run(subject: plan.subject, text: text, test_ids: tests, timeout: longer, mutation: mutation)
+      end
+
+      def unchanged_failed(unchanged)
+        how = unchanged.status == :timeout ? "time out" : "fail"
+        hint = unchanged.status == :timeout ? " (raise baseline_timeout)" : ""
+        Execution::Runner::Outcome.new(status: :error, runtime: unchanged.runtime,
+                                       message: "tests also #{how} with the method unchanged#{hint}")
+      end
+
+      # The method saboteur would redefine must be the one Ruby is running.
+      def load_problem(subject)
+        owner = @reevaluator.resolve_owner(subject)
+        target = subject.singleton ? owner.singleton_class : owner
+        method = own_method(target, subject.name)
+        return "method not loaded" unless method
+
+        file, line = method.source_location
+        return nil if file && Selection::CoverageMap.canonical(file) == subject.file && line == subject.start_line
+
+        "shadowed definition (runtime method at #{file}:#{line})"
+      rescue Error => e
+        "owner not loaded: #{e.message}"
+      end
+
+      def own_method(target, name)
+        return nil unless target.method_defined?(name) || target.private_method_defined?(name)
+
+        method = target.instance_method(name)
+        method = method.super_method while method && method.owner != target
+        method
+      end
+
+      def note_leftovers(subject, what, outcome)
+        return if outcome.leftover_pids.empty?
+
+        @diagnostics.warn("#{subject.id} #{what}: killed #{outcome.leftover_pids.size} process(es) it left running")
+      end
+
+      def skip_all(candidates, subject, reason, covered_by: [])
+        candidates.map do |mutation, text|
+          MutantResult.new(subject: subject, mutation: mutation, status: :skipped, mutated_source: text,
+                           reason: reason, covered_by: covered_by)
+        end
+      end
+    end
+  end
+end

data/lib/saboteur/engine/subject_selection.rb

@@ -0,0 +1,63 @@
+# SPDX-License-Identifier: MIT
+# frozen_string_literal: true
+
+module Saboteur
+  class Engine
+    # The subjects to mutate: every def in the included files, narrowed by
+    # --subject patterns and by --since (methods with a changed line).
+    class SubjectSelection
+      # Lines changed since the --since ref, by file; nil without --since.
+      attr_reader :changed_lines
+
+      def initialize(config, diagnostics)
+        @config = config
+        @diagnostics = diagnostics
+        @root = File.realpath(config.root)
+        @changed_lines = nil
+      end
+
+      def select(files)
+        subjects = find(files)
+        subjects = by_pattern(subjects, @config["subjects"]) if @config["subjects"].any?
+        subjects = by_change(subjects, @config["since"]) if @config["since"]
+        subjects
+      end
+
+      # "Klass#meth", "Klass.meth", or a namespace prefix ("Klass", "A::B").
+      def self.match?(id, pattern)
+        id == pattern || %w[# . ::].any? { |separator| id.start_with?("#{pattern}#{separator}") }
+      end
+
+      private
+
+      def find(files)
+        finder = SubjectFinder.new
+        files.flat_map do |file|
+          parsed = finder.call(file)
+          parsed.errors.first(1).each do |message|
+            @diagnostics.warn("#{file.delete_prefix("#{@root}/")} does not parse: #{message}")
+          end
+          parsed.subjects
+        end
+      end
+
+      def by_pattern(subjects, patterns)
+        patterns.each do |pattern|
+          next if subjects.any? { |subject| self.class.match?(subject.id, pattern) }
+
+          @diagnostics.warn("--subject #{pattern} matches no subject (patterns are names like " \
+                            "Klass, Klass#meth or Klass.meth, not globs)")
+        end
+        subjects.select { |subject| patterns.any? { |pattern| self.class.match?(subject.id, pattern) } }
+      end
+
+      def by_change(subjects, ref)
+        @changed_lines = Selection::GitDiff.changed_lines(ref, root: @root)
+                                           .transform_keys { |file| Selection::CoverageMap.canonical(file) }
+        subjects.select do |subject|
+          @changed_lines[subject.file]&.any? { |line| subject.contains_line?(line) }
+        end
+      end
+    end
+  end
+end