saboteur 0.2.0 → 0.4.0

data/lib/saboteur/engine.rb

@@ -3,66 +3,136 @@
 
 require "fileutils"
 require "json"
+require_relative "replay"
 
 module Saboteur
-  # One run, as a pipeline: boot the target, select subjects, fetch the
+  # A single-use instance runs one pipeline: boot the target, select subjects, fetch the
   # coverage map, then schedule each subject's mutants. The parts live in
   # engine/; this class only wires them together.
   class Engine
     # Everything besides sources and tests that can change a verdict.
     VERDICT_INPUTS = %w[integration require rspec_args rails reeval sources arid timeout_factor timeout_min
-                        max_memory_mb max_processes seed].freeze
+                        max_memory_mb max_processes seed tests].freeze
 
     attr_reader :config, :hooks
 
-    def initialize(config, err: $stderr)
+    def initialize(config, err: $stderr, digest: nil)
       @config = config
       @hooks = Hooks.new
       @diagnostics = Diagnostics.new(err)
       @root = File.realpath(config.root)
+      @skipped_subjects = {}
+      @replay = Replay.new(digest: digest) if digest
     end
 
     def call
       Boot.refuse_recursion!
       started = monotonic
-      ignore_list = IgnoreList.load(config.path("ignore_file"))
+      ignore_list = @replay ? IgnoreList.new([]) : IgnoreList.load(config.path("ignore_file"))
+      @diagnostics.warnings.concat(custom_map_warnings(load_custom_map))
       boot = Boot.new(config, hooks)
-      integration = boot.load_target
-      @rails = boot.rails
-      files = config.included_files
-      subjects, scope = select_subjects(files)
-      results = subjects.empty? ? [] : mutate(subjects, files, integration, scope)
-      @diagnostics.warnings.concat(ignore_list.apply(results))
-      Report.new(results: results, subjects: subjects, wall_time: monotonic - started, root: @root,
-                 invalid_mutations: @diagnostics.invalid_mutations, warnings: @diagnostics.warnings,
-                 review: !scope.nil?, seed: config["seed"])
+      begin
+        integration = boot.load_target
+        @rails = boot.rails
+        files = config.included_files
+        subjects, scope = select_subjects(files)
+        results = subjects.empty? ? [] : mutate(subjects, files, integration, scope)
+        @diagnostics.warnings.concat(ignore_list.apply(results))
+        build_report(results, subjects, scope, started)
+      ensure
+        # The run boundary for the per-worker databases: cleanup runs however
+        # the call ends - a boot that raised partway through, a baseline or
+        # coverage failure, no subjects, no runnable work - so a host that
+        # embeds Engine and calls it repeatedly never holds copies or
+        # snapshots until process exit. cleanup_workers is idempotent; the
+        # Rails profile's at_exit handler stays as the fallback for an
+        # abrupt process exit.
+        safely_cleanup_workers(boot.rails)
+      end
+    end
+
+    # Stale entries and duplicate literals never fail a run; they warn so the
+    # map does not rot silently. doctor fails on stale keys.
+    def load_custom_map
+      @replay ? CustomMap.new([]) : CustomMap.load(config.path("custom_map"), root: config.root)
+    end
+
+    def custom_map_warnings(map)
+      path = config.path("custom_map")
+      warnings = map.stale_keys.map { |key| "stale custom map key #{key}: matches no file" }
+      return warnings unless path && File.file?(path)
+
+      warnings + map.duplicate_keys(path).map { |key| "duplicate custom map key #{key}: Ruby keeps the last" }
     end
 
     private
 
+    # The run's immutable report, built once the last result is in.
+    def build_report(results, subjects, scope, started)
+      Report.new(results: results, subjects: subjects, wall_time: monotonic - started, root: @root,
+                 invalid_mutations: @diagnostics.invalid_mutations, warnings: @diagnostics.warnings,
+                 review: !scope.nil?, seed: config["seed"], skipped_subjects: @skipped_subjects,
+                 diff_scope: scope)
+    end
+
     # The subjects to mutate and, in diff mode, the scope of changed lines.
     def select_subjects(files)
       config.unmatched_patterns.each { |pattern| @diagnostics.warn("#{pattern} matches no file") }
       selection = SubjectSelection.new(config, @diagnostics)
       subjects = selection.select(files)
-      [subjects, selection.changed_lines && DiffScope.new(selection.changed_lines)]
+      scope = selection.changed_lines && DiffScope.new(selection.changed_lines)
+      return replay_subject(subjects, scope) if @replay
+
+      [subjects, scope]
     end
 
     def mutate(subjects, files, integration, scope)
+      warn_about_unscoped_suite(integration)
+      check_baseline(integration)
       map = CoverageStore.new(config: config, integration: integration, hooks: hooks,
                               supervisor: supervisor(memory_factor: 4), diagnostics: @diagnostics).fetch(files)
-      @diagnostics.inform("#{subjects.size} subjects, #{map.tests.size} tests in coverage map")
-      warn_about_threads
+      inform_about_run(subjects, map)
       cache = result_cache
       scheduler = scheduler_for(map, integration, cache)
+      lists = candidate_lists(subjects, scope, map)
+      work = subjects.zip(lists).reject { |_, list| list.empty? }
+      return [] if work.empty?
+
+      runnable_subjects, runnable_lists = work.transpose
       @rails&.prepare_workers(config["jobs"])
+      # The execution phase forks next: the worker batch, or one child per
+      # mutant when serial. Pack the heap now that the map and candidates
+      # are retained, to reduce later copying of shared pages.
+      Execution::ForkReadiness.prepare_heap
       results = Distribution.new(jobs: config["jobs"], scheduler: scheduler, diagnostics: @diagnostics,
-                                 cache: cache, on_start: method(:start_worker)).run(subjects, candidate_lists(subjects,
-                                                                                                              scope))
+                                 cache: cache, on_start: method(:start_worker)).run(runnable_subjects,
+                                                                                    runnable_lists)
       cache&.save
       results
     end
 
+    def inform_about_run(subjects, map)
+      @diagnostics.inform("#{subjects.size} subjects, #{map.tests.size} tests in coverage map")
+      warn_about_threads
+    end
+
+    # A --since run without --test runs the loaded suite twice for the
+    # baselines; a cold coverage cache adds one more pass. Say so
+    # before those hours pass, with the way to scope it.
+    def warn_about_unscoped_suite(integration)
+      return unless config["since"] && config["tests"].empty?
+
+      message = "running the loaded suite unscoped: two baseline passes; coverage may add a pass " \
+                "(#{integration.test_files.size} spec files). Pass --test to scope it."
+      @diagnostics.warn(message)
+      @diagnostics.inform(message)
+    end
+
+    def check_baseline(integration)
+      BaselineGuard.new(config: config, integration: integration, hooks: hooks,
+                        supervisor: supervisor(memory_factor: 4), diagnostics: @diagnostics).call
+    end
+
     def scheduler_for(map, integration, cache)
       reevaluator = Execution::Reevaluator.new(config["reeval"])
       runner = Execution::Runner.new(integration: integration, reevaluator: reevaluator, hooks: hooks,
@@ -72,6 +142,7 @@ module Saboteur
     end
 
     def result_cache
+      return nil if @replay
       return nil unless config["incremental"]
 
       inputs = JSON.generate(VERDICT_INPUTS.to_h { |key| [key, config[key]] })
@@ -79,6 +150,14 @@ module Saboteur
                       suite_digest: SuiteFiles.new(config).digest)
     end
 
+    # A cleanup failure must never mask the run's own outcome - not even
+    # one cleanup_workers did not anticipate.
+    def safely_cleanup_workers(rails)
+      rails&.cleanup_workers
+    rescue StandardError => e
+      @diagnostics.warn(e.message)
+    end
+
     # In each worker: its own database, then the user's worker_start hooks.
     def start_worker(number)
       @rails&.start_worker(number)
@@ -86,14 +165,24 @@ module Saboteur
     end
 
     # Each subject's candidates, narrowed to changed lines in diff mode.
-    def candidate_lists(subjects, scope)
+    def candidate_lists(subjects, scope, map)
+      return [@replay_candidate] if @replay
+
       candidates = Candidates.new(Candidates.build_sources(config), @diagnostics)
-      subjects.map do |subject|
-        list = candidates.for(subject)
-        scope ? scope.restrict(subject, list) : list
+      selection = CandidateSelection.new(candidates: candidates, map: map,
+                                         limit: config["max_mutants_per_method"], diagnostics: @diagnostics)
+      subjects.map { |subject| selection.for(subject, scope: scope) }.tap do
+        @skipped_subjects = selection.skipped_subjects
       end
     end
 
+    def replay_subject(subjects, scope)
+      candidates = Candidates.new(Candidates.build_sources(config), @diagnostics)
+      subject, candidate = @replay.select(subjects, candidates, scope: scope)
+      @replay_candidate = [candidate]
+      [[subject], nil]
+    end
+
     def supervisor(memory_factor: 1)
       limits = Execution::Supervisor::Limits.new(max_memory_mb: config["max_memory_mb"] * memory_factor,
                                                  max_processes: config["max_processes"])
@@ -108,10 +197,17 @@ module Saboteur
       end
     end
 
+    # Threads do not survive fork, so name them: with the name the owner
+    # gave the thread, else the first frame of its backtrace, so the user
+    # can tell whether they matter (a connection pool reaper, Sidekiq, an
+    # SDK's background flusher).
     def warn_about_threads
-      others = Thread.list.size - 1
-      @diagnostics.warn("#{others} other thread(s) alive at fork time; they do not run in children") if
-        others.positive?
+      others = Thread.list - [Thread.current]
+      return if others.empty?
+
+      names = others.map { |thread| thread.name || thread.backtrace&.first || "unknown" }
+      list = names.join(", ")
+      @diagnostics.warn("#{others.size} other thread(s) alive at fork time; they do not run in children: #{list}")
     end
 
     def monotonic
@@ -125,9 +221,13 @@ require_relative "engine/boot"
 require_relative "engine/subject_selection"
 require_relative "engine/suite_files"
 require_relative "engine/coverage_store"
+require_relative "engine/baseline_guard"
 require_relative "engine/candidates"
+require_relative "engine/candidate_selection"
 require_relative "engine/diff_scope"
 require_relative "engine/result_cache"
 require_relative "engine/verdict_check"
 require_relative "engine/scheduler"
 require_relative "engine/distribution"
+
+require_relative "engine/run_fingerprint"

data/lib/saboteur/execution/fork_readiness.rb

@@ -0,0 +1,46 @@
+# SPDX-License-Identifier: MIT
+# frozen_string_literal: true
+
+module Saboteur
+  module Execution
+    # Gets the heap ready at a fork boundary.
+    #
+    # Boot and candidate construction can leave garbage, fragmentation and
+    # young objects in the heap. Forked children initially share its pages.
+    # Later collection and object writes can make pages private. Preparing the
+    # heap before a fork batch can reduce that copying. Call it once per phase
+    # boundary, not per fork: preparing again while siblings are alive can
+    # break sharing with those siblings.
+    module ForkReadiness
+      # Process.warmup (CRuby 3.3+) is made for this: a full collection that
+      # also promotes survivors to the old generation and compacts. Without
+      # it, a lone GC.compact still packs the heap (it runs a full collection
+      # itself first) but does not promise old-generation promotion. Without
+      # either, a full collection is the best that Ruby can do.
+      def self.prepare_heap
+        return warmup_heap if Process.respond_to?(:warmup)
+        return compact_heap if GC.respond_to?(:compact)
+
+        full_collection
+      end
+
+      def self.warmup_heap
+        Process.warmup
+        nil
+      end
+      private_class_method :warmup_heap
+
+      def self.compact_heap
+        GC.compact
+        nil
+      end
+      private_class_method :compact_heap
+
+      def self.full_collection
+        GC.start(full_mark: true, immediate_sweep: true)
+        nil
+      end
+      private_class_method :full_collection
+    end
+  end
+end

data/lib/saboteur/execution/reevaluator.rb

@@ -29,11 +29,12 @@ module Saboteur
         target = subject.singleton ? owner.singleton_class : owner
         visibility = visibility_of(target, subject.name)
         module_function = module_function?(owner, subject)
+        copy_visibility = visibility_of(owner.singleton_class, subject.name) if module_function
         aliases = aliases_of(target, subject.name)
 
         @strategy == "nesting" ? eval_nested(subject, text) : eval_in_owner(owner, subject, text)
 
-        restore(owner, target, subject.name, visibility, module_function)
+        restore(owner, target, subject.name, visibility, copy_visibility)
         repoint(target, subject.name, aliases)
       end
 
@@ -82,7 +83,8 @@ module Saboteur
       def module_function?(owner, subject)
         return false if subject.singleton || owner.is_a?(Class)
         return false unless owner.private_method_defined?(subject.name, false)
-        return false unless owner.singleton_class.public_method_defined?(subject.name, false)
+        return false unless owner.singleton_class.method_defined?(subject.name, false) ||
+                            owner.singleton_class.private_method_defined?(subject.name, false)
 
         owner.instance_method(subject.name).source_location ==
           owner.singleton_class.instance_method(subject.name).source_location
@@ -132,9 +134,10 @@ module Saboteur
         end
       end
 
-      def restore(owner, target, name, visibility, module_function)
-        if module_function
+      def restore(owner, target, name, visibility, copy_visibility)
+        if copy_visibility
           owner.send(:module_function, name)
+          owner.singleton_class.send(copy_visibility, name)
         elsif visibility != :public
           target.send(visibility, name)
         end

data/lib/saboteur/execution/supervisor.rb

@@ -94,7 +94,10 @@ module Saboteur
       # Runs the block in a forked child with a pipe writer; the child exits
       # when the block returns. Returns a Result. cpu_seconds bounds every
       # descendant's CPU time (default: twice the deadline, at least 10 s).
-      def run(timeout:, cpu_seconds: nil, &block)
+      # on_tick is called from the parent's wait loop after the fork, about
+      # every poll, so callers can report progress without a thread alive
+      # at fork time.
+      def run(timeout:, cpu_seconds: nil, on_tick: nil, &block)
         depth = ENV.fetch(CHILD_ENV, "0").to_i
         raise Error, "refusing to fork: saboteur children nested #{depth} deep (limit #{MAX_DEPTH})" if
           depth >= MAX_DEPTH
@@ -107,7 +110,7 @@ module Saboteur
         supervisor = Process.pid
         pid = fork { child(reader, writer, supervisor, cpu_seconds, depth + 1, &block) }
         writer.close
-        contain(pid, reader, started, timeout)
+        contain(pid, reader, started, timeout, on_tick)
       ensure
         reader&.close unless reader&.closed?
       end
@@ -168,24 +171,31 @@ module Saboteur
 
       # --- parent ---------------------------------------------------------
 
-      def contain(pid, reader, started, timeout)
+      def contain(pid, reader, started, timeout, on_tick)
         tree = ProcessTree.new(pid)
         self.class.live[pid] = tree
         own_group(pid)
-        payload, stop, detail = watch(tree, reader, started + timeout)
-        exit_status = stop == :timeout ? terminate(pid) : nil
-        leftover_pids = tree.kill_all - [pid]
-        exit_status ||= reap(pid)
-        self.class.live.delete(pid)
-        reap_adopted(leftover_pids)
-        reap_group(pid)
+        begin
+          payload, stop, detail = watch(tree, reader, started + timeout, on_tick)
+        ensure
+          # Containment holds on every path out of watch: a raising on_tick
+          # (progress IO failed) or any other error must not skip the
+          # kill/reap, or the child and its tree would run unsupervised
+          # until this process exits.
+          exit_status = stop == :timeout ? terminate(pid) : nil
+          leftover_pids = tree.kill_all - [pid]
+          exit_status ||= reap(pid)
+          self.class.live.delete(pid)
+          reap_adopted(leftover_pids)
+          reap_group(pid)
+        end
         Result.new(payload: payload, exit_status: exit_status, stopped_by: stop, detail: detail,
                    runtime: monotonic - started, leftover_pids: leftover_pids)
       end
 
       # Reads until EOF, polling the tree's size and memory. Returns
       # [payload, stop, detail].
-      def watch(tree, reader, deadline)
+      def watch(tree, reader, deadline, on_tick)
         payload = +""
         next_poll = monotonic + POLL_INTERVAL
         tree.live_members
@@ -193,6 +203,8 @@ module Saboteur
           now = monotonic
           return [payload, :timeout, "exceeded deadline"] if now >= deadline
 
+          on_tick&.call
+
           if now >= next_poll
             stop = over_limit(tree)
             return [payload, *stop] if stop