saboteur 0.2.0 → 0.3.0

data/lib/saboteur/engine/run_fingerprint.rb

@@ -0,0 +1,316 @@
+# SPDX-License-Identifier: MIT
+# frozen_string_literal: true
+
+require "digest"
+require "json"
+
+module Saboteur
+  class Engine
+    # Read-only identity for declared project inputs, not a Ruby dependency graph.
+    # Capture before boot; verify before reuse/save. Unknown inputs are unsafe.
+    class RunFingerprint
+      FORMAT_VERSION = 1
+      MANIFESTS = %w[Gemfile gems.rb Gemfile.lock gems.locked .rspec .rspec-local].freeze
+      LOCATION_SETTINGS = %w[cache_dir json_report html_report].freeze
+      ENTRY_KINDS = %w[ruby manifest boot hook source_plugin extra].freeze
+      EXCLUDED_DIRECTORIES = %w[.git vendor node_modules].freeze
+      Entry = Data.define(:path, :kind, :state, :sha256, :required)
+      Change = Data.define(:path, :reason)
+      Snapshot = Data.define(:format_version, :digest, :inventory, :settings_digest, :runtime_digest, :complete,
+                             :issues) do
+        def to_h
+          super.merge(inventory: inventory.map(&:to_h))
+        end
+      end
+      Verification = Data.define(:unchanged, :complete, :current_digest, :changes, :issues)
+
+      def self.restore_snapshot(hash)
+        fields = canonical_mapping(hash).transform_keys(&:to_sym)
+        validate_snapshot_fields(fields)
+        inventory = fields.fetch(:inventory).map { |entry| restore_entry(entry) }.freeze
+        raise ArgumentError, "duplicate inventory paths" unless inventory.map(&:path).uniq.size == inventory.size
+
+        issues = restore_issues(fields.fetch(:issues))
+        validate_snapshot_identity(fields, inventory, issues)
+        Snapshot.new(**fields, inventory: inventory, issues: issues)
+      rescue KeyError, TypeError, NoMethodError => e
+        raise ArgumentError, "invalid fingerprint snapshot: #{e.class}"
+      end
+
+      def self.restore_issues(issues)
+        raise ArgumentError, "invalid issues" unless
+          issues.is_a?(Array) && issues.all?(String) && issues == issues.uniq.sort
+
+        issues.map { |issue| issue.dup.freeze }.freeze
+      end
+
+      def self.validate_snapshot_identity(fields, inventory, issues)
+        raise ArgumentError, "invalid complete snapshot" if
+          fields[:complete] && (issues.any? || inventory.any? do |entry|
+            entry.state == "unreadable" || (entry.required && entry.state == "missing")
+          end)
+        raise ArgumentError, "unsorted inventory" unless inventory.map(&:path) == inventory.map(&:path).sort
+
+        inputs = { format: FORMAT_VERSION, files: inventory.map(&:to_h),
+                   settings: fields[:settings_digest], runtime: fields[:runtime_digest] }
+        canonical = canonical_value(inputs)
+        raise ArgumentError, "fingerprint identity mismatch" unless
+          Digest::SHA256.hexdigest(JSON.generate(canonical)) == fields[:digest]
+      end
+
+      def self.validate_snapshot_fields(fields)
+        raise ArgumentError, "invalid snapshot fields" unless fields.keys.sort == Snapshot.members.sort
+        raise ArgumentError, "unsupported fingerprint format" unless fields[:format_version] == FORMAT_VERSION
+        raise ArgumentError, "invalid completeness" unless [true, false].include?(fields[:complete])
+
+        %i[digest settings_digest runtime_digest].each do |key|
+          raise ArgumentError, "invalid fingerprint digest" unless fields.fetch(key).match?(/\A[0-9a-f]{64}\z/)
+        end
+      end
+
+      def self.restore_entry(entry)
+        values = canonical_mapping(entry).transform_keys(&:to_sym)
+        raise ArgumentError, "invalid fingerprint entry" unless valid_entry?(values)
+
+        Entry.new(**values)
+      end
+
+      def self.valid_entry?(values)
+        values.keys.sort == %i[kind path required sha256 state] && valid_path?(values[:path]) &&
+          [true,
+           false].include?(values[:required]) && ENTRY_KINDS.include?(values[:kind]) &&
+          (!%w[ruby boot hook
+               source_plugin].include?(values[:kind]) || values[:required]) && valid_state?(values[:state],
+                                                                                            values[:sha256])
+      end
+
+      def self.valid_path?(path)
+        unless path.is_a?(String) && !path.empty? && !path.include?("\0") &&
+               !path.include?("\\") && !path.end_with?("/")
+          return false
+        end
+
+        candidate = path.delete_prefix("external:")
+        absolute = path.start_with?("external:")
+        return false unless candidate.start_with?("/") == absolute
+
+        segments = candidate.split("/")
+        segments.shift if absolute
+        segments.none? { |part| part.empty? || %w[. ..].include?(part) }
+      end
+
+      def self.valid_state?(state, digest)
+        return digest.is_a?(String) && digest.match?(/\A[0-9a-f]{64}\z/) if state == "present"
+
+        %w[missing unreadable].include?(state) && digest.nil?
+      end
+
+      def self.runtime_identity
+        { engine: RUBY_ENGINE, version: RUBY_VERSION, platform: RUBY_PLATFORM,
+          patchlevel: RUBY_PATCHLEVEL, saboteur: VERSION }
+      end
+
+      def initialize(config:, root: config.root, extra_files: [], runtime_identity: self.class.runtime_identity)
+        @config = config
+        @root = File.realpath(root)
+        raise ArgumentError, "root differs from config.root" unless @root == File.realpath(config.root)
+
+        @extra_files = extra_files.map do |file|
+          options = file.is_a?(Hash) ? file : { path: file, required: true }
+          [File.expand_path(options.fetch(:path), @root), options.fetch(:required, true)]
+        end.freeze
+        @runtime_identity = canonical(runtime_identity)
+      end
+
+      def snapshot
+        issues = output_overlap_issues
+        entries = inventory(issues).sort_by(&:path).freeze
+        issues = issues.uniq.sort
+        settings_digest = hash_value(@config.to_h.except(*LOCATION_SETTINGS))
+        runtime_digest = hash_value(@runtime_identity)
+        digest = hash_value(format: FORMAT_VERSION, files: entries.map(&:to_h),
+                            settings: settings_digest, runtime: runtime_digest)
+        Snapshot.new(format_version: FORMAT_VERSION, digest: digest, inventory: entries,
+                     settings_digest: settings_digest, runtime_digest: runtime_digest,
+                     complete: issues.empty?, issues: issues.freeze)
+      end
+
+      def verify(previous)
+        current = snapshot
+        changes = changes_between(previous, current)
+        complete = previous.complete && current.complete && previous.format_version == FORMAT_VERSION
+        Verification.new(unchanged: complete && previous.digest == current.digest,
+                         complete: complete, current_digest: current.digest,
+                         changes: changes.freeze, issues: current.issues)
+      end
+
+      def self.canonical_value(value)
+        case value
+        when Hash then canonical_mapping(value)
+        when Array then value.map { |entry| canonical_value(entry) }.freeze
+        when String then value.dup.freeze
+        when NilClass, TrueClass, FalseClass, Integer then value
+        when Float
+          raise ArgumentError, "non-finite identity number" unless value.finite?
+
+          value
+        else raise ArgumentError, "unsupported identity value"
+        end
+      end
+
+      def self.canonical_mapping(value)
+        raise ArgumentError, "invalid identity key" unless value.keys.all? do |key|
+          key.is_a?(String) || key.is_a?(Symbol)
+        end
+        raise ArgumentError, "duplicate normalized identity key" unless value.keys.map(&:to_s).uniq.size == value.size
+
+        value.keys.sort_by(&:to_s).to_h { |key| [key.to_s, canonical_value(value[key])] }.freeze
+      end
+
+      private
+
+      def canonical_path(path)
+        expanded = File.expand_path(path, @root)
+        return File.realpath(expanded) if File.exist?(expanded)
+
+        parent = File.dirname(expanded)
+        File.join(canonical_path(parent), File.basename(expanded))
+      end
+
+      def output_overlap_issues
+        paths = required_files.map(&:first) + MANIFESTS.map { |name| File.join(@root, name) }
+        outputs = %w[json_report html_report].filter_map { |key| @config.fetch(key) }
+                                             .map { |path| canonical_path(path) }
+        issues = outputs.filter_map do |path|
+          if path.end_with?(".rb") || paths.include?(path) || File.directory?(path)
+            "output overlaps input: #{display_path(path)}"
+          end
+        end
+        issues.concat(cache_overlap_issues(paths))
+        issues
+      end
+
+      def cache_overlap_issues(paths)
+        cache = canonical_path(@config["cache_dir"])
+        protected = [@root] + %w[app lib spec].map { |name| File.join(@root, name) } + paths
+        overlaps = protected.any? do |path|
+          path == cache || path.start_with?("#{cache}/") || (path != @root && cache.start_with?("#{path}/"))
+        end
+        overlaps ? ["cache directory overlaps inputs: #{display_path(cache)}"] : []
+      end
+
+      def inventory(issues)
+        paths = project_files(issues).to_h { |path| [path, "ruby"] }
+        MANIFESTS.each { |name| paths[File.join(@root, name)] = "manifest" }
+        required = merged_required_files
+        required.each { |path, (kind, _)| paths[path] = kind }
+        paths.map do |path, kind|
+          needed = required.key?(path) ? required[path].last : kind != "manifest"
+          read_entry(path, kind, needed, issues)
+        end
+      end
+
+      def merged_required_files
+        required_files.group_by(&:first).transform_values do |entries|
+          roles = entries.map { |_, kind, _| kind }
+          role = (roles - ["extra"]).min || "extra"
+          [role, entries.any? { |_, _, needed| needed }]
+        end
+      end
+
+      # Walk without following symlink directories. A project symlink is an
+      # undeclared dependency boundary, not a silently omitted source tree.
+      def project_files(issues, directory = @root)
+        Dir.children(directory).sort.flat_map do |name|
+          path = File.join(directory, name)
+          next [] if excluded?(path, name)
+
+          if File.symlink?(path)
+            issues << "symlink input: #{display_path(path)}".freeze
+            next []
+          end
+          next project_files(issues, path) if File.directory?(path)
+
+          File.file?(path) && path.end_with?(".rb") ? [path] : []
+        end
+      rescue SystemCallError
+        issues << "unreadable directory: #{display_path(directory)}".freeze
+        []
+      end
+
+      def excluded?(path, name)
+        EXCLUDED_DIRECTORIES.include?(name) || path == canonical_path(@config["cache_dir"]) ||
+          %w[json_report html_report].filter_map { |key| @config.fetch(key) }.any? do |output|
+            path == canonical_path(output)
+          end
+      end
+
+      def required_files
+        paths = %w[require hooks].flat_map do |key|
+          @config[key].map { |file| [resolve(file), key == "hooks" ? "hook" : "boot", true] }
+        end
+        paths += @config["sources"].reject { |entry| entry == "builtin" }.map do |entry|
+          [resolve(entry.partition(":").first), "source_plugin", true]
+        end
+        paths + @extra_files.map { |path, needed| [path, "extra", needed] }
+      end
+
+      def resolve(file)
+        path = File.expand_path(file, @root)
+        return path if File.exist?(path) || File.extname(path) == ".rb"
+        return "#{path}.rb" if File.exist?("#{path}.rb")
+
+        path
+      end
+
+      def read_entry(path, kind, required, issues)
+        name = display_path(path).freeze
+        unless File.exist?(path)
+          issues << "missing required input: #{name}".freeze if required
+          return Entry.new(path: name, kind: kind.freeze, state: "missing", sha256: nil, required: required)
+        end
+        Entry.new(path: name, kind: kind.freeze, state: "present", sha256: Digest::SHA256.file(path).hexdigest.freeze,
+                  required: required)
+      rescue SystemCallError, IOError
+        issues << "unreadable input: #{name}".freeze
+        Entry.new(path: name, kind: kind.freeze, state: "unreadable", sha256: nil, required: required)
+      end
+
+      def display_path(path)
+        path.start_with?("#{@root}/") ? path.delete_prefix("#{@root}/") : "external:#{path}"
+      end
+
+      def changes_between(previous, current)
+        before = previous.inventory.to_h { |entry| [entry.path, entry] }
+        after = current.inventory.to_h { |entry| [entry.path, entry] }
+        changes = (before.keys | after.keys).sort.filter_map do |path|
+          next if before[path] == after[path]
+
+          Change.new(path: path.freeze, reason: change_reason(before[path], after[path]))
+        end
+        if previous.settings_digest != current.settings_digest
+          changes << Change.new(path: "settings",
+                                reason: "changed")
+        end
+        changes << Change.new(path: "runtime", reason: "changed") if previous.runtime_digest != current.runtime_digest
+        changes
+      end
+
+      def change_reason(before, after)
+        return "added" unless before
+        return "removed" unless after
+        return "unreadable" if after.state == "unreadable"
+
+        "changed"
+      end
+
+      def canonical(value)
+        self.class.canonical_value(value)
+      end
+
+      def hash_value(value)
+        Digest::SHA256.hexdigest(JSON.generate(canonical(value))).freeze
+      end
+    end
+  end
+end

data/lib/saboteur/engine/verdict_check.rb

@@ -63,12 +63,14 @@ module Saboteur
       end
 
       def checked(result, failing_runs, failing)
+        return result if result.reused?
+
         if named_kill?(result) && failing_runs.key?(sequence(result))
           why = failing_runs[sequence(result)]
           result.as_error("killing test #{result.killed_by.first} also fails with the method unchanged" \
-                          "#{": #{why}" if why}")
+                          "#{": #{why}" if why}", rejected_kill: true)
         elsif failing.include?(result)
-          result.as_error("tests also fail with the method unchanged")
+          result.as_error("tests also fail with the method unchanged", rejected_kill: true)
         else
           result
         end

data/lib/saboteur/engine.rb

@@ -3,29 +3,32 @@
 
 require "fileutils"
 require "json"
+require_relative "replay"
 
 module Saboteur
-  # One run, as a pipeline: boot the target, select subjects, fetch the
+  # A single-use instance runs one pipeline: boot the target, select subjects, fetch the
   # coverage map, then schedule each subject's mutants. The parts live in
   # engine/; this class only wires them together.
   class Engine
     # Everything besides sources and tests that can change a verdict.
     VERDICT_INPUTS = %w[integration require rspec_args rails reeval sources arid timeout_factor timeout_min
-                        max_memory_mb max_processes seed].freeze
+                        max_memory_mb max_processes seed tests].freeze
 
     attr_reader :config, :hooks
 
-    def initialize(config, err: $stderr)
+    def initialize(config, err: $stderr, digest: nil)
       @config = config
       @hooks = Hooks.new
       @diagnostics = Diagnostics.new(err)
       @root = File.realpath(config.root)
+      @skipped_subjects = {}
+      @replay = Replay.new(digest: digest) if digest
     end
 
     def call
       Boot.refuse_recursion!
       started = monotonic
-      ignore_list = IgnoreList.load(config.path("ignore_file"))
+      ignore_list = @replay ? IgnoreList.new([]) : IgnoreList.load(config.path("ignore_file"))
       boot = Boot.new(config, hooks)
       integration = boot.load_target
       @rails = boot.rails
@@ -35,7 +38,7 @@ module Saboteur
       @diagnostics.warnings.concat(ignore_list.apply(results))
       Report.new(results: results, subjects: subjects, wall_time: monotonic - started, root: @root,
                  invalid_mutations: @diagnostics.invalid_mutations, warnings: @diagnostics.warnings,
-                 review: !scope.nil?, seed: config["seed"])
+                 review: !scope.nil?, seed: config["seed"], skipped_subjects: @skipped_subjects)
     end
 
     private
@@ -45,24 +48,45 @@ module Saboteur
       config.unmatched_patterns.each { |pattern| @diagnostics.warn("#{pattern} matches no file") }
       selection = SubjectSelection.new(config, @diagnostics)
       subjects = selection.select(files)
-      [subjects, selection.changed_lines && DiffScope.new(selection.changed_lines)]
+      scope = selection.changed_lines && DiffScope.new(selection.changed_lines)
+      return replay_subject(subjects, scope) if @replay
+
+      [subjects, scope]
     end
 
     def mutate(subjects, files, integration, scope)
+      check_baseline(integration)
       map = CoverageStore.new(config: config, integration: integration, hooks: hooks,
                               supervisor: supervisor(memory_factor: 4), diagnostics: @diagnostics).fetch(files)
-      @diagnostics.inform("#{subjects.size} subjects, #{map.tests.size} tests in coverage map")
-      warn_about_threads
+      inform_about_run(subjects, map)
       cache = result_cache
       scheduler = scheduler_for(map, integration, cache)
+      lists = candidate_lists(subjects, scope, map)
+      work = subjects.zip(lists).reject { |_, list| list.empty? }
+      return [] if work.empty?
+
+      runnable_subjects, runnable_lists = work.transpose
       @rails&.prepare_workers(config["jobs"])
+      # The execution phase forks next: the worker batch, or one child per
+      # mutant when serial. Pack the heap now that the map and candidates
+      # are retained, to reduce later copying of shared pages.
+      Execution::ForkReadiness.prepare_heap
       results = Distribution.new(jobs: config["jobs"], scheduler: scheduler, diagnostics: @diagnostics,
-                                 cache: cache, on_start: method(:start_worker)).run(subjects, candidate_lists(subjects,
-                                                                                                              scope))
+                                 cache: cache, on_start: method(:start_worker)).run(runnable_subjects, runnable_lists)
       cache&.save
       results
     end
 
+    def inform_about_run(subjects, map)
+      @diagnostics.inform("#{subjects.size} subjects, #{map.tests.size} tests in coverage map")
+      warn_about_threads
+    end
+
+    def check_baseline(integration)
+      BaselineGuard.new(config: config, integration: integration, hooks: hooks,
+                        supervisor: supervisor(memory_factor: 4), diagnostics: @diagnostics).call
+    end
+
     def scheduler_for(map, integration, cache)
       reevaluator = Execution::Reevaluator.new(config["reeval"])
       runner = Execution::Runner.new(integration: integration, reevaluator: reevaluator, hooks: hooks,
@@ -72,6 +96,7 @@ module Saboteur
     end
 
     def result_cache
+      return nil if @replay
       return nil unless config["incremental"]
 
       inputs = JSON.generate(VERDICT_INPUTS.to_h { |key| [key, config[key]] })
@@ -86,14 +111,24 @@ module Saboteur
     end
 
     # Each subject's candidates, narrowed to changed lines in diff mode.
-    def candidate_lists(subjects, scope)
+    def candidate_lists(subjects, scope, map)
+      return [@replay_candidate] if @replay
+
       candidates = Candidates.new(Candidates.build_sources(config), @diagnostics)
-      subjects.map do |subject|
-        list = candidates.for(subject)
-        scope ? scope.restrict(subject, list) : list
+      selection = CandidateSelection.new(candidates: candidates, map: map,
+                                         limit: config["max_mutants_per_method"], diagnostics: @diagnostics)
+      subjects.map { |subject| selection.for(subject, scope: scope) }.tap do
+        @skipped_subjects = selection.skipped_subjects
       end
     end
 
+    def replay_subject(subjects, scope)
+      candidates = Candidates.new(Candidates.build_sources(config), @diagnostics)
+      subject, candidate = @replay.select(subjects, candidates, scope: scope)
+      @replay_candidate = [candidate]
+      [[subject], nil]
+    end
+
     def supervisor(memory_factor: 1)
       limits = Execution::Supervisor::Limits.new(max_memory_mb: config["max_memory_mb"] * memory_factor,
                                                  max_processes: config["max_processes"])
@@ -125,9 +160,13 @@ require_relative "engine/boot"
 require_relative "engine/subject_selection"
 require_relative "engine/suite_files"
 require_relative "engine/coverage_store"
+require_relative "engine/baseline_guard"
 require_relative "engine/candidates"
+require_relative "engine/candidate_selection"
 require_relative "engine/diff_scope"
 require_relative "engine/result_cache"
 require_relative "engine/verdict_check"
 require_relative "engine/scheduler"
 require_relative "engine/distribution"
+
+require_relative "engine/run_fingerprint"

data/lib/saboteur/execution/fork_readiness.rb

@@ -0,0 +1,46 @@
+# SPDX-License-Identifier: MIT
+# frozen_string_literal: true
+
+module Saboteur
+  module Execution
+    # Gets the heap ready at a fork boundary.
+    #
+    # Boot and candidate construction can leave garbage, fragmentation and
+    # young objects in the heap. Forked children initially share its pages.
+    # Later collection and object writes can make pages private. Preparing the
+    # heap before a fork batch can reduce that copying. Call it once per phase
+    # boundary, not per fork: preparing again while siblings are alive can
+    # break sharing with those siblings.
+    module ForkReadiness
+      # Process.warmup (CRuby 3.3+) is made for this: a full collection that
+      # also promotes survivors to the old generation and compacts. Without
+      # it, a lone GC.compact still packs the heap (it runs a full collection
+      # itself first) but does not promise old-generation promotion. Without
+      # either, a full collection is the best that Ruby can do.
+      def self.prepare_heap
+        return warmup_heap if Process.respond_to?(:warmup)
+        return compact_heap if GC.respond_to?(:compact)
+
+        full_collection
+      end
+
+      def self.warmup_heap
+        Process.warmup
+        nil
+      end
+      private_class_method :warmup_heap
+
+      def self.compact_heap
+        GC.compact
+        nil
+      end
+      private_class_method :compact_heap
+
+      def self.full_collection
+        GC.start(full_mark: true, immediate_sweep: true)
+        nil
+      end
+      private_class_method :full_collection
+    end
+  end
+end