saboteur 0.4.0 → 0.5.0

data/lib/saboteur/engine.rb

@@ -4,6 +4,8 @@
 require "fileutils"
 require "json"
 require_relative "replay"
+require_relative "mutation_source/hazard_guard"
+require_relative "baseline/run_evidence"
 
 module Saboteur
   # A single-use instance runs one pipeline: boot the target, select subjects, fetch the
@@ -11,32 +13,43 @@ module Saboteur
   # engine/; this class only wires them together.
   class Engine
     # Everything besides sources and tests that can change a verdict.
-    VERDICT_INPUTS = %w[integration require rspec_args rails reeval sources arid timeout_factor timeout_min
-                        max_memory_mb max_processes seed tests].freeze
+    VERDICT_INPUTS = %w[integration require rspec_args rails reeval sources operators arid timeout_factor timeout_min
+                        max_memory_mb max_processes seed tests whole_file].freeze
 
     attr_reader :config, :hooks
+    # The specs --guess-tests narrowed the run to, or nil when it did not.
+    attr_reader :guessed_tests
 
-    def initialize(config, err: $stderr, digest: nil)
+    def initialize(config, err: $stderr, digest: nil, capture_evidence: false, evidence_files: [])
       @config = config
       @hooks = Hooks.new
       @diagnostics = Diagnostics.new(err)
       @root = File.realpath(config.root)
       @skipped_subjects = {}
       @replay = Replay.new(digest: digest) if digest
+      if capture_evidence && (digest || config["since"] || config["incremental"])
+        raise ConfigError, "baseline evidence requires fresh whole-scope execution: no since, replay or incremental"
+      end
+
+      @capture_evidence = capture_evidence
+      @evidence_files = evidence_files
+      @stable_suite = false
     end
 
     def call
       Boot.refuse_recursion!
+      refuse_fail_fast_evidence
       started = monotonic
+      guess_tests
+      @run_evidence = Baseline::RunEvidence.new(config: config, extra_files: @evidence_files) if @capture_evidence
       ignore_list = @replay ? IgnoreList.new([]) : IgnoreList.load(config.path("ignore_file"))
-      @diagnostics.warnings.concat(custom_map_warnings(load_custom_map))
+      warn_custom_map
       boot = Boot.new(config, hooks)
       begin
-        integration = boot.load_target
+        integration, files, load_lines = boot_target(boot)
         @rails = boot.rails
-        files = config.included_files
         subjects, scope = select_subjects(files)
-        results = subjects.empty? ? [] : mutate(subjects, files, integration, scope)
+        results = subjects.empty? ? [] : mutate(subjects, files, integration, scope, ignore_list, load_lines:)
         @diagnostics.warnings.concat(ignore_list.apply(results))
         build_report(results, subjects, scope, started)
       ensure
@@ -51,6 +64,10 @@ module Saboteur
       end
     end
 
+    def warn_custom_map
+      @diagnostics.warnings.concat(custom_map_warnings(load_custom_map))
+    end
+
     # Stale entries and duplicate literals never fail a run; they warn so the
     # map does not rot silently. doctor fails on stale keys.
     def load_custom_map
@@ -67,12 +84,50 @@ module Saboteur
 
     private
 
+    # --guess-tests with --since: scope the run to the specs found next to
+    # the changed files, as --test would, but only when each changed file
+    # has one. Otherwise the whole suite runs, as without the flag.
+    def guess_tests
+      return unless config["guess_tests"]
+
+      guess = TestGuess.new(config).call
+      message =
+        if guess.changed.zero?
+          "--guess-tests: no changed source file to guess specs for, so the whole suite runs"
+        elsif guess.unmatched.any?
+          "--guess-tests: no spec found for #{guess.unmatched.join(', ')}, so the whole suite runs " \
+            "(name specs after their source files or pass --test)"
+        else
+          # The guesses now stand as explicit tests, like --test.
+          @config = Config.new(config.to_h.merge("tests" => guess.specs, "guess_tests" => false), root: config.root)
+          @guessed_tests = guess.specs
+          "--guess-tests: running only #{guess.specs.join(', ')}. A spec that reaches a changed file only " \
+            "through another file does not run, so a mutant only it would kill can survive, and a line " \
+            "no guessed spec executes is reported as NoCoverage"
+        end
+      @diagnostics.warn(message)
+      @diagnostics.inform(message)
+    end
+
     # The run's immutable report, built once the last result is in.
     def build_report(results, subjects, scope, started)
       Report.new(results: results, subjects: subjects, wall_time: monotonic - started, root: @root,
                  invalid_mutations: @diagnostics.invalid_mutations, warnings: @diagnostics.warnings,
                  review: !scope.nil?, seed: config["seed"], skipped_subjects: @skipped_subjects,
-                 diff_scope: scope)
+                 diff_scope: scope, generation_events: @diagnostics.generation_events,
+                 omissions: @diagnostics.omissions, run_evidence: complete_evidence(results, subjects),
+                 hazard_suppressions: @diagnostics.hazard_suppressions, early_stop: @early_stop,
+                 matrix: config["matrix"], inline_suppressions: @diagnostics.inline_suppressions)
+    end
+
+    def complete_evidence(results, subjects)
+      return nil unless @run_evidence
+
+      @run_evidence.plan(subjects, subjects.map { [] }) unless @evidence_planned
+      @run_evidence.finish(results: results, stable_suite: @stable_suite, skipped_subjects: @skipped_subjects,
+                           generation_events: @diagnostics.generation_events, omissions: @diagnostics.omissions,
+                           discovery_failures: @diagnostics.discovery_failures,
+                           inline_suppressions: @diagnostics.inline_suppressions)
     end
 
     # The subjects to mutate and, in diff mode, the scope of changed lines.
@@ -86,10 +141,10 @@ module Saboteur
       [subjects, scope]
     end
 
-    def mutate(subjects, files, integration, scope)
+    def mutate(subjects, files, integration, scope, ignore_list, load_lines: {})
       warn_about_unscoped_suite(integration)
       check_baseline(integration)
-      map = CoverageStore.new(config: config, integration: integration, hooks: hooks,
+      map = CoverageStore.new(config: config, integration: integration, hooks: hooks, load_lines:,
                               supervisor: supervisor(memory_factor: 4), diagnostics: @diagnostics).fetch(files)
       inform_about_run(subjects, map)
       cache = result_cache
@@ -104,13 +159,49 @@ module Saboteur
       # mutant when serial. Pack the heap now that the map and candidates
       # are retained, to reduce later copying of shared pages.
       Execution::ForkReadiness.prepare_heap
-      results = Distribution.new(jobs: config["jobs"], scheduler: scheduler, diagnostics: @diagnostics,
-                                 cache: cache, on_start: method(:start_worker)).run(runnable_subjects,
-                                                                                    runnable_lists)
+      results = run_candidates(runnable_subjects, runnable_lists, scheduler, cache, ignore_list)
       cache&.save
       results
     end
 
+    def run_candidates(subjects, lists, scheduler, cache, ignore_list)
+      @diagnostics.start_progress(lists.sum(&:size))
+      distribution = Distribution.new(jobs: config["jobs"], scheduler: scheduler, diagnostics: @diagnostics,
+                                      cache: cache, on_start: method(:start_worker),
+                                      after_subject: canary_check, stop_after: fail_fast_check(ignore_list))
+      distribution.run(subjects, lists).tap { record_early_stop(distribution.unreached) }
+    ensure
+      @diagnostics.finish_progress
+    end
+
+    # Baseline evidence certifies a whole selection; a run that may stop early
+    # cannot.
+    def refuse_fail_fast_evidence
+      return unless @capture_evidence && config["fail_fast"]
+
+      raise ConfigError, "fail_fast cannot be used with baseline evidence: it certifies the whole selection"
+    end
+
+    # Subjects that were selected but never ran, with candidates, are part of
+    # the report: the verdict covers less than the selection.
+    def record_early_stop(unreached)
+      return if unreached.empty?
+
+      @early_stop = EarlyStop.new(reason: "fail_fast", unreached: unreached.map(&:id))
+    end
+
+    # --fail-fast: stop after the first subject that has a survivor the
+    # ignore list does not accept. Granularity is one subject.
+    def fail_fast_check(ignore_list)
+      return nil unless config["fail_fast"]
+
+      lambda do |results|
+        survivor = results.any? { |result| result.survived? && !ignore_list.accepts?(result) }
+        @diagnostics.inform("--fail-fast: stopping after the first subject with a survivor") if survivor
+        survivor
+      end
+    end
+
     def inform_about_run(subjects, map)
       @diagnostics.inform("#{subjects.size} subjects, #{map.tests.size} tests in coverage map")
       warn_about_threads
@@ -123,7 +214,11 @@ module Saboteur
       return unless config["since"] && config["tests"].empty?
 
       message = "running the loaded suite unscoped: two baseline passes; coverage may add a pass " \
-                "(#{integration.test_files.size} spec files). Pass --test to scope it."
+                "(#{integration.test_files.size} spec files). " \
+                "Pass --test to run only the specs you name: faster, but a mutant of a changed method is then killed " \
+                "only by those specs, so specs elsewhere that call this code are not run. " \
+                "--guess-tests does the same with the specs named after the changed files, " \
+                "a guess that can miss specs elsewhere."
       @diagnostics.warn(message)
       @diagnostics.inform(message)
     end
@@ -131,16 +226,25 @@ module Saboteur
     def check_baseline(integration)
       BaselineGuard.new(config: config, integration: integration, hooks: hooks,
                         supervisor: supervisor(memory_factor: 4), diagnostics: @diagnostics).call
+      @stable_suite = true
     end
 
     def scheduler_for(map, integration, cache)
-      reevaluator = Execution::Reevaluator.new(config["reeval"])
+      reevaluator = Execution::Reevaluator.new(config["reeval"], whole_file: config["whole_file"])
       runner = Execution::Runner.new(integration: integration, reevaluator: reevaluator, hooks: hooks,
-                                     supervisor: supervisor, log_dir: fresh_log_dir, seed: config["seed"])
+                                     supervisor: supervisor, log_dir: fresh_log_dir, seed: config["seed"],
+                                     sandbox: sandbox_hook)
       Scheduler.new(config: config, runner: runner, map: map, reevaluator: reevaluator, diagnostics: @diagnostics,
                     cache: cache)
     end
 
+    # The sandbox hook for each mutant child (#152); the policy and its
+    # one-time note live in Engine::SandboxPolicy.
+    def sandbox_hook
+      @sandbox_policy ||= SandboxPolicy.new(config: config, root: @root, diagnostics: @diagnostics)
+      @sandbox_policy.hook
+    end
+
     def result_cache
       return nil if @replay
       return nil unless config["incremental"]
@@ -165,19 +269,43 @@ module Saboteur
     end
 
     # Each subject's candidates, narrowed to changed lines in diff mode.
+    # The hazard guard is off only when the user explicitly asks for the
+    # dangerous mutants (issue #102); the canary then still watches the tree.
+    def hazard_guard
+      return nil if config["allow_hazardous"]
+
+      MutationSource::HazardGuard.new(config["hazard"])
+    end
+
+    # Checks the tree after each subject's mutants; a vanished file means a
+    # mutant deleted it, so abort with the most recent mutant's digest.
+    def canary_check
+      canary = Canary.new(@root, cache_dir: config["cache_dir"])
+      lambda do |subject, results|
+        last = results.reject(&:reused?).last || results.last
+        digest = last&.mutation&.digest(subject_start: subject.start_byte,
+                                        original_source: subject.file_source)
+        canary.check!(subject: subject, digest: digest)
+      end
+    end
+
     def candidate_lists(subjects, scope, map)
       return [@replay_candidate] if @replay
 
-      candidates = Candidates.new(Candidates.build_sources(config), @diagnostics)
+      candidates = Candidates.new(Candidates.build_sources(config), @diagnostics, hazard_guard: hazard_guard)
       selection = CandidateSelection.new(candidates: candidates, map: map,
                                          limit: config["max_mutants_per_method"], diagnostics: @diagnostics)
       subjects.map { |subject| selection.for(subject, scope: scope) }.tap do
         @skipped_subjects = selection.skipped_subjects
+        if @run_evidence
+          @run_evidence.plan(subjects, subjects.map { |subject| selection.selected.fetch(subject) })
+          @evidence_planned = true
+        end
       end
     end
 
     def replay_subject(subjects, scope)
-      candidates = Candidates.new(Candidates.build_sources(config), @diagnostics)
+      candidates = Candidates.new(Candidates.build_sources(config), @diagnostics, hazard_guard: hazard_guard)
       subject, candidate = @replay.select(subjects, candidates, scope: scope)
       @replay_candidate = [candidate]
       [[subject], nil]
@@ -210,6 +338,47 @@ module Saboteur
       @diagnostics.warn("#{others.size} other thread(s) alive at fork time; they do not run in children: #{list}")
     end
 
+    # The lines that ran while the app and spec files loaded: the executable
+    # counts from its first line, so this snapshot holds the whole boot.
+    # Read here, in the parent: a forked child does not inherit Coverage
+    # counters. They become the map's load lines, which tell ran-at-load
+    # mutants apart from uncovered ones.
+    def capture_load_lines(files)
+      return {} unless Coverage.state == :suspended
+
+      wanted = files.to_set { |file| Selection::CoverageMap.canonical(file) }
+      Coverage.result(stop: false, clear: false).each_with_object({}) do |(path, data), hits|
+        file = Selection::CoverageMap.canonical(path)
+        next unless wanted.include?(file)
+
+        lines = data[:lines].each_with_index.filter_map { |count, index| index + 1 if count&.positive? }
+        lines -= uncalled_shared_lines(data)
+        hits[file] = lines unless lines.empty?
+      end
+    end
+
+    # A one-line or endless definition executes its def line even when its
+    # body is never called. If any sibling on that physical line is uncalled,
+    # discard its ambiguous line evidence rather than crediting every body.
+    def uncalled_shared_lines(data)
+      shared = data.fetch(:methods, {}).select { |(_owner, _name, first, _column, last, *), _| first == last }
+      calls = shared.group_by { |(_owner, _name, first, *), _count| first }
+      calls.filter_map { |line, entries| line if entries.any? { |_key, count| count.zero? } }
+    end
+
+    # Loads the target and stops coverage, returning what the rest of the
+    # run needs from that boot.
+    def boot_target(boot)
+      integration = boot.load_target
+      files = config.included_files
+      [integration, files, suspend_coverage(files)]
+    end
+
+    def suspend_coverage(files)
+      Coverage.suspend if Coverage.state == :running
+      capture_load_lines(files)
+    end
+
     def monotonic
       Process.clock_gettime(Process::CLOCK_MONOTONIC)
     end
@@ -219,15 +388,20 @@ end
 require_relative "engine/diagnostics"
 require_relative "engine/boot"
 require_relative "engine/subject_selection"
+require_relative "engine/unchanged_syntax"
 require_relative "engine/suite_files"
+require_relative "engine/test_guess"
+require_relative "engine/sandbox_policy"
 require_relative "engine/coverage_store"
 require_relative "engine/baseline_guard"
+require_relative "engine/bytecode_equivalence"
 require_relative "engine/candidates"
 require_relative "engine/candidate_selection"
 require_relative "engine/diff_scope"
 require_relative "engine/result_cache"
 require_relative "engine/verdict_check"
 require_relative "engine/scheduler"
+require_relative "engine/canary"
 require_relative "engine/distribution"
 
 require_relative "engine/run_fingerprint"

data/lib/saboteur/execution/protocol.rb

@@ -8,9 +8,10 @@ module Saboteur
     # What a mutant child tells its supervisor, one JSON object per line on
     # a pipe. Every message carries the protocol version.
     #
+    #   captures a stale Method capture of the original def exists (site, stale)
     #   started  the mutated method is in place; tests are about to run
     #   result   passed, failed (ids), executed (ids), runtime, exited (status
-    #            when a test called exit)
+    #            when a test called exit), replacement_called (false only after a verified trace)
     #   error    message, backtrace: the harness failed, not the tests
     #
     # A child that dies after "started" without "result" crashed while its

data/lib/saboteur/execution/reevaluator.rb

@@ -12,19 +12,60 @@ module Saboteur
     #   nesting    - rebuild the original `module A; class B` nesting on one line
     #                and eval at top level, so constant lookup matches the file.
     #
+    # A subject named in whole_file skips both: the child evaluates the whole
+    # file with the mutated method spliced in, the way mutineer does. That is
+    # the way in for methods surgical re-evaluation refuses, at a price: the
+    # file's top-level code runs again in the child, so load-time side effects
+    # repeat and constants it assigns are rebuilt. It reaches only methods
+    # SubjectFinder already finds.
+    #
     # Both keep the original file and line numbers, carry a frozen_string_literal
     # magic comment, and restore the method's visibility afterwards.
     class Reevaluator
       STRATEGIES = %w[class_eval nesting].freeze
       FROZEN_HEADER = "# frozen_string_literal: true\n"
 
-      def initialize(strategy = "nesting")
+      def initialize(strategy = "nesting", whole_file: [])
         raise Error, "unknown reeval strategy: #{strategy}" unless STRATEGIES.include?(strategy.to_s)
 
         @strategy = strategy.to_s
+        @whole_file = whole_file
+      end
+
+      # "file:line" of another def in the subject's own file that replaced it:
+      # a later definition there won, so the mutant could never run. Only a
+      # positively identified different def counts. nil when the subject is not
+      # whole-file, its def is the one in place, or this cannot be told (a
+      # method from another file, such as an unknown forwarding wrapper, no
+      # Ruby source, or no method): the caller then keeps the verdict.
+      def replaced_definition(subject)
+        return nil unless whole_file?(subject)
+
+        owner = resolve_owner(subject)
+        target = subject.singleton ? owner.singleton_class : owner
+        method = target.instance_method(subject.name)
+        # A def-time wrapper (a sorbet sig) replaces the method; its own
+        # source_location is the wrapper's, not the def's.
+        file, line = (Wrapper.original_def(method) || method).source_location
+        return nil unless file == subject.file && line != subject.start_line
+
+        "#{file}:#{line}"
+      rescue StandardError
+        nil
       end
 
-      def call(subject, text)
+      # True when the subject's id matches a whole_file entry (exact or a
+      # File.fnmatch glob).
+      def whole_file?(subject)
+        @whole_file.any? { |pattern| File.fnmatch?(pattern, subject.id) }
+      end
+
+      # prefix: source evaluated right before the def text, at the lines it
+      # occupies in the file - a sorbet `sig { ... }` call, so the wrapper
+      # gem re-wraps the mutant and its runtime checks stay on.
+      def call(subject, text, prefix: nil)
+        return eval_whole_file(subject, text) if whole_file?(subject)
+
         owner = resolve_owner(subject)
         target = subject.singleton ? owner.singleton_class : owner
         visibility = visibility_of(target, subject.name)
@@ -32,7 +73,13 @@ module Saboteur
         copy_visibility = visibility_of(owner.singleton_class, subject.name) if module_function
         aliases = aliases_of(target, subject.name)
 
-        @strategy == "nesting" ? eval_nested(subject, text) : eval_in_owner(owner, subject, text)
+        extra_lines = prefix ? prefix.count("\n") + 1 : 0
+        full_text = prefix ? "#{prefix}\n#{text}" : text
+        if @strategy == "nesting"
+          eval_nested(subject, full_text, extra_lines)
+        else
+          eval_in_owner(owner, subject, full_text, extra_lines)
+        end
 
         restore(owner, target, subject.name, visibility, copy_visibility)
         repoint(target, subject.name, aliases)
@@ -90,23 +137,38 @@ module Saboteur
           owner.singleton_class.instance_method(subject.name).source_location
       end
 
-      def eval_in_owner(owner, subject, text)
-        code, line = with_magic_comment(subject, text)
+      # The file as it is on disk, with the method's bytes replaced by text,
+      # evaluated at top level under the file's own name and first line.
+      # Redefinition warnings are expected and silenced.
+      def eval_whole_file(subject, text)
+        source = subject.file_source.dup.force_encoding(Encoding::UTF_8)
+        spliced = source.byteslice(0, subject.start_byte) + text + source.byteslice(subject.end_byte..)
+        verbose = $VERBOSE
+        $VERBOSE = nil
+        TOPLEVEL_BINDING.eval(spliced, subject.file, 1)
+      ensure
+        $VERBOSE = verbose
+      end
+
+      def eval_in_owner(owner, subject, text, extra_lines)
+        code, line = with_magic_comment(subject, text, extra_lines)
         eval_target = subject.sclass ? owner.singleton_class : owner
         eval_target.class_eval(code, subject.file, line)
       end
 
-      def eval_nested(subject, text)
+      def eval_nested(subject, text, extra_lines)
         headers = subject.nesting.map(&:header)
         wrapped = "#{headers.join('; ')}; #{text}; #{(['end'] * headers.size).join('; ')}"
-        code, line = with_magic_comment(subject, wrapped)
+        code, line = with_magic_comment(subject, wrapped, extra_lines)
         TOPLEVEL_BINDING.eval(code, subject.file, line)
       end
 
-      def with_magic_comment(subject, code)
-        return [code, subject.start_line] unless subject.frozen_literals
+      # The first line of code sits extra_lines above the def, so the def
+      # keeps its file line numbers.
+      def with_magic_comment(subject, code, extra_lines)
+        return [code, subject.start_line - extra_lines] unless subject.frozen_literals
 
-        [FROZEN_HEADER + code, subject.start_line - 1]
+        [FROZEN_HEADER + code, subject.start_line - extra_lines - 1]
       end
 
       # Methods defined with alias/alias_method copy the old body. Callers