saboteur 0.7.0 → 0.8.1

data/lib/saboteur/engine/sandbox_policy.rb

@@ -26,6 +26,7 @@ module Saboteur
       # which refuses instead.
       def hook
         return nil unless @config["sandbox"]
+        return seatbelt_hook if @config["sandbox"] == "seatbelt" && Execution::Seatbelt.darwin?
 
         abi = sandbox_abi
         return restrict_hook if abi >= 3
@@ -38,8 +39,7 @@ module Saboteur
         # same line twice (D's review, #178).
         unless @sandbox_note_given
           @sandbox_note_given = true
-          @diagnostics.warn("sandbox is on but #{self.class.sandbox_gap(abi)}; children run with the hazard guard " \
-                            "and canary only")
+          @diagnostics.warn(degraded_note(abi))
         end
         nil
       end
@@ -54,16 +54,72 @@ module Saboteur
       end
 
       def self.sandbox_gap(abi)
-        return "this kernel lacks Landlock (needs Linux 6.2+)" if abi.zero?
+        if abi.zero?
+          # Landlock is Linux-only: on macOS there is no kernel to upgrade,
+          # so name the platform instead of a Linux version (#667).
+          return "macOS has no Landlock" if Execution::Seatbelt.darwin?
+
+          return "this kernel lacks Landlock (needs Linux 6.2+)"
+        end
 
         "this kernel's Landlock ABI is #{abi} but ABI 3 (Linux 6.2+) is needed to restrict truncate()"
       end
 
+      # The degraded-run note: on macOS the gap is permanent, so point at
+      # the Seatbelt opt-in; on Linux the kernel wording stands alone
+      # (#667). Severity stays a warning - degrading confinement is never
+      # routine.
+      def degraded_note(abi)
+        gap = self.class.sandbox_gap(abi)
+        tail = "children run with the hazard guard and canary only"
+        return "sandbox is on but #{gap}; #{tail}" unless Execution::Seatbelt.darwin?
+
+        "sandbox is on but #{gap}; #{tail} - opt into the macOS sandbox with c.sandbox = \"seatbelt\" " \
+          "(docs/guides/macos-seatbelt-sandbox.md), or set c.sandbox = false to silence this"
+      end
+
       def restrict_hook
         writable = sandbox_writable_dirs
         ->(keep_fds) { Execution::Sandbox.restrict!(writable: writable, keep_fds: keep_fds) }
       end
 
+      # c.sandbox = "seatbelt" (the macOS opt-in, #436): the probe decides.
+      # Confine when Seatbelt truly enforces; a clean "no Seatbelt here"
+      # degrades to the guard and canary with a one-time note; anything
+      # else - an errored probe, or an API claiming success its profile
+      # cannot prove - refuses the run, exactly like the Landlock probe
+      # (#178): opting into a sandbox must never buy silent confinement
+      # theatre. Fail-closed lives one layer down too: a child whose
+      # profile refuses to apply raises instead of running the mutant.
+      def seatbelt_hook
+        case (verdict = Execution::Seatbelt.probe)
+        when :ok
+          writable = sandbox_writable_dirs
+          ->(keep_fds) { Execution::Seatbelt.restrict!(writable: writable, keep_fds: keep_fds) }
+        when :unavailable
+          seatbelt_note("this macOS lacks Seatbelt")
+          nil
+        else
+          raise Error, "sandbox is \"seatbelt\" but the Seatbelt probe #{verdict_description(verdict)}; refusing to " \
+                       "run unsandboxed - set c.sandbox = true to run with the hazard guard and canary only, " \
+                       "or false to opt out"
+        end
+      end
+
+      def verdict_description(verdict)
+        case verdict
+        when :unenforced then "reported a profile that did not enforce - a macOS update may have changed it (#436)"
+        else "failed (#{verdict}) - #436 tracks the deprecation"
+        end
+      end
+
+      def seatbelt_note(reason)
+        return if @sandbox_note_given
+
+        @sandbox_note_given = true
+        @diagnostics.warn(%(sandbox is "seatbelt" but #{reason}; children run with the hazard guard and canary only))
+      end
+
       # c.sandbox_writable adds the dirs a suite genuinely needs (db/, log/,
       # ...): without a grant, writes there fail with EACCES in the child.
       # Only the user grants are created: the project tmp is granted when it

data/lib/saboteur/engine/scheduler.rb

@@ -12,6 +12,7 @@ module Saboteur
     # (VerdictCheck). Each mutant runs only the tests that executed its edit,
     # likeliest killers first, with a timeout from their recorded durations.
     class Scheduler
+      BUDGET_RETRY_REASON = "the --max-duration budget stopped the defined-order retry"
       WHOLE_FILE_HINT = "; add it to c.whole_file to re-evaluate the whole file"
 
       # A subject to run, with what its mutants need. killers counts the
@@ -27,7 +28,11 @@ module Saboteur
         tests.sort_by.with_index { |id, index| [-killers.fetch(id, 0), index] }
       end
 
-      def initialize(config:, runner:, map:, reevaluator:, diagnostics:, cache: nil, deadline: nil)
+      # verify_loaded: the shadowed-definition guard resolves each subject's
+      # owner in this process. --test-command never boots the app here, so
+      # external runs pass false; their neutral check covers a red tree.
+      def initialize(config:, runner:, map:, reevaluator:, diagnostics:, cache: nil, deadline: nil,
+                     verify_loaded: true)
         @config = config
         @deadline = deadline
         @runner = runner
@@ -35,6 +40,7 @@ module Saboteur
         @reevaluator = reevaluator
         @diagnostics = diagnostics
         @cache = cache
+        @verify_loaded = verify_loaded
       end
 
       def call(subject, candidates)
@@ -49,6 +55,7 @@ module Saboteur
 
         tests = @map.tests_for(subject)
         return unloadable(candidates, subject) if tests.empty?
+        return load_time_results(candidates, subject) if load_only?(subject)
 
         plan = Plan.new(subject:, tests:, killers: Hash.new(0), slowdown: 1.0, wrapper: wrapper)
         reused = all_reused(plan, candidates)
@@ -57,7 +64,7 @@ module Saboteur
         results = run_candidates(candidates) do |mutation, text|
           reuse(plan, mutation, text) || execute(plan, mutation, text)
         end
-        checked(plan, results)
+        checked(plan, results, truncated: results.size < candidates.size)
       end
 
       # The per-mutant timeout: timeout_factor times what the tests took
@@ -74,18 +81,25 @@ module Saboteur
 
       # Verdicts that did not hold up are warned about once per subject;
       # only checked verdicts are stored for --incremental.
-      # A spent budget admits no check: forking the unchanged run with no
-      # time left would turn genuine kills into fabricated errors and cache
-      # them. Fresh kills stand unverified instead - outside the score,
-      # never cached - and the distribution marks the subject so the run
-      # reports budget-partial.
-      def checked(plan, results)
-        return results.map { |result| unverifiable(result) } if budget_spent?
-
+      # A spent budget admits no check for a subject that finished its
+      # mutants: forking the unchanged run with no time left would turn
+      # genuine kills into fabricated errors and cache them. Fresh kills
+      # stand unverified instead - outside the score, never cached - and
+      # the distribution marks the subject so the run reports
+      # budget-partial.
+      # A subject the budget stopped mid-list is different (#588): its
+      # completed mutants were already measured, so their checks still
+      # run, uncapped - the budget stops new mutation work, not the
+      # confirmation of work already done. The overshoot stays bounded by
+      # the subject's distinct killing sequences.
+      def checked(plan, results, truncated: false)
+        return results.map { |result| unverifiable(result) } if budget_spent? && !truncated
+
+        @confirming_truncation = truncated
         subject = plan.subject
         check = VerdictCheck.new(run_unchanged: unchanged_runner(subject, plan),
                                  baseline_timeout: @config["baseline_timeout"])
-        final = check.call(results)
+        final = check.call(results).map { |result| retried_in_defined_order(plan, result) }
         # Unverified verdicts are never cached: a budget-cut check proves
         # nothing, and a stored copy would read as confirmed next run.
         final.reject { |result| result.reused? || result.status == :unverified }.each { |r| @cache&.record(r) }
@@ -94,6 +108,50 @@ module Saboteur
           @diagnostics.warn("#{subject.id}: #{overturned} verdict(s) failed with the method unchanged")
         end
         final
+      ensure
+        @confirming_truncation = false
+      end
+
+      # A kill rejected because its covering examples fail with the method
+      # unchanged may only lack the setup examples the subset run skipped
+      # (an order-dependent file, #557). Retry once with the file's examples
+      # in defined order, through the last covering example. It stands only
+      # if that whole run passes unchanged; the mutant's verdict is then the
+      # mutant's run on the same examples. Otherwise the error stays.
+      #
+      # Cost bound: at most two extra forks per rejected kill, the unchanged
+      # prefix run and one mutant run. Ordinarily both spend --max-duration
+      # like every other fork: neither starts once the budget is spent, and
+      # neither gets more than the time left. A retry the budget stops is
+      # unverified, not an error or a kill. The exception is a truncated
+      # subject (#588): its completed kills still retry, at the full
+      # timeout. Keep this bound if the retry ever grows.
+      def retried_in_defined_order(plan, result)
+        return result unless result.rejected_kill? && !result.reused? && result.covered_by.any?
+
+        tests = DefinedOrderPrefix.new(@map.tests).for(result.covered_by)
+        return result unless tests
+
+        unchanged = unchanged_runner(plan.subject, plan).call(tests, @config["baseline_timeout"])
+        return result.as_unverified(unchanged.message || BUDGET_RETRY_REASON) if unchanged.status == :unverified
+        return result unless unchanged.status == :survived
+
+        rerun_in_prefix(plan, result, tests)
+      end
+
+      def rerun_in_prefix(plan, result, tests)
+        return result.as_unverified(BUDGET_RETRY_REASON) if budget_spent? && !@confirming_truncation
+
+        full = timeout_for(tests, plan.slowdown)
+        capped = @confirming_truncation ? full : budget_capped(full)
+        outcome = @runner.run(subject: plan.subject, text: result.mutated_source, test_ids: tests,
+                              timeout: capped, mutation: result.mutation, eval_prefix: eval_prefix(plan))
+        return result.as_unverified(BUDGET_RETRY_REASON) if outcome.status == :timeout && capped < full
+
+        MutantResult.new(subject: plan.subject, mutation: result.mutation, status: outcome.status,
+                         mutated_source: result.mutated_source, reason: outcome.message, runtime: outcome.runtime,
+                         killed_by: outcome.failed_ids, covered_by: tests, kill_cause: outcome.kill_cause,
+                         executed: outcome.executed_ids)
       end
 
       # Admission is per sequence, not just at checked() entry: a check
@@ -103,11 +161,11 @@ module Saboteur
       # the check never got its answer (#296).
       def unchanged_runner(subject, plan)
         lambda do |tests, timeout|
-          if budget_spent?
+          if budget_spent? && !@confirming_truncation
             Execution::Runner::Outcome.new(status: :unverified, runtime: 0.0,
                                            message: "the --max-duration budget is spent")
           else
-            capped = budget_capped(timeout)
+            capped = @confirming_truncation ? timeout : budget_capped(timeout)
             outcome = @runner.run(subject: subject, text: subject.source, test_ids: tests, timeout: capped,
                                   eval_prefix: eval_prefix(plan))
             note_leftovers(subject, "verdict check", outcome)
@@ -191,13 +249,24 @@ module Saboteur
         [@config["timeout_factor"] * runtime, @config["timeout_min"]].max
       end
 
-      # What a subject no test ran gets. Code that only ran while the app
-      # loaded is not "no coverage" - it ran - but no forked test can re-run
-      # it, so it gets its own status, out of the score, with a hint.
+      # What a subject no test ran gets. Load-only code is not "no coverage":
+      # it ran, but no forked test can re-run it, so it gets its own status.
       def unloadable(candidates, subject, line_numbers = nil)
+        return skip_all(candidates, subject, Selection::CoverageMap::PRELOADED) if @map.preloaded?(subject.file)
         return skip_all(candidates, subject, "no coverage") unless
           @map.ran_at_load?(subject.file, line_numbers || subject.coverage_lines)
 
+        load_time_results(candidates, subject)
+      end
+
+      # A def-time macro's only callers ran while the app loaded: the
+      # child re-evals the def, but closure consumers captured the
+      # pristine definition, so no test can observe a mutant (#409).
+      def load_only?(subject)
+        !subject.is_a?(ConstantSubject) && @map.called_only_at_load?(subject.file, subject.start_line)
+      end
+
+      def load_time_results(candidates, subject)
         candidates.map do |mutation, text|
           MutantResult.new(subject: subject, mutation: mutation, status: :ran_at_load, mutated_source: text,
                            reason: Selection::CoverageMap::RAN_AT_LOAD)
@@ -219,7 +288,50 @@ module Saboteur
         timeout = timeout_for(tests, plan.slowdown)
         outcome = @runner.run(subject: plan.subject, text: text, test_ids: tests, timeout: timeout,
                               mutation: mutation, eval_prefix: eval_prefix(plan))
-        outcome.status == :timeout ? calibrate(plan, mutation, text, tests, timeout) : outcome
+        outcome = calibrate(plan, mutation, text, tests, timeout) if outcome.status == :timeout
+        flaky_check(plan, mutation, text, tests, outcome)
+      end
+
+      # --flaky-check (#476): one reordered confirmation run per decisive
+      # fresh verdict. A disagreement is unverified - outside the score and
+      # never cached - not "rerun until it passes". A reused verdict is
+      # already trusted evidence and never reaches here. A budget-cut
+      # outcome is already unverified and earns no second fork.
+      def flaky_check(plan, mutation, text, tests, outcome)
+        return outcome unless flaky_checkable?(outcome)
+        return outcome if budget_spent?
+
+        timeout = timeout_for(tests, plan.slowdown)
+
+        rerun = @runner.run(subject: plan.subject, text: text, test_ids: reordered(tests),
+                            timeout: timeout, mutation: mutation, eval_prefix: eval_prefix(plan))
+        return outcome if rerun.status == outcome.status
+
+        Execution::Runner::Outcome.new(status: :unverified, runtime: outcome.runtime + rerun.runtime,
+                                       message: flaky_message(outcome, rerun))
+      end
+
+      # A shuffled order that is guaranteed to differ (a two-test shuffle
+      # has even odds of coming out identical): order-dependence is exactly
+      # what the confirmation run hunts.
+      def reordered(tests)
+        shuffled = tests.shuffle(random: Random.new(@config["seed"]))
+        shuffled == tests && shuffled.size > 1 ? shuffled.rotate : shuffled
+      end
+
+      def flaky_checkable?(outcome)
+        case @config["flaky_check"]
+        when "survivors" then outcome.status == :survived
+        when "decisive" then %i[survived killed].include?(outcome.status)
+        else false
+        end
+      end
+
+      def flaky_message(outcome, rerun)
+        decisive = (outcome.failed_ids | rerun.failed_ids).first(3)
+        named = decisive.empty? ? "" : " (decisive tests: #{decisive.join(', ')})"
+        "flaky verdict: first #{outcome.status}, then #{rerun.status} on a reordered confirmation " \
+          "run#{named} - the mutant is unverified, not #{outcome.status}"
       end
 
       # Each mutant is checked separately: changing a body can change which
@@ -329,6 +441,8 @@ module Saboteur
       # sig is one, see Execution::Wrapper) is not a shadowing - the child
       # re-evaluates it together with the def, so the check passes.
       def load_problem(subject)
+        return [nil, nil] unless @verify_loaded
+
         return [nil, nil] if @reevaluator.whole_file?(subject)
 
         owner = @reevaluator.resolve_owner(subject)

data/lib/saboteur/engine/selection_announcement.rb

@@ -0,0 +1,59 @@
+# SPDX-License-Identifier: MIT
+# frozen_string_literal: true
+
+module Saboteur
+  class Engine
+    # Nothing to mutate is the run's outcome, already stated by the
+    # "0 subjects, 0 mutants" summary: note it once (#468). As a report
+    # warning it would print again at the end of the run, and a zero-subject
+    # run ends right away, so the repeat reads like a second failure.
+    module SelectionAnnouncement
+      private
+
+      # A --since diff with no changed source file (a spec-only or docs-only
+      # change) yields zero subjects, so the suite never runs: say there is
+      # nothing to test rather than promise a whole-suite run that does not
+      # happen (#607).
+      def selection_message(guess)
+        if guess.changed.zero? && config["since"]
+          "inferred specs: no changed source file to infer a spec for, so there is nothing to test"
+        elsif guess.changed.zero?
+          "inferred specs: no source file to infer a spec for, so the whole suite runs"
+        elsif guess.specs.empty?
+          guess.no_specs_message(since: config["since"])
+        else
+          scope_to_guesses(guess)
+        end
+      end
+
+      def announce_selection(guess)
+        if guess.changed.zero? || guess.specs.any?
+          @diagnostics.announce(selection_message(guess))
+        else
+          @diagnostics.inform(selection_message(guess))
+        end
+      end
+
+      # A spec file the diff touched that cannot run is named, also before a
+      # stop on an unmatched file: --test, the way out, never reads the diff.
+      def inform_skipped_specs(guess) = (@diagnostics.inform(guess.skipped_message) if guess.skipped_specs.any?)
+
+      # Stop before boot: name the spec-less files and the ways out. The
+      # hinted spec paths are the conventional ones - they do not exist (that
+      # is why the files are unmatched), so the message never prints them as
+      # a runnable --test command, and it never reconstructs the source
+      # scope: the user appends --test or --all to their own command. The
+      # example is still shell-safe: the ref is user input, so it is quoted
+      # and pinned to the flag with = (a ref like "-x" must not become a
+      # switch).
+      def scope_to_guesses(guess)
+        # The guesses now stand as explicit tests, like --test. guess_tests is
+        # dropped rather than set to false: an explicit false is the deprecated
+        # opt-out and would warn.
+        @config = config.with({ "tests" => guess.specs }, except: ["guess_tests"])
+        @guessed_tests = guess.specs
+        guess.scope_message(since: config["since"])
+      end
+    end
+  end
+end

data/lib/saboteur/engine/shard.rb

@@ -0,0 +1,69 @@
+# SPDX-License-Identifier: MIT
+# frozen_string_literal: true
+
+require "digest"
+
+module Saboteur
+  class Engine
+    # Deterministic 1-based k/n split of the planned candidate inventory.
+    # The key names the candidate, never its position, so shards stay stable
+    # when an earlier subject gains or loses a mutant.
+    class Shard
+      FORMAT = %r{\A(\d+)/(\d+)\z}
+      INVENTORY_JOINER = "\0"
+
+      attr_reader :index, :total, :all_keys, :selected_keys, :inventory_digest
+
+      def self.parse(value)
+        match = FORMAT.match(value.to_s)
+        index, total = match.captures.map(&:to_i) if match
+        return new(index, total) if match && index.positive? && total >= 2 && index <= total
+
+        raise ConfigError, "shard must be k/n with 1 <= k <= n and n >= 2 (for example 1/4)"
+      end
+
+      def initialize(index, total)
+        @index = index
+        @total = total
+      end
+
+      # Returns lists in the same shape as candidate_lists: [[mutation, text]] per subject.
+      def select(subjects, lists)
+        keyed = subjects.zip(lists).map do |subject, list|
+          keys = list.map { |mutation, _| key(subject, mutation) }
+          [keys, list]
+        end
+        @all_keys = keyed.flat_map(&:first)
+        @inventory_digest = Digest::SHA256.hexdigest(@all_keys.sort.join(INVENTORY_JOINER))
+        @selected_keys = @all_keys.select { |key| shard_for(key) == index - 1 }
+        selected = @selected_keys.to_set
+        keyed.map do |keys, list|
+          list.select.with_index { |_, position| selected.include?(keys[position]) }
+        end
+      end
+
+      def planned_count = all_keys.size
+      def selected_count = selected_keys.size
+
+      def line
+        "Shard #{index}/#{total}: #{selected_count} of #{planned_count} planned mutants; " \
+          "inventory sha256:#{inventory_digest}; partial run"
+      end
+
+      def to_h
+        { index: index, of: total, selected: selected_count, planned: planned_count,
+          inventoryDigest: inventory_digest, partial: true }
+      end
+
+      private
+
+      def key(subject, mutation)
+        "#{subject.id} #{mutation.digest(subject_start: subject.start_byte, original_source: subject.file_source)}"
+      end
+
+      def shard_for(key)
+        Digest::SHA256.digest(key).unpack1("Q>") % total
+      end
+    end
+  end
+end