saboteur 0.7.0 → 0.8.0

data/lib/saboteur/engine/run_budget.rb

@@ -41,6 +41,33 @@ module Saboteur
         config["max_duration"] && (@run_started + config["max_duration"])
       end
 
+      # Once the baselines are timed, a cold coverage build is the last
+      # unpriced phase before the mutants: it costs about one more full pass
+      # and keeps nothing when the budget cuts it off, so a resume starts it
+      # from zero again. Refuse up front when the remaining budget cannot
+      # cover that pass, naming the shortfall and the remedies (#427).
+      def refuse_unaffordable_coverage_build(store, files, baseline_elapsed = @baseline_elapsed,
+                                             completed_passes = @completed_baseline_passes)
+        return unless config["max_duration"]
+        return if store.reusable?(files)
+        return if completed_passes.to_i.zero?
+
+        per_pass = baseline_elapsed / completed_passes
+        remaining = config["max_duration"] - (Process.clock_gettime(Process::CLOCK_MONOTONIC) - @run_started)
+        # A spent budget is the existing stop path's news, and an untimed
+        # (zero-second) baseline carries no estimate to project from.
+        return if remaining <= 0 || per_pass <= 0
+        return if remaining >= per_pass
+
+        needed = per_pass - remaining
+        raise BudgetPhaseTimeout,
+              "the --max-duration budget cannot cover the coverage map build: baseline passes took about " \
+              "#{per_pass.round}s each, only #{remaining.round(1)}s remain, and a cold build needs about one " \
+              "more full pass and keeps nothing when it is cut off. Re-run with --max-duration about " \
+              "#{needed.round}s higher, fewer --baseline-passes, or once without --max-duration so the " \
+              "coverage cache is built for later --incremental runs"
+      end
+
       # Boot and baselines can spend the budget by themselves: stop before
       # an expensive cold coverage build and mark every selected subject
       # unrun.

data/lib/saboteur/engine/sandbox_policy.rb

@@ -26,6 +26,7 @@ module Saboteur
       # which refuses instead.
       def hook
         return nil unless @config["sandbox"]
+        return seatbelt_hook if @config["sandbox"] == "seatbelt" && Execution::Seatbelt.darwin?
 
         abi = sandbox_abi
         return restrict_hook if abi >= 3
@@ -64,6 +65,43 @@ module Saboteur
         ->(keep_fds) { Execution::Sandbox.restrict!(writable: writable, keep_fds: keep_fds) }
       end
 
+      # c.sandbox = "seatbelt" (the macOS opt-in, #436): the probe decides.
+      # Confine when Seatbelt truly enforces; a clean "no Seatbelt here"
+      # degrades to the guard and canary with a one-time note; anything
+      # else - an errored probe, or an API claiming success its profile
+      # cannot prove - refuses the run, exactly like the Landlock probe
+      # (#178): opting into a sandbox must never buy silent confinement
+      # theatre. Fail-closed lives one layer down too: a child whose
+      # profile refuses to apply raises instead of running the mutant.
+      def seatbelt_hook
+        case (verdict = Execution::Seatbelt.probe)
+        when :ok
+          writable = sandbox_writable_dirs
+          ->(keep_fds) { Execution::Seatbelt.restrict!(writable: writable, keep_fds: keep_fds) }
+        when :unavailable
+          seatbelt_note("this macOS lacks Seatbelt")
+          nil
+        else
+          raise Error, "sandbox is \"seatbelt\" but the Seatbelt probe #{verdict_description(verdict)}; refusing to " \
+                       "run unsandboxed - set c.sandbox = true to run with the hazard guard and canary only, " \
+                       "or false to opt out"
+        end
+      end
+
+      def verdict_description(verdict)
+        case verdict
+        when :unenforced then "reported a profile that did not enforce - a macOS update may have changed it (#436)"
+        else "failed (#{verdict}) - #436 tracks the deprecation"
+        end
+      end
+
+      def seatbelt_note(reason)
+        return if @sandbox_note_given
+
+        @sandbox_note_given = true
+        @diagnostics.warn(%(sandbox is "seatbelt" but #{reason}; children run with the hazard guard and canary only))
+      end
+
       # c.sandbox_writable adds the dirs a suite genuinely needs (db/, log/,
       # ...): without a grant, writes there fail with EACCES in the child.
       # Only the user grants are created: the project tmp is granted when it

data/lib/saboteur/engine/scheduler.rb

@@ -12,6 +12,7 @@ module Saboteur
     # (VerdictCheck). Each mutant runs only the tests that executed its edit,
     # likeliest killers first, with a timeout from their recorded durations.
     class Scheduler
+      BUDGET_RETRY_REASON = "the --max-duration budget stopped the defined-order retry"
       WHOLE_FILE_HINT = "; add it to c.whole_file to re-evaluate the whole file"
 
       # A subject to run, with what its mutants need. killers counts the
@@ -27,7 +28,11 @@ module Saboteur
         tests.sort_by.with_index { |id, index| [-killers.fetch(id, 0), index] }
       end
 
-      def initialize(config:, runner:, map:, reevaluator:, diagnostics:, cache: nil, deadline: nil)
+      # verify_loaded: the shadowed-definition guard resolves each subject's
+      # owner in this process. --test-command never boots the app here, so
+      # external runs pass false; their neutral check covers a red tree.
+      def initialize(config:, runner:, map:, reevaluator:, diagnostics:, cache: nil, deadline: nil,
+                     verify_loaded: true)
         @config = config
         @deadline = deadline
         @runner = runner
@@ -35,6 +40,7 @@ module Saboteur
         @reevaluator = reevaluator
         @diagnostics = diagnostics
         @cache = cache
+        @verify_loaded = verify_loaded
       end
 
       def call(subject, candidates)
@@ -49,6 +55,7 @@ module Saboteur
 
         tests = @map.tests_for(subject)
         return unloadable(candidates, subject) if tests.empty?
+        return load_time_results(candidates, subject) if load_only?(subject)
 
         plan = Plan.new(subject:, tests:, killers: Hash.new(0), slowdown: 1.0, wrapper: wrapper)
         reused = all_reused(plan, candidates)
@@ -85,7 +92,7 @@ module Saboteur
         subject = plan.subject
         check = VerdictCheck.new(run_unchanged: unchanged_runner(subject, plan),
                                  baseline_timeout: @config["baseline_timeout"])
-        final = check.call(results)
+        final = check.call(results).map { |result| retried_in_defined_order(plan, result) }
         # Unverified verdicts are never cached: a budget-cut check proves
         # nothing, and a stored copy would read as confirmed next run.
         final.reject { |result| result.reused? || result.status == :unverified }.each { |r| @cache&.record(r) }
@@ -96,6 +103,46 @@ module Saboteur
         final
       end
 
+      # A kill rejected because its covering examples fail with the method
+      # unchanged may only lack the setup examples the subset run skipped
+      # (an order-dependent file, #557). Retry once with the file's examples
+      # in defined order, through the last covering example. It stands only
+      # if that whole run passes unchanged; the mutant's verdict is then the
+      # mutant's run on the same examples. Otherwise the error stays.
+      #
+      # Cost bound: at most two extra forks per rejected kill, the unchanged
+      # prefix run and one mutant run. Both spend --max-duration like every
+      # other fork: neither starts once the budget is spent, and neither gets
+      # more than the time left. A retry the budget stops is unverified, not
+      # an error or a kill. Keep this bound if the retry ever grows.
+      def retried_in_defined_order(plan, result)
+        return result unless result.rejected_kill? && !result.reused? && result.covered_by.any?
+
+        tests = DefinedOrderPrefix.new(@map.tests).for(result.covered_by)
+        return result unless tests
+
+        unchanged = unchanged_runner(plan.subject, plan).call(tests, @config["baseline_timeout"])
+        return result.as_unverified(unchanged.message || BUDGET_RETRY_REASON) if unchanged.status == :unverified
+        return result unless unchanged.status == :survived
+
+        rerun_in_prefix(plan, result, tests)
+      end
+
+      def rerun_in_prefix(plan, result, tests)
+        return result.as_unverified(BUDGET_RETRY_REASON) if budget_spent?
+
+        full = timeout_for(tests, plan.slowdown)
+        capped = budget_capped(full)
+        outcome = @runner.run(subject: plan.subject, text: result.mutated_source, test_ids: tests,
+                              timeout: capped, mutation: result.mutation, eval_prefix: eval_prefix(plan))
+        return result.as_unverified(BUDGET_RETRY_REASON) if outcome.status == :timeout && capped < full
+
+        MutantResult.new(subject: plan.subject, mutation: result.mutation, status: outcome.status,
+                         mutated_source: result.mutated_source, reason: outcome.message, runtime: outcome.runtime,
+                         killed_by: outcome.failed_ids, covered_by: tests, kill_cause: outcome.kill_cause,
+                         executed: outcome.executed_ids)
+      end
+
       # Admission is per sequence, not just at checked() entry: a check
       # admitted with time left can hit its capped timeout mid-check, and a
       # later sequence can arrive after the budget is spent. Neither may
@@ -204,6 +251,20 @@ module Saboteur
         end
       end
 
+      # A def-time macro's only callers ran while the app loaded: the
+      # child re-evals the def, but closure consumers captured the
+      # pristine definition, so no test can observe a mutant (#409).
+      def load_only?(subject)
+        !subject.is_a?(ConstantSubject) && @map.called_only_at_load?(subject.file, subject.start_line)
+      end
+
+      def load_time_results(candidates, subject)
+        candidates.map do |mutation, text|
+          MutantResult.new(subject: subject, mutation: mutation, status: :ran_at_load, mutated_source: text,
+                           reason: Selection::CoverageMap::RAN_AT_LOAD)
+        end
+      end
+
       # One mutation whose edited lines no test ran (see unloadable).
       def uncoverable(plan, mutation, text)
         line_numbers = mutation.edits.flat_map { |edit| plan.subject.coverage_lines_for(edit) }.uniq
@@ -219,7 +280,50 @@ module Saboteur
         timeout = timeout_for(tests, plan.slowdown)
         outcome = @runner.run(subject: plan.subject, text: text, test_ids: tests, timeout: timeout,
                               mutation: mutation, eval_prefix: eval_prefix(plan))
-        outcome.status == :timeout ? calibrate(plan, mutation, text, tests, timeout) : outcome
+        outcome = calibrate(plan, mutation, text, tests, timeout) if outcome.status == :timeout
+        flaky_check(plan, mutation, text, tests, outcome)
+      end
+
+      # --flaky-check (#476): one reordered confirmation run per decisive
+      # fresh verdict. A disagreement is unverified - outside the score and
+      # never cached - not "rerun until it passes". A reused verdict is
+      # already trusted evidence and never reaches here. A budget-cut
+      # outcome is already unverified and earns no second fork.
+      def flaky_check(plan, mutation, text, tests, outcome)
+        return outcome unless flaky_checkable?(outcome)
+        return outcome if budget_spent?
+
+        timeout = timeout_for(tests, plan.slowdown)
+
+        rerun = @runner.run(subject: plan.subject, text: text, test_ids: reordered(tests),
+                            timeout: timeout, mutation: mutation, eval_prefix: eval_prefix(plan))
+        return outcome if rerun.status == outcome.status
+
+        Execution::Runner::Outcome.new(status: :unverified, runtime: outcome.runtime + rerun.runtime,
+                                       message: flaky_message(outcome, rerun))
+      end
+
+      # A shuffled order that is guaranteed to differ (a two-test shuffle
+      # has even odds of coming out identical): order-dependence is exactly
+      # what the confirmation run hunts.
+      def reordered(tests)
+        shuffled = tests.shuffle(random: Random.new(@config["seed"]))
+        shuffled == tests && shuffled.size > 1 ? shuffled.rotate : shuffled
+      end
+
+      def flaky_checkable?(outcome)
+        case @config["flaky_check"]
+        when "survivors" then outcome.status == :survived
+        when "decisive" then %i[survived killed].include?(outcome.status)
+        else false
+        end
+      end
+
+      def flaky_message(outcome, rerun)
+        decisive = (outcome.failed_ids | rerun.failed_ids).first(3)
+        named = decisive.empty? ? "" : " (decisive tests: #{decisive.join(', ')})"
+        "flaky verdict: first #{outcome.status}, then #{rerun.status} on a reordered confirmation " \
+          "run#{named} - the mutant is unverified, not #{outcome.status}"
       end
 
       # Each mutant is checked separately: changing a body can change which
@@ -329,6 +433,8 @@ module Saboteur
       # sig is one, see Execution::Wrapper) is not a shadowing - the child
       # re-evaluates it together with the def, so the check passes.
       def load_problem(subject)
+        return [nil, nil] unless @verify_loaded
+
         return [nil, nil] if @reevaluator.whole_file?(subject)
 
         owner = @reevaluator.resolve_owner(subject)

data/lib/saboteur/engine/selection_announcement.rb

@@ -0,0 +1,54 @@
+# SPDX-License-Identifier: MIT
+# frozen_string_literal: true
+
+module Saboteur
+  class Engine
+    # Nothing to mutate is the run's outcome, already stated by the
+    # "0 subjects, 0 mutants" summary: note it once (#468). As a report
+    # warning it would print again at the end of the run, and a zero-subject
+    # run ends right away, so the repeat reads like a second failure.
+    module SelectionAnnouncement
+      private
+
+      def selection_message(guess)
+        if guess.changed.zero?
+          target = config["since"] ? "no changed source file" : "no source file"
+          "inferred specs: #{target} to infer a spec for, so the whole suite runs"
+        elsif guess.specs.empty?
+          guess.no_specs_message(since: config["since"])
+        else
+          scope_to_guesses(guess)
+        end
+      end
+
+      def announce_selection(guess)
+        if guess.changed.zero? || guess.specs.any?
+          @diagnostics.announce(selection_message(guess))
+        else
+          @diagnostics.inform(selection_message(guess))
+        end
+      end
+
+      # A spec file the diff touched that cannot run is named, also before a
+      # stop on an unmatched file: --test, the way out, never reads the diff.
+      def inform_skipped_specs(guess) = (@diagnostics.inform(guess.skipped_message) if guess.skipped_specs.any?)
+
+      # Stop before boot: name the spec-less files and the ways out. The
+      # hinted spec paths are the conventional ones - they do not exist (that
+      # is why the files are unmatched), so the message never prints them as
+      # a runnable --test command, and it never reconstructs the source
+      # scope: the user appends --test or --all to their own command. The
+      # example is still shell-safe: the ref is user input, so it is quoted
+      # and pinned to the flag with = (a ref like "-x" must not become a
+      # switch).
+      def scope_to_guesses(guess)
+        # The guesses now stand as explicit tests, like --test. guess_tests is
+        # dropped rather than set to false: an explicit false is the deprecated
+        # opt-out and would warn.
+        @config = config.with({ "tests" => guess.specs }, except: ["guess_tests"])
+        @guessed_tests = guess.specs
+        guess.scope_message(since: config["since"])
+      end
+    end
+  end
+end

data/lib/saboteur/engine/shard.rb

@@ -0,0 +1,69 @@
+# SPDX-License-Identifier: MIT
+# frozen_string_literal: true
+
+require "digest"
+
+module Saboteur
+  class Engine
+    # Deterministic 1-based k/n split of the planned candidate inventory.
+    # The key names the candidate, never its position, so shards stay stable
+    # when an earlier subject gains or loses a mutant.
+    class Shard
+      FORMAT = %r{\A(\d+)/(\d+)\z}
+      INVENTORY_JOINER = "\0"
+
+      attr_reader :index, :total, :all_keys, :selected_keys, :inventory_digest
+
+      def self.parse(value)
+        match = FORMAT.match(value.to_s)
+        index, total = match.captures.map(&:to_i) if match
+        return new(index, total) if match && index.positive? && total >= 2 && index <= total
+
+        raise ConfigError, "shard must be k/n with 1 <= k <= n and n >= 2 (for example 1/4)"
+      end
+
+      def initialize(index, total)
+        @index = index
+        @total = total
+      end
+
+      # Returns lists in the same shape as candidate_lists: [[mutation, text]] per subject.
+      def select(subjects, lists)
+        keyed = subjects.zip(lists).map do |subject, list|
+          keys = list.map { |mutation, _| key(subject, mutation) }
+          [keys, list]
+        end
+        @all_keys = keyed.flat_map(&:first)
+        @inventory_digest = Digest::SHA256.hexdigest(@all_keys.sort.join(INVENTORY_JOINER))
+        @selected_keys = @all_keys.select { |key| shard_for(key) == index - 1 }
+        selected = @selected_keys.to_set
+        keyed.map do |keys, list|
+          list.select.with_index { |_, position| selected.include?(keys[position]) }
+        end
+      end
+
+      def planned_count = all_keys.size
+      def selected_count = selected_keys.size
+
+      def line
+        "Shard #{index}/#{total}: #{selected_count} of #{planned_count} planned mutants; " \
+          "inventory sha256:#{inventory_digest}; partial run"
+      end
+
+      def to_h
+        { index: index, of: total, selected: selected_count, planned: planned_count,
+          inventoryDigest: inventory_digest, partial: true }
+      end
+
+      private
+
+      def key(subject, mutation)
+        "#{subject.id} #{mutation.digest(subject_start: subject.start_byte, original_source: subject.file_source)}"
+      end
+
+      def shard_for(key)
+        Digest::SHA256.digest(key).unpack1("Q>") % total
+      end
+    end
+  end
+end