saboteur 0.7.0 → 0.8.0

data/lib/saboteur/execution/process_tree.rb

@@ -2,6 +2,7 @@
 # frozen_string_literal: true
 
 require_relative "process_table"
+require_relative "marker_sweep"
 
 module Saboteur
   module Execution
@@ -12,14 +13,19 @@ module Saboteur
     # setsid (a nested saboteur run does exactly that) and leave the group.
     # Parent links survive that, so the tree is rebuilt from the process table
     # on every poll, and every pid ever seen in it is remembered, so a
-    # descendant whose parent exits (re-parented to init) is still known.
+    # descendant whose parent exits (re-parented to init) is still known -
+    # but only if some poll SAW the parent link while the intermediate lived.
+    # A descendant that leaves the group and loses that parent between two
+    # polls is never seen (#365); the marker sweep in kill_all is the net
+    # for exactly that case.
     #
     # Identity is pid plus start time, so a recycled pid is never signalled.
     class ProcessTree
       attr_reader :root
 
-      def initialize(root)
+      def initialize(root, marker: nil)
         @root = root
+        @marker = marker
         @known = {} # pid => start time
       end
 
@@ -34,6 +40,27 @@ module Saboteur
         end
       end
 
+      # kill_all minus an excluded root, with the pre-kill member
+      # descriptions: [killed pids, notes]. The notes are taken first -
+      # after the kill the command lines are gone (#486).
+      def kill_all_except(root)
+        notes = describe_except(root)
+        [kill_all - [root], notes]
+      end
+
+      # One "pid (command)" note per live member other than an excluded
+      # root, for the leftover-children refusal (#486). Best effort: a
+      # process that vanished mid-look lists its pid alone. Call before the
+      # kill - after it, the command lines are gone.
+      def describe_except(root, table = ProcessTable.snapshot)
+        live_members(table).filter_map do |entry|
+          next if entry.pid == root
+
+          command = ProcessTable.command(entry.pid)
+          command ? "#{entry.pid} (#{command})" : entry.pid.to_s
+        end
+      end
+
       # Count and resident memory of the live members. rss_baseline_kb is the
       # supervisor's own RSS: forked children share its pages, so only growth
       # beyond it is counted per process.
@@ -44,15 +71,20 @@ module Saboteur
       end
 
       # Stops every member (a stopped process cannot fork), rescans until no new
-      # member appears, then kills them all. Returns the pids it killed.
+      # member appears, then kills them all. Each round also sweeps for
+      # never-seen escapees carrying the run's marker (#365). Returns the
+      # pids it killed.
       def kill_all(rounds: 20)
         killed = []
         rounds.times do
           live = stop_until_stable
-          break if live.empty?
+          escaped = marked_escapees
+          break if live.empty? && escaped.empty?
 
           live.each { |entry| signal("KILL", entry.pid) }
-          killed.concat(live.map(&:pid))
+          signalled = escaped.select { |entry| escapee_alive?(entry) }
+          signalled.each { |entry| signal("KILL", entry.pid) }
+          killed.concat(live.map(&:pid), signalled.map(&:pid))
           sleep 0.01
         end
         killed.uniq
@@ -60,6 +92,34 @@ module Saboteur
 
       private
 
+      # Live, non-zombie pids carrying the run's marker that the tree never
+      # learned: the setsid-and-orphan escapees (#365). Members match the
+      # marker too, but the same round kills them anyway.
+      def marked_escapees
+        return [] unless @marker && MarkerSweep.available?
+
+        table = ProcessTable.snapshot
+        root_started = @known[root]
+        MarkerSweep.marked_pids(@marker, Supervisor::TREE_ENV).filter_map do |pid|
+          entry = table[pid]
+          next unless entry && !entry.zombie && !@known.key?(pid)
+
+          # A process started before the run's root carries the marker only
+          # through an exec chain the run did not spawn: never ours (#389).
+          next if root_started && entry.started < root_started
+
+          entry
+        end
+      end
+
+      # A pid is not an identity: the escapee can exit between the sweep's
+      # snapshot and the KILL, and the kernel can hand the pid to someone
+      # else. Signal only the process the sweep actually saw (#389).
+      def escapee_alive?(entry)
+        fresh = ProcessTable.snapshot[entry.pid]
+        fresh && !fresh.zombie && fresh.started == entry.started
+      end
+
       def grow(table)
         @known[root] = table[root].started if @known.empty? && table[root]
         loop do

data/lib/saboteur/execution/reevaluator.rb

@@ -54,9 +54,54 @@ module Saboteur
         nil
       end
 
+      # The method object the subject's def currently resolves to,
+      # normalized through a known def-time wrapper. The runner compares
+      # what the mutant's evaluation installed with what is in place after
+      # the tests: a suite that re-loads the file mid-run (a before(:each)
+      # `load "lib/..."`, rolify-style) quietly replaces the mutant, and a
+      # "survivor" then only proves the tests never ran it. nil when this
+      # cannot be told (a missing or non-Ruby method): the caller keeps the
+      # verdict.
+      def installed_definition(subject)
+        owner = resolve_owner(subject)
+        target = subject.singleton ? owner.singleton_class : owner
+        method = target.instance_method(subject.name)
+        Wrapper.original_def(method) || method
+      rescue StandardError
+        nil
+      end
+
+      # The definition now in place for the subject versus the one the
+      # mutant's evaluation installed: a "file:line" site when the tests
+      # provably replaced it - a before(:each) re-loading the subject's
+      # own file (rolify-style `load "lib/..."`, #597) puts the original
+      # back before any test calls it, so a "survivor" proves nothing.
+      # Only a positively identified overwrite counts: the replacement
+      # comes from the subject's own file, and no prepend or
+      # allow_any_instance_of-style wrapper whose super chain still
+      # reaches the mutant (those tests did run it). Anything
+      # undecidable returns nil and the verdict stands.
+      def overwritten_site(subject, installed)
+        now = installed_definition(subject)
+        return nil if now.nil? || now == installed
+
+        method = now
+        method = method.super_method until method.nil? || method == installed
+        return nil if method # the mutant is still reachable through super
+
+        file, line = now.source_location
+        return nil unless file && File.expand_path(file) == File.expand_path(subject.file)
+
+        "#{file}:#{line}"
+      rescue StandardError
+        nil
+      end
+
       # True when the subject's id matches a whole_file entry (exact or a
       # File.fnmatch glob).
       def whole_file?(subject)
+        return true if subject.whole_file_only?
+
         @whole_file.any? { |pattern| File.fnmatch?(pattern, subject.id) }
       end
 
@@ -87,12 +132,15 @@ module Saboteur
 
       # Resolves the subject's owner the way Ruby would resolve the nesting in
       # the file. const_get also triggers autoload (Zeitwerk) before any reopening.
+      # A `class << self` scope resolves the scopes after it (a factory
+      # constant assigned there binds on the singleton class, #352); when the
+      # subject's own scope is the singleton class, the owner is still the
+      # attached class.
       def resolve_owner(subject)
-        scopes = [Object]
-        subject.nesting.reject(&:sclass?).each do |scope|
-          scopes << resolve_path(scope.path, scopes)
+        scopes = subject.nesting.each_with_object([Object]) do |scope, resolved|
+          resolved << (scope.sclass? ? resolved.last.singleton_class : resolve_path(scope.path, resolved))
         end
-        owner = scopes.last
+        owner = subject.sclass ? scopes[-2] : scopes.last
         raise Error, "#{subject.owner} is not a class or module" unless owner.is_a?(Module)
 
         owner
@@ -139,28 +187,36 @@ module Saboteur
 
       # The file as it is on disk, with the method's bytes replaced by text,
       # evaluated at top level under the file's own name and first line.
-      # Redefinition warnings are expected and silenced.
       def eval_whole_file(subject, text)
         source = subject.file_source.dup.force_encoding(Encoding::UTF_8)
         spliced = source.byteslice(0, subject.start_byte) + text + source.byteslice(subject.end_byte..)
-        verbose = $VERBOSE
-        $VERBOSE = nil
-        TOPLEVEL_BINDING.eval(spliced, subject.file, 1)
-      ensure
-        $VERBOSE = verbose
+        silencing_warnings { TOPLEVEL_BINDING.eval(spliced, subject.file, 1) }
       end
 
       def eval_in_owner(owner, subject, text, extra_lines)
         code, line = with_magic_comment(subject, text, extra_lines)
         eval_target = subject.sclass ? owner.singleton_class : owner
-        eval_target.class_eval(code, subject.file, line)
+        silencing_warnings { eval_target.class_eval(code, subject.file, line) }
       end
 
       def eval_nested(subject, text, extra_lines)
         headers = subject.nesting.map(&:header)
         wrapped = "#{headers.join('; ')}; #{text}; #{(['end'] * headers.size).join('; ')}"
         code, line = with_magic_comment(subject, wrapped, extra_lines)
-        TOPLEVEL_BINDING.eval(code, subject.file, line)
+        silencing_warnings { TOPLEVEL_BINDING.eval(code, subject.file, line) }
+      end
+
+      # Re-evaluation redefines the method by design, and a "method
+      # redefined" warning is only ever emitted under $VERBOSE. Suites that
+      # raise on warnings (--warnings plus a raising Warning.warn or
+      # Warning.process handler, dry-rb/hanami-style) must not see it:
+      # before the guard every mutant errored as RuntimeError (#603).
+      def silencing_warnings
+        verbose = $VERBOSE
+        $VERBOSE = nil
+        yield
+      ensure
+        $VERBOSE = verbose
       end
 
       # The first line of code sits extra_lines above the def, so the def

data/lib/saboteur/execution/runner.rb

@@ -144,6 +144,7 @@ module Saboteur
         @reevaluator.call(job.subject, job.text, prefix: job.eval_prefix)
         replaced = @reevaluator.replaced_definition(job.subject)
         Protocol.emit(writer, "replaced", site: replaced) if replaced
+        @installed_definition = job.mutation ? @reevaluator.installed_definition(job.subject) : nil
       end
 
       # Capture detection is best-effort: a subject shape it cannot scan
@@ -156,18 +157,40 @@ module Saboteur
 
       def emit_result(writer, job, has_captures)
         method = replacement_method(job) if has_captures && job.mutation
-        called = method ? false : nil
-        trace = TracePoint.new(:call) { called = true } if method
+        flags = { replacement_called: method ? false : nil, mutant_called: false }
+        target = method || (job.mutation ? @installed_definition : nil)
+        trace = call_probe(target, flags)
         started_at = Time.now
-        result = if trace
-                   trace.enable(target: method) { @hooks.around(job.mutation) { run_tests(job.test_ids) } }
-                 else
-                   @hooks.around(job.mutation) { run_tests(job.test_ids) }
-                 end
+        suite = proc { @hooks.around(job.mutation) { run_tests(job.test_ids) } }
+        result = trace ? trace.enable(target: target, &suite) : suite.call
         if result.nil?
           Protocol.emit(writer, "error", message: "around_mutant hook did not run the tests")
         else
+          called = flags[:replacement_called]
           Protocol.emit(writer, "result", **with_crash_detail(result, started_at), replacement_called: called)
+          # A reload after the mutant already ran (an after-hook restore)
+          # is no reason to doubt the verdict; only a never-invoked mutant
+          # can have been hidden by the overwrite.
+          emit_overwritten(writer, job, flags[:mutant_called])
+        end
+      end
+
+      # A reload after the mutant already ran (an after-hook restore) is
+      # no reason to doubt the verdict; only a never-invoked mutant can
+      # have been hidden by the overwrite.
+      def emit_overwritten(writer, job, mutant_called)
+        overwritten = job.mutation && @installed_definition &&
+                      @reevaluator.overwritten_site(job.subject, @installed_definition)
+        Protocol.emit(writer, "overwritten", site: overwritten) if overwritten && !mutant_called
+      end
+
+      # One call probe on the installed definition feeds both verdicts:
+      # replacement_called for stale captures, and overwrite detection for
+      # a mutant the suite replaced before any test ran it (#597).
+      def call_probe(target, flags)
+        target && TracePoint.new(:call) do
+          flags[:mutant_called] = true
+          flags[:replacement_called] = true unless flags[:replacement_called].nil?
         end
       end
 
@@ -392,6 +415,9 @@ module Saboteur
         captures = events.find { |event| event["event"] == "captures" }
         return unreachable(outcome, captures) if unobservable?(outcome, captures, job, result["replacement_called"])
 
+        overwritten = events.find { |event| event["event"] == "overwritten" }
+        return overwritten_by_later_load(outcome, overwritten) if replaced_while_running?(outcome, overwritten, job)
+
         outcome
       end
 
@@ -414,6 +440,19 @@ module Saboteur
         captures && job.mutation && outcome.status == :survived && replacement_called == false
       end
 
+      # A surviving mutant the tests themselves overwrote: a before(:each)
+      # re-loading the file replaced it before any test ran (rolify's suite
+      # shape, #597), so the verdict says nothing about the tests.
+      def replaced_while_running?(outcome, overwritten, job)
+        overwritten && job.mutation && outcome.status == :survived
+      end
+
+      def overwritten_by_later_load(outcome, overwritten)
+        Outcome.new(status: :unreachable, runtime: outcome.runtime,
+                    message: "the tests (or their hooks) replaced the definition while they ran " \
+                             "(#{overwritten['site']}), so this mutant never ran")
+      end
+
       # A survivor that no test could have observed: a Method object
       # captured at load still calls the original. Not a verdict on the
       # tests, so it is its own status, out of the score.

data/lib/saboteur/execution/sandbox.rb

@@ -57,8 +57,15 @@ module Saboteur
         return @abi_version if defined?(@abi_version)
         return @abi_version = 0 unless linux?
 
-        version = LibC.syscall_fn.call(SYS_CREATE_RULESET, 0, 0, CREATE_RULESET_VERSION, 0)
-        errno = last_errno
+        # Build the errno reader BEFORE the probe: its lazy dlopen/dlsym run
+        # libc calls that can set errno, so building it between the probe and
+        # the read can replace the real answer with a stale 0.
+        LibC.errno_location_fn
+        version, errno = probe_once
+        # A failed syscall always sets errno, so (failure, errno 0) is never
+        # the kernel's real answer - the read was clobbered between the call
+        # and here. Probe once more, now with every Fiddle function warm.
+        version, errno = probe_once if !version.positive? && errno.zero?
         @abi_version = probe_abi(version, errno)
       rescue Fiddle::DLError
         # No usable libc (an unusual musl/ldso layout): the sandbox cannot
@@ -70,6 +77,12 @@ module Saboteur
       # no Landlock to give: run unsandboxed, noted once. Anything else
       # (EPERM under a seccomp policy, ...) means the kernel is there but
       # refused - fail closed rather than run without the promised sandbox.
+      # One create-ruleset version probe with its errno read paired to it.
+      def probe_once
+        version = LibC.syscall_fn.call(SYS_CREATE_RULESET, 0, 0, CREATE_RULESET_VERSION, 0)
+        [version, last_errno]
+      end
+
       def probe_abi(version, errno)
         return version if version.positive?
         return 0 if UNSUPPORTED_ERRNOS.include?(errno)

data/lib/saboteur/execution/seatbelt.rb

@@ -0,0 +1,226 @@
+# SPDX-License-Identifier: MIT
+# frozen_string_literal: true
+
+require "fiddle"
+
+module Saboteur
+  module Execution
+    # The macOS counterpart of Sandbox (Landlock): a Seatbelt profile
+    # applied with sandbox_init inside the mutant child. sandbox-exec and
+    # sandbox_init are deprecated by Apple with no published replacement
+    # and no removal timeline (#436 tracks how Apple, Chrome, Codex,
+    # Claude Code, Bazel and Nix respond), so Seatbelt is opt-in and the
+    # probe below fails closed: a child that cannot be confined runs with
+    # the hazard guard and canary only, never silently "sandboxed".
+    #
+    # Like Sandbox, requiring this file is inert off macOS: libsandbox is
+    # dlopened lazily by the first probe, and the probe forks so the
+    # irreversible sandbox_init never touches the run's own process.
+    module Seatbelt
+      LIBSANDBOX = "/usr/lib/system/libsystem_sandbox.dylib"
+      LIBSYSTEM = "/usr/lib/libSystem.B.dylib"
+      F_GETPATH = 50
+      PATH_MAX = 1024
+
+      # The probe profile must deny the write it then attempts.
+      PROBE_PROFILE = "(version 1)(deny default)"
+
+      # sandbox_init error texts that say the facility is genuinely gone
+      # (a future macOS without Seatbelt): degrade, noted once. Anything
+      # else (a profile syntax tightening, EPERM, a lost child) fails
+      # closed: SandboxPolicy#seatbelt_hook refuses the run rather than
+      # confining nothing, exactly like the Landlock probe (#178) - never
+      # silently unsandboxed.
+      UNSUPPORTED_ERRORS = ["operation not supported", "function not implemented"].freeze
+
+      module_function
+
+      def darwin?
+        RUBY_PLATFORM.include?("darwin")
+      end
+
+      # One forked behavioral probe, memoized per process:
+      # :unsupported_platform off macOS, :ok when a deny-default profile
+      # really blocks a write, :unavailable when sandbox_init reports the
+      # facility missing, :unenforced when the API claims success but the
+      # write goes through, :error for anything else. Behavioral, not
+      # symbol presence: a macOS update that rejects the profile language
+      # shows up here, not in a dlopen.
+      def probe
+        return @probe if defined?(@probe)
+        return @probe = :unsupported_platform unless darwin?
+
+        @probe = run_probe
+      rescue Fiddle::DLError
+        @probe = :unavailable
+      end
+
+      def available?
+        probe == :ok
+      end
+
+      # Called at the top of a mutant child, mirroring Sandbox.restrict!:
+      # irreversible and fail-closed - a child whose profile does not apply
+      # must not run the mutant. keep_fds is Sandbox's hook contract
+      # (#234): Landlock gates opens, so its children keep writing through
+      # descriptors the parent opened (their logs); Seatbelt checks
+      # file-write* on EVERY write, so those inherited files join the
+      # profile as literal grants instead. Pipes and sockets among them
+      # have no path and need no grant.
+      def restrict!(writable:, keep_fds: [])
+        profile = Profile.build(writable: writable.map { |dir| resolved_grant(dir) },
+                                inherited_files: fd_paths(keep_fds))
+        rc, message = sandbox_init(profile)
+        unless rc.zero?
+          raise Error, "the Seatbelt sandbox refused to engage (#{message}); the mutant does not run unconfined"
+        end
+
+        nil
+      end
+
+      # Seatbelt compares the kernel's resolved path, so a grant spelled
+      # through a symlink (macOS /var is /private/var) never matches and the
+      # "granted" write is denied. Resolve what exists; a dir not yet
+      # created grants nothing either way, so keep its spelling rather than
+      # fail the child on a dead grant. fcntl(F_GETPATH) already returns
+      # the resolved path for inherited files, so they need no such pass.
+      def resolved_grant(dir)
+        File.exist?(dir) ? File.realpath(dir) : dir
+      end
+
+      # fcntl(F_GETPATH) maps an inherited descriptor back to its path; a
+      # descriptor without one (pipe, socket) is skipped.
+      def fd_paths(keep_fds)
+        keep_fds.filter_map do |fd|
+          buffer = Fiddle::Pointer.malloc(PATH_MAX)
+          LibSystem.fcntl_fn.call(fd, F_GETPATH, buffer).negative? ? nil : buffer.to_s
+        end
+      end
+
+      def classify(verdict)
+        case verdict
+        when "ok" then :ok
+        when "unenforced" then :unenforced
+        when /\Ainit_failed:(?<message>.*)\z/m
+          message = Regexp.last_match[:message].downcase
+          UNSUPPORTED_ERRORS.any? { |text| message.include?(text) } ? :unavailable : :error
+        else
+          :error
+        end
+      end
+
+      def run_probe
+        reader, writer = IO.pipe
+        pid = fork { probe_child(writer) }
+        writer.close
+        verdict = reader.read
+        reader.close
+        Process.waitpid(pid)
+        classify(verdict)
+      ensure
+        reader.close unless reader.closed?
+        writer.close unless writer.closed?
+      end
+
+      # Child only: confine, then attempt the write the profile must block.
+      # The parent reads a single verdict word; exit! so no at_exit hook or
+      # finalizer runs under the sandbox.
+      def probe_child(writer)
+        rc, message = sandbox_init(PROBE_PROFILE)
+        verdict =
+          if rc.zero?
+            begin
+              File.open(File::NULL, "w") { true }
+              "unenforced"
+            rescue SystemCallError, IOError
+              "ok"
+            end
+          else
+            "init_failed:#{message}"
+          end
+        writer.write(verdict)
+        writer.close
+        exit!(0)
+      end
+
+      # int sandbox_init(const char *profile, uint64_t flags, char **errorbuf)
+      def sandbox_init(profile)
+        errbuf = Fiddle::Pointer.malloc(Fiddle::SIZEOF_VOIDP)
+        rc = LibSandbox.sandbox_init_fn.call(profile, 0, errbuf)
+        return [0, nil] if rc.zero?
+
+        # errorbuf is a char**: dereference once to reach the message.
+        message = begin
+          address = errbuf[0, Fiddle::SIZEOF_VOIDP].unpack1("J")
+          address&.positive? ? Fiddle::Pointer.new(address).to_s : "unknown"
+        rescue StandardError
+          "unknown"
+        end
+        [rc, "sandbox_init: #{message}"]
+      end
+
+      # fcntl for the F_GETPATH lookup; lazy like LibSandbox.
+      module LibSystem
+        module_function
+
+        def fcntl_fn
+          @fcntl_fn ||= Fiddle::Function.new(
+            Fiddle.dlopen(Seatbelt::LIBSYSTEM)["fcntl"],
+            [Fiddle::TYPE_INT, Fiddle::TYPE_INT, Fiddle::TYPE_VOIDP],
+            Fiddle::TYPE_INT
+          )
+        end
+      end
+
+      # Lazy, so requiring the file on Linux or a stripped macOS stays inert
+      # (mirrors Sandbox::LibC, #178).
+      module LibSandbox
+        module_function
+
+        def handle
+          @handle ||= Fiddle.dlopen(Seatbelt::LIBSANDBOX)
+        end
+
+        def sandbox_init_fn
+          @sandbox_init_fn ||= Fiddle::Function.new(
+            handle["sandbox_init"],
+            [Fiddle::TYPE_VOIDP, Fiddle::TYPE_LONG_LONG, Fiddle::TYPE_VOIDP],
+            Fiddle::TYPE_INT
+          )
+        end
+      end
+
+      # A Landlock-shaped Seatbelt profile: reads, execution, network and
+      # IPC stay allowed everywhere (allow default); only writes are
+      # denied, then re-allowed under the granted dirs and /dev/null -
+      # exactly the Sandbox grant semantics. allow-default is deliberate:
+      # deny-default profiles break on macOS updates whenever Apple adds a
+      # mach service the profile did not name, the failure mode behind the
+      # macOS 26 sandbox breakage reports (#436).
+      module Profile
+        HEADER = "(version 1)\n"
+
+        module_function
+
+        # inherited_files: already-open logs the child must keep writing
+        # (#234) - granted per file, never the cache dir that holds them.
+        def build(writable:, inherited_files: [])
+          lines = [HEADER, "(allow default)\n", "(deny file-write*)\n"]
+          writable.each do |dir|
+            lines << %((allow file-write* (subpath "#{escape(dir)}"))\n)
+          end
+          (inherited_files + [File::NULL]).uniq.each do |file|
+            lines << %((allow file-write* (literal "#{escape(file)}"))\n)
+          end
+          lines.join
+        end
+
+        # SBPL string literals are double-quoted with backslash escapes; a
+        # grant path must never inject profile source.
+        def escape(path)
+          path.gsub(/["\\]/) { |char| "\\#{char}" }
+        end
+      end
+    end
+  end
+end