saboteur 0.3.0 → 0.5.0

data/lib/saboteur/execution/runner.rb

@@ -2,6 +2,7 @@
 # frozen_string_literal: true
 
 require "json"
+require_relative "stale_captures"
 
 module Saboteur
   module Execution
@@ -14,8 +15,12 @@ module Saboteur
     class Runner
       # kill_cause: "assertion" when the killing test failed an expectation,
       # else the class of the error it died with (a crash, not a check).
-      Outcome = Data.define(:status, :runtime, :failed_ids, :message, :leftover_pids, :kill_cause) do
-        def initialize(status:, runtime:, failed_ids: [], message: nil, leftover_pids: [], kill_cause: nil)
+      # executed_ids: the tests the child reports it ran; empty when it could
+      # not say (a test called exit).
+      Outcome = Data.define(:status, :runtime, :failed_ids, :message, :leftover_pids, :kill_cause,
+                            :executed_ids) do
+        def initialize(status:, runtime:, failed_ids: [], message: nil, leftover_pids: [], kill_cause: nil,
+                       executed_ids: [])
           super
         end
       end
@@ -27,7 +32,12 @@ module Saboteur
       ].freeze
 
       # Everything one child needs. mutation is nil for the verdict check.
-      Job = Data.define(:subject, :text, :test_ids, :timeout, :mutation, :log_path)
+      # eval_prefix: wrapper source (a sorbet sig) evaluated with the def.
+      Job = Data.define(:subject, :text, :test_ids, :timeout, :mutation, :log_path, :eval_prefix) do
+        def initialize(subject:, text:, test_ids:, timeout:, log_path:, mutation: nil, eval_prefix: nil)
+          super
+        end
+      end
 
       LOG_TAIL_BYTES = 2000
 
@@ -35,13 +45,18 @@ module Saboteur
       # only when it ends in an error.
       # seed: every child seeds Ruby's random numbers with it before its
       # tests, so a test that uses rand sees the same values in every run.
-      def initialize(integration:, reevaluator:, log_dir:, hooks: Hooks.new, supervisor: Supervisor.new, seed: 1)
+      # sandbox: when given, a callable run first in every child (Landlock
+      # makes the project tree read-only there, #152). It raises rather than
+      # run half-restricted, which the child reports as an error.
+      def initialize(integration:, reevaluator:, log_dir:, hooks: Hooks.new, supervisor: Supervisor.new, seed: 1,
+                     sandbox: nil)
         @integration = integration
         @reevaluator = reevaluator
         @hooks = hooks
         @log_dir = log_dir
         @supervisor = supervisor
         @seed = seed
+        @sandbox = sandbox
       end
 
       # text: method source to evaluate in the child (the unchanged source for
@@ -49,10 +64,10 @@ module Saboteur
       # the neutral check). Returns an Outcome with status
       # :killed, :survived, :timeout, :resource_limit (memory or process cap) or
       # :error.
-      def run(subject:, text:, test_ids:, timeout:, mutation: nil)
+      def run(subject:, text:, test_ids:, timeout:, mutation: nil, eval_prefix: nil)
         @hooks.run(:before_fork, mutation)
         @sequence = @sequence.to_i + 1
-        job = Job.new(subject:, text:, test_ids:, timeout:, mutation:,
+        job = Job.new(subject:, text:, test_ids:, timeout:, mutation:, eval_prefix:,
                       log_path: File.join(@log_dir, "child-#{Process.pid}-#{@sequence}.log"))
         result = @supervisor.run(timeout: timeout) { |writer| child(writer, job) }
         outcome = outcome_for(result, job)
@@ -65,21 +80,67 @@ module Saboteur
       # --- child ----------------------------------------------------------
 
       def child(writer, job)
-        Process.setproctitle(title(job))
+        # Redirect before restricting: the sweep closes inherited stdio
+        # redirected to a regular file outside the grants (#178), and a
+        # reopened-afterwards $stdout would start from a closed stream.
+        # The log lives under the granted cache dir, so it survives.
         redirect_output(job.log_path)
+        @sandbox&.call([writer.fileno])
+        Process.setproctitle(title(job))
         @hooks.run(:after_fork)
-        @reevaluator.call(job.subject, job.text)
+        # Before the evaluation: a whole-file subject runs the file's own
+        # top-level code there, and it must see the run's random numbers.
+        Kernel.srand(@seed)
+        evaluate_mutant(writer, job)
+        stale = stale_captures(job)
+        Protocol.emit(writer, "captures", site: stale.first, stale: stale.size) if stale.any?
         Protocol.emit(writer, "started")
         Kernel.srand(@seed)
-        result = @hooks.around(job.mutation) { run_tests(job.test_ids) }
+        emit_result(writer, job, stale.any?)
+      rescue Exception => e # rubocop:disable Lint/RescueException
+        Protocol.emit(writer, "error", message: "#{e.class}: #{e.message}", backtrace: e.backtrace&.first(5))
+        exit!(1)
+      end
+
+      def evaluate_mutant(writer, job)
+        @reevaluator.call(job.subject, job.text, prefix: job.eval_prefix)
+        replaced = @reevaluator.replaced_definition(job.subject)
+        Protocol.emit(writer, "replaced", site: replaced) if replaced
+      end
+
+      # Capture detection is best-effort: a subject shape it cannot scan
+      # must never turn the mutant run itself into an error.
+      def stale_captures(job)
+        StaleCaptures.scan(@reevaluator, job.subject)
+      rescue StandardError
+        []
+      end
+
+      def emit_result(writer, job, has_captures)
+        method = replacement_method(job) if has_captures && job.mutation
+        called = method ? false : nil
+        trace = TracePoint.new(:call) { called = true } if method
+        result = if trace
+                   trace.enable(target: method) { @hooks.around(job.mutation) { run_tests(job.test_ids) } }
+                 else
+                   @hooks.around(job.mutation) { run_tests(job.test_ids) }
+                 end
         if result.nil?
           Protocol.emit(writer, "error", message: "around_mutant hook did not run the tests")
         else
-          Protocol.emit(writer, "result", **result)
+          Protocol.emit(writer, "result", **result, replacement_called: called)
         end
-      rescue Exception => e # rubocop:disable Lint/RescueException
-        Protocol.emit(writer, "error", message: "#{e.class}: #{e.message}", backtrace: e.backtrace&.first(5))
-        exit!(1)
+      end
+
+      # Targeted tracing distinguishes the new Ruby body from an old Method
+      # capture, even though both have the same owner, name and source line.
+      def replacement_method(job)
+        owner = @reevaluator.resolve_owner(job.subject)
+        target = job.subject.singleton ? owner.singleton_class : owner
+        method = target.instance_method(job.subject.name)
+        Wrapper.original_def(method) || method
+      rescue StandardError
+        nil
       end
 
       # What `ps` shows for this child: which method, and which mutant.
@@ -124,7 +185,7 @@ module Saboteur
         error = events.find { |event| event["event"] == "error" }
 
         if result
-          classify_result(result, runtime, job.test_ids)
+          classify_events(result, events, runtime, job)
         else
           message = if error
                       error["message"]
@@ -145,17 +206,59 @@ module Saboteur
           unexpected = failed - test_ids
           return mismatch(runtime, "failed tests not requested: #{unexpected.first(3).join(', ')}") if unexpected.any?
 
-          return Outcome.new(status: :killed, runtime: runtime, failed_ids: failed,
+          return Outcome.new(status: :killed, runtime: runtime, failed_ids: failed, executed_ids: executed,
                              message: result.fetch("failure_messages", {})[failed.first],
                              kill_cause: kill_cause(result.fetch("failure_classes", {})[failed.first]))
         end
-        return Outcome.new(status: :killed, runtime: runtime, message: "suite-level failure") unless result["passed"]
+        unless result["passed"]
+          return Outcome.new(status: :killed, runtime: runtime, executed_ids: executed,
+                             message: "suite-level failure")
+        end
 
         if executed.sort != test_ids.sort
           return mismatch(runtime, "executed #{executed.size} of #{test_ids.size} requested tests")
         end
 
-        Outcome.new(status: :survived, runtime: runtime)
+        Outcome.new(status: :survived, runtime: runtime, executed_ids: executed)
+      end
+
+      def classify_events(result, events, runtime, job)
+        outcome = classify_result(result, runtime, job.test_ids)
+        replaced = events.find { |event| event["event"] == "replaced" }
+        return replaced_by_later_definition(outcome, replaced) if never_ran?(outcome, replaced, job)
+
+        captures = events.find { |event| event["event"] == "captures" }
+        return unreachable(outcome, captures) if unobservable?(outcome, captures, job, result["replacement_called"])
+
+        outcome
+      end
+
+      # A surviving mutant whose definition a later one in the same file
+      # replaced: the tests could not have run it, so it is no verdict on them.
+      def never_ran?(outcome, replaced, job)
+        replaced && job.mutation && outcome.status == :survived
+      end
+
+      def replaced_by_later_definition(outcome, replaced)
+        Outcome.new(status: :unreachable, runtime: outcome.runtime,
+                    message: "the whole-file evaluation left another definition in place " \
+                             "(#{replaced['site']}), so this mutant never ran")
+      end
+
+      # Only mutant runs convert: the verdict check re-evaluates the
+      # unchanged method, and a stale capture there must not overturn a real
+      # kill.
+      def unobservable?(outcome, captures, job, replacement_called)
+        captures && job.mutation && outcome.status == :survived && replacement_called == false
+      end
+
+      # A survivor that no test could have observed: a Method object
+      # captured at load still calls the original. Not a verdict on the
+      # tests, so it is its own status, out of the score.
+      def unreachable(outcome, captures)
+        Outcome.new(status: :unreachable, runtime: outcome.runtime,
+                    message: "a Method object captured before re-evaluation still calls the original " \
+                             "(captured definition at #{captures['site']})")
       end
 
       def kill_cause(failure_class)

data/lib/saboteur/execution/sandbox.rb

@@ -0,0 +1,271 @@
+# SPDX-License-Identifier: MIT
+# frozen_string_literal: true
+
+require "fiddle"
+require "fiddle/import"
+
+module Saboteur
+  module Execution
+    # The third hazard layer (issue #152): a mutant child restricts itself
+    # with Landlock so the project tree is read-only, except the cache dir,
+    # the project's tmp/ and the system temp dir.
+    #
+    # Landlock needs no privileges and works from a multithreaded process,
+    # which a mount/user namespace cannot (Ruby's timer thread makes
+    # unshare(CLONE_NEWUSER) fail with EINVAL). What it guarantees depends
+    # on the kernel's Landlock ABI: ABI 3 (Linux 6.2) also restricts
+    # truncate(); on ABI 1-2 (Linux 5.13-6.1) a mutant can still truncate
+    # project files, and the run notes that once. Where the kernel has no
+    # Landlock at all (before 5.13) the child runs unsandboxed, exactly as
+    # before - and a probe that errors for any other reason refuses the
+    # run instead of silently dropping the sandbox. Landlock restricts the
+    # filesystem only: network, sockets and IPC are never covered.
+    #
+    # libc is dlopened lazily on first use, and first use is always behind
+    # the linux? gate: requiring this file (which saboteur.rb does
+    # unconditionally) must stay inert on macOS and non-glibc Linux, where
+    # libc.so.6 does not exist (#178).
+    module Sandbox
+      SYS_CREATE_RULESET = 444
+      SYS_ADD_RULE = 445
+      SYS_RESTRICT_SELF = 446
+      CREATE_RULESET_VERSION = 1
+      RULE_PATH_BENEATH = 1
+      PR_SET_NO_NEW_PRIVS = 38
+      O_PATH = 0o10000000
+      O_ACCMODE = 0o3
+      F_GETFL = 3
+
+      # Every write-side right, so reads and execution keep working anywhere.
+      BASE_RIGHTS = (1 << 1) | (1 << 4) | (1 << 5) | (1 << 6) | (1 << 7) | (1 << 8) |
+                    (1 << 9) | (1 << 10) | (1 << 11) | (1 << 12)
+      REFER = 1 << 13 # ABI 2 (5.19)
+      TRUNCATE = 1 << 14 # ABI 3 (6.2)
+      WRITE_FILE = 1 << 1
+
+      # A probe answered with these says Landlock is genuinely absent.
+      UNSUPPORTED_ERRNOS = [Errno::ENOSYS::Errno, Errno::EOPNOTSUPP::Errno].freeze
+
+      module_function
+
+      def linux?
+        RUBY_PLATFORM.include?("linux")
+      end
+
+      # The kernel's Landlock ABI (0 when unsupported), memoized per process.
+      def abi_version
+        return @abi_version if defined?(@abi_version)
+        return @abi_version = 0 unless linux?
+
+        version = LibC.syscall_fn.call(SYS_CREATE_RULESET, 0, 0, CREATE_RULESET_VERSION, 0)
+        errno = last_errno
+        @abi_version = probe_abi(version, errno)
+      rescue Fiddle::DLError
+        # No usable libc (an unusual musl/ldso layout): the sandbox cannot
+        # be built here, so degrade to the unsandboxed note, not a crash.
+        @abi_version = 0
+      end
+
+      # A positive answer is the ABI. ENOSYS/EOPNOTSUPP mean the kernel has
+      # no Landlock to give: run unsandboxed, noted once. Anything else
+      # (EPERM under a seccomp policy, ...) means the kernel is there but
+      # refused - fail closed rather than run without the promised sandbox.
+      def probe_abi(version, errno)
+        return version if version.positive?
+        return 0 if UNSUPPORTED_ERRNOS.include?(errno)
+
+        raise SystemCallError.new("landlock_create_ruleset probe", errno)
+      end
+
+      def available?
+        abi_version >= 1
+      end
+
+      # The rights mask this kernel understands; newer rights are gated by
+      # the probed ABI so old kernels do not reject the ruleset as invalid.
+      def rights_mask
+        mask = BASE_RIGHTS
+        mask |= REFER if abi_version >= 2
+        mask |= TRUNCATE if abi_version >= 3
+        mask
+      end
+
+      # Called at the top of a mutant child. Irreversible and fail-closed: a
+      # child whose sandbox cannot be fully built must not run the mutant.
+      # keep_fds: descriptors the child still needs after restricting (its
+      # report pipe to the parent).
+      def restrict!(writable:, keep_fds: [])
+        InheritedFds.close_writable_files(writable, keep_fds)
+        ruleset = create_ruleset
+        begin
+          writable.each { |dir| allow(ruleset, dir, rights_mask) }
+          allow(ruleset, File::NULL, WRITE_FILE)
+          check("prctl(PR_SET_NO_NEW_PRIVS)") { LibC.prctl_fn.call(PR_SET_NO_NEW_PRIVS, 1, 0, 0, 0) }
+          check("landlock_restrict_self") { LibC.syscall_fn.call(SYS_RESTRICT_SELF, ruleset, 0, 0, 0) }
+        ensure
+          LibC.close_fn.call(ruleset)
+        end
+        nil
+      end
+
+      def create_ruleset
+        attr = [rights_mask].pack("Q<")
+        fd = LibC.syscall_fn.call(SYS_CREATE_RULESET, Fiddle::Pointer[attr], attr.bytesize, 0, 0)
+        errno = last_errno
+        check_fd("landlock_create_ruleset", fd, errno)
+        fd
+      end
+
+      def allow(ruleset, path, rights)
+        parent = LibC.open_fn.call(path, O_PATH)
+        errno = last_errno
+        check_fd("open #{path}", parent, errno)
+        attr = "#{[rights].pack('Q<')}#{[parent].pack('l<')}\0\0\0\0"
+        check("landlock_add_rule #{path}") do
+          LibC.syscall_fn.call(SYS_ADD_RULE, ruleset, RULE_PATH_BENEATH, Fiddle::Pointer[attr], 0)
+        end
+      ensure
+        LibC.close_fn.call(parent) if parent && parent >= 0
+      end
+
+      # Landlock only gates opens after restrict: an already open writable
+      # file keeps writing through its descriptor. Close every inherited
+      # regular-file descriptor open for writing outside the granted dirs.
+      # Sockets, pipes and anonymous inodes have no path and are untouched.
+      module InheritedFds
+        module_function
+
+        # stdio (0/1/2) is swept like any other descriptor: a terminal,
+        # pipe or /dev/null is not a regular file and stays, but stdio
+        # redirected to a writable regular file outside the grants is a
+        # write channel Landlock cannot gate, so it is closed.
+        def close_writable_files(writable, keep_fds)
+          Dir.children("/proc/self/fd").each do |entry|
+            descriptor = entry.to_i
+            next if keep_fds.include?(descriptor)
+
+            sweep_fd(descriptor, File.join("/proc/self/fd", entry), writable)
+          end
+          nil
+        end
+
+        # Fail closed throughout: a descriptor whose target or flags cannot
+        # be read may be a writable file we cannot see, so it is closed
+        # rather than skipped. Deleted files still stat through the fd (the
+        # inode lives until the last descriptor closes), so they are swept
+        # on their stripped path.
+        def sweep_fd(descriptor, link, writable)
+          target = File.readlink(link)
+          stat = File.stat(link)
+          return unless stat.file?
+
+          granted = target.delete_suffix(" (deleted)")
+          return if writable.any? { |dir| granted == dir || granted.start_with?("#{dir}/") }
+
+          flags = LibC.fcntl_fn.call(descriptor, F_GETFL, 0)
+          return close_fd(descriptor) if flags.negative?
+          return if flags.nobits?(O_ACCMODE) # O_RDONLY
+
+          close_fd(descriptor)
+        rescue SystemCallError
+          begin
+            close_fd(descriptor)
+          rescue IOError, SystemCallError
+            nil # the descriptor raced away; nothing left to close
+          end
+        end
+
+        # Close through the owning Ruby IO when there is one: a raw close
+        # leaves the IO object holding a dead fd, whose number a later open
+        # recycles - and the stale IO's finalizer would then close the NEW
+        # file (seen as corrupted writes in the spec suite).
+        def close_fd(descriptor)
+          owner = ObjectSpace.each_object(IO).find do |io|
+            !io.closed? && io.fileno == descriptor
+          rescue IOError
+            false
+          end
+          owner ? owner.close : LibC.close_fn.call(descriptor)
+        end
+      end
+
+      def check(what)
+        result = yield
+        return if result.zero?
+
+        raise SystemCallError.new(what, last_errno)
+      end
+
+      def check_fd(what, handle, errno)
+        raise SystemCallError.new(what, errno) if handle.negative?
+      end
+
+      # Read errno before anything else can clobber it: any Ruby allocation
+      # between the failed call and this read can reset it to 0.
+      def last_errno
+        LibC.errno_location_fn.call.to_s(4).unpack1("i") # errno is an int, not a long (LP64)
+      end
+
+      # Every libc touchpoint, each built on first call - never at require
+      # time, so loading this file stays inert off glibc/Linux.
+      module LibC
+        module_function
+
+        # glibc's soname first, then musl's (arch-qualified), then the
+        # process global symbol table: a musl system has no libc.so.6, but
+        # its loader already mapped the C library Ruby is linked against,
+        # and Fiddle::Handle::DEFAULT resolves symbols from there.
+        def handle
+          @handle ||= libc_candidates.lazy.map { |name| try_dlopen(name) }.find(&:itself) ||
+                      Fiddle::Handle::DEFAULT
+        end
+
+        def libc_candidates
+          arch = RbConfig::CONFIG["host_cpu"]
+          ["libc.so.6", "libc.musl-#{arch}.so.1", "ld-musl-#{arch}.so.1"]
+        end
+
+        def try_dlopen(name)
+          Fiddle.dlopen(name)
+        rescue Fiddle::DLError
+          nil
+        end
+
+        def syscall_fn
+          @syscall_fn ||= Fiddle::Function.new(
+            handle["syscall"],
+            [Fiddle::TYPE_LONG, Fiddle::TYPE_LONG, Fiddle::TYPE_LONG, Fiddle::TYPE_LONG, Fiddle::TYPE_LONG],
+            Fiddle::TYPE_LONG
+          )
+        end
+
+        def prctl_fn
+          @prctl_fn ||= Fiddle::Function.new(
+            handle["prctl"],
+            [Fiddle::TYPE_INT, Fiddle::TYPE_LONG, Fiddle::TYPE_LONG, Fiddle::TYPE_LONG, Fiddle::TYPE_LONG],
+            Fiddle::TYPE_INT
+          )
+        end
+
+        def open_fn
+          @open_fn ||=
+            Fiddle::Function.new(handle["open"], [Fiddle::TYPE_VOIDP, Fiddle::TYPE_INT], Fiddle::TYPE_INT)
+        end
+
+        def close_fn
+          @close_fn ||= Fiddle::Function.new(handle["close"], [Fiddle::TYPE_INT], Fiddle::TYPE_INT)
+        end
+
+        def fcntl_fn
+          @fcntl_fn ||= Fiddle::Function.new(
+            handle["fcntl"], [Fiddle::TYPE_INT, Fiddle::TYPE_INT, Fiddle::TYPE_LONG], Fiddle::TYPE_INT
+          )
+        end
+
+        def errno_location_fn
+          @errno_location_fn ||= Fiddle::Function.new(handle["__errno_location"], [], Fiddle::TYPE_VOIDP)
+        end
+      end
+    end
+  end
+end

data/lib/saboteur/execution/stale_captures.rb

@@ -0,0 +1,43 @@
+# SPDX-License-Identifier: MIT
+# frozen_string_literal: true
+
+module Saboteur
+  module Execution
+    # Method and UnboundMethod objects captured before a re-evaluation keep
+    # pointing at the original body; a handler registry such as
+    # `RULES = [Limits.method(:min_ok?)]` is the common case. Tests that call
+    # through such a capture can never observe a mutant, so after each
+    # re-evaluation the child scans for captures of the original def.
+    module StaleCaptures
+      # "file:line" sites of the original def for which a stale capture
+      # exists, deduplicated. Empty when nothing captured the method.
+      def self.scan(reevaluator, subject)
+        owner = reevaluator.resolve_owner(subject)
+        target = subject.singleton ? owner.singleton_class : owner
+        current = target.instance_method(subject.name)
+        # A def-time wrapper (sorbet's sig) is what instance_method returns;
+        # the def it dispatches to is the definition captures can hold.
+        current = Wrapper.original_def(current) || current
+        # The scan must not mistake transient Method objects of its own
+        # making (the re-eval, this instance_method) for captures; a full GC
+        # clears them, and only long-lived captures from load time remain.
+        GC.start
+        stale = [UnboundMethod, Method].flat_map do |kind|
+          ObjectSpace.each_object(kind).select do |captured|
+            stale?(captured, target, subject, current)
+          end
+        end
+        stale.map { |captured| captured.source_location.join(":") }.uniq
+      end
+
+      def self.stale?(captured, target, subject, current)
+        return false unless captured.owner == target && captured.name == subject.name
+        return false if captured == current
+
+        file, line = captured.source_location
+        file && line == subject.start_line &&
+          Selection::CoverageMap.canonical(file) == Selection::CoverageMap.canonical(subject.file)
+      end
+    end
+  end
+end