mutineer-mongoid 2.0.0

data/lib/mutineer/daemon_server.rb

@@ -0,0 +1,351 @@
+# frozen_string_literal: true
+
+require "json"
+require "tempfile"
+require_relative "child_stdout"
+require_relative "orphan_guard"
+
+module Mutineer
+  # App-side daemon (persistent worker).
+  #
+  # Runs UNDER THE APP'S OWN BUNDLE/RUBY (the tool's DaemonClient spawns it via
+  # `bundle exec ruby`). It boots the app ONCE, then serves per-mutant test-run
+  # requests over stdin/stdout as newline-delimited JSON (the protocol keeps a
+  # private copy of the original stdout; the app's own stdout goes to stderr).
+  # For each request it FORKS a child that loads the mutated source text the
+  # tool sent, runs the covering tests, and exits with a status the parent
+  # decodes into a verdict.
+  #
+  # HARD CONSTRAINT: this file must be loadable WITHOUT Prism or the rest of
+  # mutineer. The app's Ruby may be < 3.4 (no stdlib Prism) and its bundle has no
+  # mutineer. So it requires ONLY stdlib + the app's own boot file; it
+  # re-implements the fork/timeout/decode loop rather than requiring
+  # `isolation.rb` (which pulls in Prism). All parsing/mutation happened
+  # tool-side; the daemon only `load`s text.
+  #
+  # Protocol (one JSON object per line, both directions):
+  #   boot in  : {"cmd":"boot","project_root":"...","boot":"config/environment","require_paths":[...],
+  #               "load_paths":["test"],"framework":"minitest","rails":true,"schema":"db/schema.rb"}
+  #   ready out: {"ready":true,"ruby":"3.3.6"}   (or {"ready":false,"error":"..."} then exit)
+  #   run  in  : {"id":N,"worker":I,"payload":{"code":"<ruby>","source_file":"app/models/order.rb"},
+  #               "tests":["test/models/order_test.rb"],"timeout":30}
+  #   verdict  : {"id":N,"verdict":"survived"|"killed"|"error"|"timeout"}
+  #   quit in  : {"cmd":"quit"}
+  #
+  # Worker isolation: when the app is Rails, each fork is routed to its own
+  # database `<db>-<worker>` via {RailsWorkerDb} BEFORE any test loads, so
+  # concurrent workers cannot clobber each other's transactional fixtures.
+  # `worker` defaults to 0 (serial). SQLite this pass; Postgres provisioning is
+  # not yet implemented.
+  #
+  # Verdict mapping: child exit 0=survived (suite passed), 1=killed (suite
+  # failed), 2=error (child raised AROUND the test: load, boot, or worker-DB
+  # routing failure); parent-detected timeout. Tagging an in-test DB error (one
+  # fired inside a test body, vs at routing time) as `error` rather than
+  # `killed` is only observable under the concurrent gate and is not yet
+  # implemented.
+  module DaemonServer
+    # Poll interval (seconds) for the per-fork deadline wait loop.
+    POLL = 0.02
+
+    class << self
+      # Serve the protocol on the given IO pair (defaults to stdio). Returns on quit.
+      #
+      # @param input [IO] request stream.
+      # @param output [IO, nil] verdict stream. nil (the default) reserves the
+      #   process's original stdout for the protocol and points fd 1 at stderr.
+      # @param errio [IO] diagnostics stream (never the IPC channel).
+      # @return [void]
+      def run(input: $stdin, output: nil, errio: $stderr)
+        # Only a stream `run` reserved itself is the daemon's to close in a fork.
+        @protocol = output ? nil : reserve_protocol_output
+        output ||= @protocol
+        @errio = errio
+        @output = output
+        boot_line = input.gets
+        return if boot_line.nil? # client vanished before boot
+
+        boot!(JSON.parse(boot_line.strip))
+        output.puts(JSON.generate("ready" => true, "ruby" => RUBY_VERSION))
+        output.flush
+
+        input.each_line do |line|
+          line = line.strip
+          next if line.empty?
+
+          begin
+            req = JSON.parse(line)
+          rescue JSON::ParserError => e
+            # A corrupt line has no id to address a reply to (and the client only
+            # ever sends valid JSON, so it cannot be a pending request). Log and
+            # read on rather than write an unaddressable verdict onto the channel.
+            @errio.puts("[daemon] dropped unparseable line: #{e.message}")
+            next
+          end
+          break if req["cmd"] == "quit"
+
+          # Build the coverage map app-side and ship it to the tool, which then
+          # selects covering tests per mutant. One-shot control message.
+          if req["cmd"] == "coverage"
+            output.puts(JSON.generate(build_coverage_map))
+            output.flush
+            next
+          end
+
+          output.puts(JSON.generate(run_mutant(req)))
+          output.flush
+        end
+      end
+
+      private
+
+      # Move the protocol channel off fd 1 before the app boots. The app may
+      # print through `puts`, `STDOUT`, or a raw fd 1 write (a logger, a
+      # subprocess). A line like that ahead of the ready message would break
+      # the client's JSON read. The protocol keeps a private dup of the
+      # original stdout. Fd 1 then points at stderr, so app output stays
+      # visible in the tool's diagnostics.
+      #
+      # @return [IO] the reserved protocol stream.
+      def reserve_protocol_output
+        protocol = STDOUT.dup
+        STDOUT.reopen(STDERR)
+        $stdout = STDOUT
+        protocol
+      end
+
+      # BOOT ONCE. chdir + require the app's boot file so the whole app is loaded
+      # and inherited by every fork. Never requires mutineer.
+      def boot!(cfg)
+        @cfg = cfg
+        @framework = cfg.fetch("framework", "minitest")
+        @source_dirs = Array(cfg["source_dirs"]).map { |d| File.expand_path(d) }
+        Dir.chdir(cfg["project_root"]) if cfg["project_root"]
+        ENV["RAILS_ENV"] ||= "test" if cfg["rails"]
+        Array(cfg["load_paths"]).each { |d| $LOAD_PATH.unshift(File.expand_path(d)) }
+        # Start Coverage BEFORE the app loads, so booted source lines are
+        # instrumented. The map build (build_via_fork) forks this booted parent.
+        if cfg["coverage"]
+          require "coverage"
+          Coverage.start(lines: true, methods: true)
+        end
+        # Clear any mutant tempfile a prior SIGKILLed timeout child orphaned in a
+        # source dir BEFORE the app boots. Zeitwerk would otherwise choke on the
+        # tempfile's non-constant name during autoload setup.
+        sweep_temps
+        require File.expand_path(cfg["boot"]) if cfg["boot"]
+        # The --require files, after the boot as in-process (Runner.execute), so
+        # the coverage peek sees what they ran at load.
+        Array(cfg["require_paths"]).each { |f| require File.expand_path(f) }
+        setup_worker_db(cfg) if cfg["rails"] || defined?(Mongoid::Clients)
+        if @framework == "rspec" && !mongoid_worker_db
+          raise "RSpec daemon execution currently requires Mongoid"
+        end
+      rescue Exception => e # rubocop:disable Lint/RescueException
+        # Boot failed (bad boot path, app error). Tell the client and exit so it
+        # can surface a clean error rather than hang on the handshake.
+        @output.puts(JSON.generate("ready" => false, "error" => "#{e.class}: #{e.message}"))
+        @output.flush
+        exit!(1)
+      end
+
+      # Load the per-worker DB adapter app-side (sibling gem file, by relative path
+      # so it bypasses the app bundle, like this daemon itself). Mongoid gets
+      # disposable child databases. ActiveRecord retains its SQLite slot adapter.
+      # A Mongoid setup failure aborts boot before any specs can run.
+      def setup_worker_db(cfg)
+        require_relative "rails_worker_db"
+        require_relative "mongoid_worker_db"
+        @worker_db = if MongoidWorkerDb.available?
+                       MongoidWorkerDb.new
+                     elsif RailsWorkerDb.available?
+                       RailsWorkerDb
+                     end
+        schema = cfg["schema"] && File.expand_path(cfg["schema"])
+        @schema_path = schema if schema && File.exist?(schema)
+        # Each worker slot is seeded once, on first use (not every mutant fork).
+        @slot_ready = {}
+      rescue LoadError => e
+        @errio.puts("[daemon] worker-DB routing unavailable: #{e.message}")
+        @worker_db = nil
+      end
+
+      # Build the coverage map app-side (Coverage was started at boot) and return
+      # it as `{map, failed_test_files, load_lines, load_methods}` for the tool to select covering tests.
+      # Capture forks route to worker 0's DB (isolated, serial). On any failure
+      # return an empty map + an error string. The tool then falls back to the
+      # full test set rather than mis-scoring everything as no_coverage.
+      def build_coverage_map
+        require_relative "coverage_map"
+        root = @cfg["project_root"] || Dir.pwd
+        cmap = CoverageMap.new(
+          source_paths: Array(@cfg["sources"]), test_paths: Array(@cfg["tests"]),
+          load_paths: Array(@cfg["load_paths"]), project_root: root,
+          boot_path: @cfg["boot"], require_paths: Array(@cfg["require_paths"]),
+          framework: @framework, cache_dir: @cfg["cache_dir"] || File.join(root, ".mutineer"),
+          capture_timeout: @cfg["capture_timeout"] || CoverageMap::DEFAULT_CAPTURE_TIMEOUT
+        ).build_via_fork(after_fork: coverage_after_fork,
+                         before_fork: mongoid_worker_db&.method(:before_fork),
+                         cleanup: mongoid_worker_db&.method(:cleanup),
+                         cache_key: mongoid_worker_db&.coverage_cache_key)
+        # Nothing reads Coverage after the map, so a mutant forked from this
+        # daemon does not pay for it (#228).
+        Coverage.suspend if Coverage.running?
+        { "map" => cmap.map, "failed_test_files" => cmap.failed_test_files,
+          "failed_clean_tests" => cmap.failed_clean_tests, "load_lines" => cmap.load_lines.to_a,
+          "load_methods" => cmap.load_methods.to_a }
+      rescue Exception => e # rubocop:disable Lint/RescueException
+        @errio.puts("[daemon] coverage build failed: #{e.class}: #{e.message}")
+        { "map" => {}, "failed_test_files" => [], "error" => "#{e.class}: #{e.message}" }
+      end
+
+      # Fork-safety hook for coverage capture. Each capture fork drops its copy
+      # of the protocol channel (see close_protocol) and, when the app has a
+      # worker-DB adapter, routes to a fresh copy of the base DB in worker 0's
+      # slot (captures run serially, so one worker is enough).
+      def coverage_after_fork
+        worker_db = @worker_db
+        schema = @schema_path
+        lambda do
+          close_protocol
+          worker_db&.after_fork(0, schema, seed: true)
+        end
+      end
+
+      # A forked child never answers on the protocol channel. Closing its
+      # inherited copy lets a daemon crash read as EOF on the client at once,
+      # not only after a slow or hung child exits. A caller-supplied output
+      # (for example STDOUT) is left open.
+      def close_protocol
+        @protocol&.close rescue nil # rubocop:disable Style/RescueModifier
+      end
+
+      # Fork a child to run one mutant in isolation; decode its exit into a verdict.
+      def run_mutant(req)
+        timeout = req.fetch("timeout", 30)
+        worker  = req.fetch("worker", 0)
+        # Seed the slot's DB until the first killed/survived fork for this worker slot.
+        first_use = @worker_db && !@slot_ready[worker]
+        daemon = Process.pid
+        mongoid_worker_db&.before_fork
+        pid = fork do
+          # New process group so a per-fork timeout can SIGKILL the whole subtree,
+          # and silence the child's stdout so test output stays out of the diagnostics.
+          Process.setpgid(0, 0) rescue nil # rubocop:disable Style/RescueModifier
+          OrphanGuard.start(daemon)
+          code =
+            begin
+              ChildStdout.silence
+              close_protocol
+              # Route THIS fork at its own worker database before any test loads.
+              # A routing failure raises here and is scored `error`, never a false verdict.
+              @worker_db&.after_fork(worker, first_use ? @schema_path : nil, seed: first_use)
+              apply_payload(req["payload"])
+              run_tests(Array(req["tests"]))
+            rescue Exception => e # rubocop:disable Lint/RescueException
+              @errio.puts("[daemon-child] #{e.class}: #{e.message}")
+              2
+            end
+          exit!(code)
+        end
+        verdict = wait_verdict(pid, timeout)
+        # Mark ready only when the child finished cleanly after seeding
+        # (killed/survived). Timeout can interrupt mid-copy; error is a routing
+        # failure. Both leave the slot unready so the next fork seeds again.
+        @slot_ready[worker] = true if first_use && %w[killed survived].include?(verdict)
+        # A SIGKILLed timeout child skipped its Tempfile unlink. Sweep the orphan
+        # so it cannot outlive the run or trip Zeitwerk on a later fork.
+        sweep_temps if verdict == "timeout"
+        { "id" => req["id"], "verdict" => verdict }
+      ensure
+        mongoid_worker_db&.cleanup(pid) if pid
+      end
+
+      # Mongoid owns disposable child databases; ActiveRecord keeps its existing
+      # slot lifecycle. Do not invoke Mongoid cleanup on an ActiveRecord adapter.
+      # @return [MongoidWorkerDb, nil]
+      def mongoid_worker_db
+        @worker_db if defined?(MongoidWorkerDb) && @worker_db.is_a?(MongoidWorkerDb)
+      end
+
+      # Remove orphaned mutant tempfiles from the source dirs (parent-side; the
+      # SIGKILL path cannot run the child's ensure). Mirrors JobPlan.sweep_orphans.
+      def sweep_temps
+        @source_dirs.to_a.each do |dir|
+          Dir.glob(File.join(dir, "mutineer_daemon*.rb")).each do |f|
+            File.unlink(f) rescue nil # rubocop:disable Style/RescueModifier
+          end
+        end
+      end
+
+      # Single-waiter deadline loop (mirrors Isolation.run and
+      # ExternalBackend.wait_with_timeout, re-implemented here because Isolation
+      # pulls in Prism which is forbidden app-side). NOTE: this is the 3rd copy of
+      # the waitpid2(WNOHANG)+deadline+pgroup-SIGKILL+decode discipline. A fix to
+      # the kill/reap/decode logic must be applied to all three in lockstep.
+      # CoverageMap#await_child applies the same deadline and group kill to
+      # coverage capture.
+      # SIGKILL the child's process group past the deadline; a signalled child
+      # (nil exitstatus) is `error`.
+      def wait_verdict(pid, timeout)
+        deadline = Process.clock_gettime(Process::CLOCK_MONOTONIC) + timeout
+        loop do
+          reaped, status = Process.waitpid2(pid, Process::WNOHANG)
+          if reaped
+            return { 0 => "survived", 1 => "killed" }.fetch(status.exitstatus, "error")
+          end
+          if Process.clock_gettime(Process::CLOCK_MONOTONIC) >= deadline
+            begin
+              Process.kill(:KILL, -pid)
+            rescue Errno::ESRCH, Errno::EPERM
+              Process.kill(:KILL, pid) rescue nil # rubocop:disable Style/RescueModifier
+            end
+            begin
+              _reaped, status = Process.waitpid2(pid)
+              if status && status.exited? && !status.signaled?
+                return { 0 => "survived", 1 => "killed" }.fetch(status.exitstatus, "error")
+              end
+            rescue Errno::ECHILD
+              # already reaped
+            end
+            return "timeout"
+          end
+          sleep POLL
+        end
+      end
+
+      # Write the tool-built mutated text beside the real source and `load` it,
+      # reopening the mutated class/method in THIS child only. It goes in the
+      # source file's directory (like Isolation.apply_whole_file) so a
+      # `require_relative` in the mutated source resolves against its real
+      # neighbours. Writing it to the tmpdir would LoadError on such files and
+      # score a spurious `error` that diverges from the in-process path. The
+      # Zeitwerk hazard (a stray `.rb` in an autoload dir) is handled by the
+      # boot/timeout `sweep_temps`, not by relocating the file. Same path for
+      # reload (whole file) and redefine (wrapped snippet).
+      def apply_payload(payload)
+        dir = File.dirname(File.expand_path(payload.fetch("source_file")))
+        Tempfile.create(["mutineer_daemon", ".rb"], dir) do |f|
+          f.write(payload.fetch("code"))
+          f.flush
+          load f.path
+        end
+      end
+
+      # Load the covering test files and run them; 0 = all passed (survived),
+      # 1 = a failure/error (killed). RSpec uses the existing fork-safe runner.
+      def run_tests(tests)
+        if @framework == "rspec"
+          require_relative "mongoid_failures"
+          return MongoidFailures.run(tests)
+        end
+        raise "unsupported framework #{@framework.inspect}" unless @framework == "minitest"
+
+        require "minitest"
+        require "rails/test_help" if defined?(Rails)
+        tests.each { |t| load File.expand_path(t) }
+        Minitest.run([]) ? 0 : 1
+      end
+    end
+  end
+end

data/lib/mutineer/external_backend.rb

@@ -0,0 +1,288 @@
+# frozen_string_literal: true
+
+require "shellwords"
+require "tempfile"
+require_relative "result"
+
+module Mutineer
+  # Raised when the smoke check (the unmutated suite) is not green, so the run
+  # aborts before scoring — a broken environment must never be reported as strong
+  # tests. The CLI maps this to a runtime error (exit 1), not a usage error.
+  class SmokeCheckError < StandardError; end
+
+  # External execution backend. Runs the user's `--test-command` as a subprocess
+  # in the app's own runtime (whatever Ruby its bundle resolves to), so mutineer
+  # (Ruby ≥ 3.4) can mutation-test apps pinned to an older Ruby.
+  #
+  # This is deliberately NOT a `TestRunners` framework adapter: those return an
+  # Integer 0/1 from inside a fork and are dispatched by framework name. This is a
+  # whole backend — it spawns a process, enforces a wall-clock timeout, and maps
+  # the exit status to a Result. The mapping is the same direction as in-process
+  # (suite passes => survived, suite fails => killed) but coarser: it cannot tell an
+  # infrastructure error from a genuine kill, so the smoke check (below) guards the
+  # persistent case and the score is disclosed as an upper bound.
+  #
+  # Child environment: bundler/gem env injected by Mutineer's Ruby is stripped, and
+  # version-manager concrete version bins (e.g. `~/.rbenv/versions/3.4.9/bin`) are
+  # removed from PATH so shims / `.ruby-version` can select the app's Ruby (#32).
+  module ExternalBackend
+    # Generous ceiling for the one-off smoke/calibration run (a cold app boot plus
+    # the full suite). The per-mutant timeout is derived from how long this took.
+    SMOKE_TIMEOUT = 900
+    # Poll interval for the deadline wait loop. Independent of Isolation's loop —
+    # this backend waits on an external process TREE, not an in-process fork.
+    POLL = 0.02
+
+    # PATH entries that pin a concrete Ruby under a version manager (ahead of
+    # shims). Optional trailing slash; normal bins are left alone.
+    VERSION_BIN_PATH = %r{
+      (?:
+        /\.rbenv/versions/[^/]+/bin
+        |/\.asdf/installs/ruby/[^/]+/bin
+        |/\.rubies/[^/]+/bin
+        |/rubies/[^/]+/bin
+      )/?\z
+    }x
+
+    # Env keys Process.spawn must unset (nil value) so Mutineer's Ruby/bundler
+    # context does not leak. Spawn merges env onto the parent; omitting a key
+    # does NOT clear it.
+    CLEAR_ENV_KEYS = %w[
+      BUNDLER_VERSION RBENV_VERSION ASDF_RUBY_VERSION RBENV_DIR
+    ].freeze
+
+    # Turn a command template into an argv array (no shell → no eval, no
+    # injection). The <tt>%{files}</tt> token expands IN PLACE to N separate argv
+    # elements — one per path, unescaped — so a path containing a space stays a
+    # single argument. It is not a space-joined string.
+    #
+    # @param command [String] the --test-command template (contains <tt>%{files}</tt>).
+    # @param files [Array<String>] test file paths to substitute.
+    # @return [Array<String>] argv.
+    def self.build_argv(command, files)
+      Shellwords.split(command).flat_map { |tok| tok == "%{files}" ? files : [tok] }
+    end
+
+    # Environment delta for the test-command child. Process.spawn merges this
+    # onto the parent env: set a key to +nil+ to unset it. Scrubs Mutineer/bundler
+    # injection and version-manager PATH pins so shims / `.ruby-version` can win.
+    # Keeps other vars (e.g. `RAILS_ENV`) so setting them on the Mutineer command
+    # still reaches the suite.
+    #
+    # @api private
+    # @return [Hash{String => String, nil}] env for Process.spawn.
+    def self.child_env
+      env = {}
+      cleared_pin = false
+      ENV.each_key do |k|
+        next unless clear_env_key?(k)
+
+        env[k] = nil
+        cleared_pin = true
+      end
+      path = ENV["PATH"].to_s.split(File::PATH_SEPARATOR)
+      kept = path.reject { |p| p.match?(VERSION_BIN_PATH) }
+      scrubbed_bin = kept.size != path.size
+      # Only reorder PATH when we scrubbed a pin; otherwise leave the user's PATH alone.
+      path = if cleared_pin || scrubbed_bin
+               (version_manager_shim_dirs + kept).uniq
+             else
+               kept
+             end
+      env["PATH"] = path.join(File::PATH_SEPARATOR)
+      env
+    end
+
+    # True when the key must be unset in the child (bundler/gem/version-manager).
+    #
+    # @api private
+    # @param key [String] environment variable name.
+    # @return [Boolean]
+    def self.clear_env_key?(key)
+      key.start_with?("BUNDLE_", "RUBY", "GEM_") || CLEAR_ENV_KEYS.include?(key)
+    end
+
+    # Existing shim directories for rbenv / asdf (chruby uses PATH without shims).
+    #
+    # @api private
+    # @return [Array<String>] absolute shim paths that exist.
+    def self.version_manager_shim_dirs
+      home = ENV["HOME"]
+      return [] if home.nil? || home.empty?
+
+      [
+        File.join(home, ".rbenv", "shims"),
+        File.join(home, ".asdf", "shims")
+      ].select { |d| File.directory?(d) }
+    end
+
+    # Runs the command for ONE mutant against whatever is currently on disk (the
+    # caller has already swapped the mutant in via FileSwap). Maps the outcome to a
+    # Result. Uses {child_env} so version-manager PATH pins from Mutineer's Ruby do
+    # not force the suite onto the wrong interpreter.
+    #
+    # @param command [String] the --test-command template.
+    # @param files [Array<String>] test file paths.
+    # @param timeout [Numeric] per-mutant wall-clock timeout in seconds.
+    # @param verbose [Boolean] print the child's captured output on a non-pass.
+    # @return [Mutineer::Result]
+    def self.run(command, files, timeout:, verbose: false)
+      kind, code, output, = spawn_capture(command, files, timeout)
+      case kind
+      when :timeout
+        # A timeout is the one non-pass we flag by default — a normal kill is also
+        # a non-zero exit, so notifying on every non-zero would spam every kill.
+        warn "[mutineer] test-command exceeded #{timeout}s and was killed (scored timeout)."
+        warn output if verbose && !output.empty?
+        Result.timeout
+      else # :exited
+        return Result.survived if code&.zero?
+        # Signal death (nil exitstatus) is infrastructure, not a suite assertion
+        # failure — match Isolation/daemon (error), do not inflate kill rate.
+        if code.nil?
+          warn output if verbose && !output.empty?
+          return Result.error("test-command terminated by signal")
+        end
+
+        warn output if verbose && !output.empty?
+        Result.killed
+      end
+    end
+
+    # Pre-flight: run the command once against the UNMUTATED tree. Green (exit 0)
+    # returns the elapsed seconds (used to calibrate the per-mutant timeout);
+    # anything else raises SmokeCheckError so the run aborts before scoring.
+    #
+    # @param command [String] the --test-command template.
+    # @param files [Array<String>] test file paths.
+    # @param timeout [Numeric] ceiling for the calibration run.
+    # @return [Float] elapsed seconds of the clean run.
+    # @raise [Mutineer::SmokeCheckError] when the clean suite is not green.
+    def self.smoke_check!(command, files, timeout: SMOKE_TIMEOUT)
+      kind, code, output, elapsed = spawn_capture(command, files, timeout)
+      return elapsed if kind == :exited && code&.zero?
+
+      reason =
+        if kind == :timeout then "did not finish within #{timeout}s"
+        elsif code.nil?      then "was terminated by a signal"
+        else                      "exited #{code}"
+        end
+      detail = output.empty? ? "" : "\n--- last output ---\n#{tail(output)}"
+      hint = ruby_version_mismatch_hint(output)
+      raise SmokeCheckError,
+            "the test command #{reason} against the UNMUTATED source — " \
+            "#{hint || generic_env_hint}.#{detail}"
+    end
+
+    # Generic smoke-failure framing when no Ruby-version mismatch is detected.
+    #
+    # @api private
+    # @return [String]
+    def self.generic_env_hint
+      "the unmutated suite is not green (failing tests, or a broken environment: " \
+        "DB, RAILS_ENV, migrations) — fix that before scoring"
+    end
+
+    # Targeted hint when Bundler reports a RubyVersionMismatch (common under
+    # rbenv/asdf/chruby when Mutineer's version bin sits ahead of shims on PATH).
+    #
+    # @api private
+    # @param output [String] captured child stdout+stderr.
+    # @return [String, nil] hint text, or nil when not a version mismatch.
+    def self.ruby_version_mismatch_hint(output)
+      return nil if output.nil? || output.empty?
+      return nil unless output.match?(/RubyVersionMismatch|Your Ruby version is .* but your Gemfile specified/i)
+
+      ran = output[/Your Ruby version is ([0-9.]+)/i, 1]
+      want = output[/Gemfile specified ([0-9.]+)/i, 1]
+      parts = +"Detected a Ruby version mismatch"
+      parts << ": the test command ran under #{ran}" if ran
+      parts << " but the app expects #{want}" if want
+      parts << ". Mutineer already scrubbed version-manager bins and RBENV_VERSION/" \
+               "ASDF_RUBY_VERSION from the child env; the suite still picked the wrong " \
+               "Ruby. Use a wrapper that re-selects the app Ruby and ensure " \
+               ".ruby-version / .tool-versions is present " \
+               "(see README: Apps on Ruby < 3.4 → Under a version manager)"
+      parts
+    end
+
+    # Spawns the command to a captured combined-output tempfile, enforces a
+    # wall-clock timeout (SIGKILL past the deadline), and returns
+    # [kind, exit_code, output, elapsed]. Mirrors Isolation's single-waiter loop:
+    # we are the only caller of waitpid on this pid, so the kill can never hit a
+    # reaped/recycled pid.
+    #
+    # @api private
+    def self.spawn_capture(command, files, timeout)
+      argv = build_argv(command, files)
+      # Unreachable in practice (validate_test_command! guarantees a non-empty
+      # command with %{files}); a neutral ArgumentError, not the smoke-specific
+      # SmokeCheckError, since this is not a smoke-check failure.
+      raise ArgumentError, "--test-command produced an empty command" if argv.empty?
+
+      out = Tempfile.create("mutineer_ext")
+      start = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+      # `pgroup: true` puts the child in its OWN process group so a timeout can
+      # kill the whole tree (`bundle exec rails test` forks parallel workers;
+      # spring/bundler add more) — killing only the leader would orphan workers
+      # that keep holding the shared DB, corrupting later serial mutants. The
+      # explicit [program, argv0] form guarantees the no-shell exec path even for a
+      # degenerate single-element argv (Process.spawn(*argv) would route a lone
+      # metachar-bearing string through /bin/sh, breaking the argv-only invariant).
+      pid = Process.spawn(child_env, [argv.first, argv.first], *argv[1..],
+                          out: out, err: %i[child out], pgroup: true)
+      kind, code = wait_with_timeout(pid, timeout)
+      elapsed = Process.clock_gettime(Process::CLOCK_MONOTONIC) - start
+      out.rewind
+      [kind, code, out.read, elapsed]
+    ensure
+      if out
+        out.close
+        File.unlink(out.path) rescue nil # rubocop:disable Style/RescueModifier
+      end
+    end
+
+    # Waits for the spawned pid, SIGKILLing its process group past the deadline.
+    # Single-waiter deadline loop (mirrors Isolation), so the kill can never hit a
+    # reaped/recycled pid.
+    #
+    # @api private
+    # @param pid [Integer] the spawned child pid.
+    # @param timeout [Numeric] wall-clock deadline in seconds.
+    # @return [Array(Symbol, Integer, nil)] `[:exited, code]` or `[:timeout, nil]`.
+    def self.wait_with_timeout(pid, timeout)
+      deadline = Process.clock_gettime(Process::CLOCK_MONOTONIC) + timeout
+      loop do
+        reaped, status = Process.waitpid2(pid, Process::WNOHANG)
+        return [:exited, status.exitstatus] if reaped
+
+        if Process.clock_gettime(Process::CLOCK_MONOTONIC) >= deadline
+          # Kill the whole process GROUP (negative pid) so forked test workers die
+          # with the leader; fall back to the leader alone if the group is already
+          # gone. The child led its group (pgroup: true at spawn).
+          begin
+            Process.kill(:KILL, -pid)
+          rescue Errno::ESRCH, Errno::EPERM
+            Process.kill(:KILL, pid) rescue nil # rubocop:disable Style/RescueModifier
+          end
+          begin
+            _reaped, status = Process.waitpid2(pid)
+            # Finished cleanly between poll and kill — honor real exit code.
+            return [:exited, status.exitstatus] if status && status.exited? && !status.signaled?
+          rescue Errno::ECHILD
+            # already reaped
+          end
+          return [:timeout, nil]
+        end
+        sleep POLL
+      end
+    end
+
+    # Last ~40 lines of captured output, for a smoke-failure message.
+    #
+    # @api private
+    def self.tail(output, lines = 40)
+      output.lines.last(lines).join
+    end
+  end
+end