mutineer 1.4.0 → 1.6.0

data/lib/mutineer/daemon_backend.rb

@@ -5,10 +5,7 @@ require_relative "result"
 require_relative "coverage_map"
 require_relative "daemon_client"
 require_relative "progress"
-# No require_relative "runner" on purpose: runner.rb requires this file, and the
-# reverse edge makes Ruby warn "circular require considered harmful" on every -w
-# load. Runner is loaded first on every real path; requiring this file alone leaves
-# it undefined. Rationale and the real fix: #75.
+require_relative "job_plan"
 
 module Mutineer
   # Daemon execution backend. Boots the app ONCE in a persistent subprocess under
@@ -19,12 +16,12 @@ module Mutineer
   # When jobs > 1 each worker runs against its OWN database, which is what makes
   # `--jobs N` safe under Rails (#26): parallel verdicts are identical to serial.
   #
-  # Job collection, `--since` filtering and coverage selection stay on {Runner} and
-  # are called from here, so the daemon path can never drift from the in-process
-  # path on which mutants run or which tests narrow a mutant (score parity).
+  # Job collection, `--since` filtering and coverage selection live in {JobPlan},
+  # which the in-process path calls too, so the daemon path can never drift from
+  # it on which mutants run or which tests narrow a mutant (score parity).
   #
   # Unlike {ExternalBackend}, which is a leaf {Runner} calls into, this module owns
-  # its orchestration and calls back for that shared vocabulary.
+  # its orchestration and takes that shared vocabulary from {JobPlan}.
   module DaemonBackend
     # Default per-mutant timeout on the daemon path (seconds), overridden by
     # config.daemon_timeout. Coverage narrowing usually keeps each job short; this
@@ -44,10 +41,10 @@ module Mutineer
     # @param config [Mutineer::Config] run configuration (daemon set).
     # @param operator_classes [Array<Class>] resolved operators.
     # @return [Array(Mutineer::AggregateResult, Hash<String,String>, Hash)] aggregate,
-    #   source map, and the {Runner.collect_jobs} extras.
+    #   source map, and the {JobPlan.collect_jobs} extras.
     def self.execute(config, operator_classes)
-      jobs, ignored_results, source_map, extras = Runner.collect_jobs(config, operator_classes)
-      jobs = Runner.filter_since(jobs, source_map, config) if config.since
+      jobs, ignored_results, source_map, extras = JobPlan.collect_jobs(config, operator_classes)
+      jobs = JobPlan.filter_since(jobs, source_map, config) if config.since
       abs_tests = config.tests.map { |t| File.expand_path(t, config.project_root) }
 
       # Nothing to mutate (`--since` matched no changed line, or every mutant is
@@ -58,7 +55,7 @@ module Mutineer
         # The daemon sweeps orphaned temps at boot and nothing boots here, so sweep
         # tool-side. A file a hard-killed run left in app/models breaks the app's own
         # Zeitwerk boot, not just Mutineer's next run.
-        Runner.sweep_orphans(Runner.source_dirs(config), DAEMON_TEMP_GLOB)
+        JobPlan.sweep_orphans(JobPlan.source_dirs(config), DAEMON_TEMP_GLOB)
         return [AggregateResult.new(ignored_results), source_map, extras]
       end
 
@@ -91,8 +88,11 @@ module Mutineer
     # Build the coverage map via a short-lived daemon (boots the app once, captures
     # per-test coverage app-side, ships the map back). Returns a query-only
     # CoverageMap, or nil when the build fails / returns empty. Callers then run the
-    # full --test set. Coverage-build IPC has no wall-clock (same limitation as
-    # in-process build_via_fork). A normal nonempty map scores like in-process;
+    # full --test set. An empty map that came with boot lines returns a map with
+    # those lines only: it narrows nothing, but still classifies `ran_at_load`.
+    # Each capture is bounded by capture_timeout (#101); the client's wait for
+    # the whole map is not, because its length grows with the number of test
+    # files. A normal nonempty map scores like in-process;
     # nil falls back to the full suite (more testing, not comparable).
     #
     # @param config [Mutineer::Config] the run config.
@@ -109,7 +109,7 @@ module Mutineer
       # A red unmutated suite must abort, even when the shipped map is empty.
       # Falling back to the full --test set would treat those failures as kills.
       if data.is_a?(Hash) && Array(data["failed_clean_tests"]).any?
-        Runner.abort_if_unclean!(CoverageMap.from_data(
+        JobPlan.abort_if_unclean!(CoverageMap.from_data(
           map: data["map"] || {},
           failed_test_files: data["failed_test_files"] || [],
           project_root: config.project_root,
@@ -120,11 +120,21 @@ module Mutineer
       unless data && !(data["map"] || {}).empty?
         reason = data.is_a?(Hash) && data["error"] ? data["error"] : "empty map"
         warn_coverage_fallback(reason)
-        return nil
+        # No narrowing ({job_result} runs every test), but the lines and
+        # methods that ran at boot still classify a survivor on one of them.
+        load_lines = data.is_a?(Hash) ? Array(data["load_lines"]) : []
+        load_methods = data.is_a?(Hash) ? Array(data["load_methods"]) : []
+        return nil if load_lines.empty? && load_methods.empty?
+
+        return CoverageMap.from_data(map: {}, failed_test_files: [], project_root: config.project_root,
+                                     load_lines: load_lines, load_methods: load_methods)
       end
 
       CoverageMap.from_data(map: data["map"], failed_test_files: data["failed_test_files"] || [],
-                            project_root: config.project_root)
+                            project_root: config.project_root, load_lines: data["load_lines"] || [],
+                            load_methods: data["load_methods"] || [])
+    rescue DaemonBootTimeout
+      raise # a second daemon for the mutant runs would hang just as long
     rescue DaemonBootError => e
       warn_coverage_fallback("#{e.class}: #{e.message}")
       nil
@@ -246,10 +256,10 @@ module Mutineer
       # Skip an invalid mutant tool-side: never ship a payload that would fail to
       # load and read as a false `killed`.
       # Narrow to covering tests (shared with the in-process path via
-      # Runner.coverage_selection, so scores match). :verdict = no_coverage/uncapturable,
-      # no fork. No map (build failed) → run the full --test set (fallback, not
-      # narrowed).
-      sel = coverage_map && Runner.coverage_selection(subject.file, mutation, subject, source, coverage_map)
+      # JobPlan.coverage_selection, so scores match). :verdict = no_coverage/uncapturable,
+      # no fork. No map, or an empty one (build failed) → run the full --test set
+      # (fallback, not narrowed).
+      sel = coverage_map && !coverage_map.map.empty? && JobPlan.coverage_selection(subject.file, mutation, subject, source, coverage_map)
       r =
         if Parser.parse_string(mutated).errors.any?
           Result.skipped
@@ -261,14 +271,16 @@ module Mutineer
             payload: { "code" => mutated, "source_file" => File.expand_path(subject.file, config.project_root) },
             tests: sel ? sel[1] : abs_tests
           )
-          result_for(verdict)
+          # A survivor whose line ran at load, as in-process (Runner.run).
+          JobPlan.load_verdict(result_for(verdict), subject.file, mutation, subject, source, coverage_map)
         end
       r.with(subject: subject, mutation: mutation, id: id)
     end
 
-    # The boot config the daemon needs to boot the app once: where to boot, the test
-    # load roots (so `require "test_helper"` resolves in every fork), framework, and
-    # whether this is Rails.
+    # The boot config the daemon needs to boot the app once: where to boot, the
+    # --require files to load after it, the test load roots (so
+    # `require "test_helper"` resolves in every fork), framework, and whether this
+    # is Rails.
     #
     # @param config [Mutineer::Config] the run config.
     # @param abs_tests [Array<String>] absolute --test paths.
@@ -278,27 +290,33 @@ module Mutineer
       {
         project_root: config.project_root,
         boot: File.expand_path(config.boot || "config/environment", config.project_root),
-        load_paths: Runner.test_load_roots(abs_tests),
-        source_dirs: Runner.source_dirs(config), # so the daemon can sweep orphan mutant temps
+        # Required after the boot, as in-process (Runner.execute); also part of
+        # the coverage digest.
+        require_paths: config.require_paths.map { |f| File.expand_path(f, config.project_root) },
+        load_paths: JobPlan.test_load_roots(abs_tests),
+        cache_dir: File.expand_path(config.cache_dir, config.project_root),
+        source_dirs: JobPlan.source_dirs(config), # so the daemon can sweep orphan mutant temps
         framework: config.framework,
         rails: config.rails,
-        # Schema for per-worker DB isolation. Sent when present; the daemon
-        # skips worker-DB schema loading if the path is absent (e.g. structure.sql apps).
+        # Schema for per-worker DB isolation. Sent when present; the daemon loads
+        # it over a worker's copy of the test DB only when that copy is out of date.
         schema: schema_path(config),
         # Coverage narrowing. Only the short-lived map-building daemon starts
         # Coverage (before boot); worker daemons boot with it OFF (no wasted
         # instrumentation/memory across every mutant fork). `sources`/`tests` are the
         # map-build inputs.
         coverage: coverage,
+        # The map-building daemon's capture limit (--capture-timeout); nil = default.
+        capture_timeout: config.capture_timeout,
         sources: config.sources.map { |s| File.expand_path(s, config.project_root) },
         tests: abs_tests
       }
     end
 
-    # Absolute path to the app's `db/schema.rb` if it exists, else nil. Used by the
-    # daemon to schema-load each fork's isolated worker database. Only `schema.rb`
-    # is supported this pass; `structure.sql` apps get nil and fall back to
-    # whatever the worker DB already holds.
+    # Absolute path to the app's `db/schema.rb` if it exists, else nil. Each worker
+    # DB starts as a copy of the test DB; the daemon loads this file over the copy
+    # when the copy's schema differs. `structure.sql` apps get nil and keep the
+    # copy as it is.
     #
     # @param config [Mutineer::Config] the run config.
     # @api private

data/lib/mutineer/daemon_client.rb

@@ -1,7 +1,9 @@
 # frozen_string_literal: true
 
 require "json"
+require "io/wait"
 require "open3"
+require_relative "external_backend"
 
 module Mutineer
   # Raised when the daemon cannot be booted or is gone for good: a bad boot path, an
@@ -11,12 +13,19 @@ module Mutineer
   # to a runtime error (exit 1).
   class DaemonBootError < StandardError; end
 
+  # A boot that has not answered the handshake within DaemonClient::BOOT_TIMEOUT
+  # (#101). Unlike other boot errors it is not worth retrying with a second
+  # daemon, which would wait just as long.
+  class DaemonBootTimeout < DaemonBootError; end
+
   # Tool-side handle for the app-side daemon.
   #
-  # Spawns `daemon_server.rb` UNDER THE APP'S BUNDLE/RUBY (cleaned env so the
-  # gem's bundler context never leaks; the daemon file is loaded by absolute path
-  # with `-r`, which bypasses the app bundle that has no mutineer), completes the
-  # ready handshake, then ships per-mutant payloads and reads structured verdicts.
+  # Spawns `daemon_server.rb` UNDER THE APP'S BUNDLE/RUBY. The child gets the
+  # environment from before the tool's Bundler activated, with
+  # <tt>unsetenv_others</tt>, because Open3 keeps a key the hash omits. The
+  # daemon file is loaded by absolute path with `-r`, which bypasses the app
+  # bundle that has no mutineer. The client completes the ready handshake,
+  # then ships per-mutant payloads and reads structured verdicts.
   # If the daemon dies mid-run it respawns (bounded) and marks the in-flight
   # mutant `error` rather than corrupting the run. Reuses the cleaned-env spawn
   # and stderr-drain proven in the spike driver and the spawn discipline of
@@ -26,11 +35,22 @@ module Mutineer
     DAEMON_PATH = File.expand_path("daemon_server.rb", __dir__)
     # How many times to respawn a crashing daemon before aborting the run.
     MAX_RESTARTS = 3
+    # Seconds a verdict may lag the request's own timeout. The daemon enforces
+    # that timeout on the mutant child, so a reply later than this means the
+    # daemon itself is wedged (#101): it is killed and respawned.
+    REPLY_GRACE = 30
+    # Seconds the daemon gets to boot the app and answer the handshake. A boot
+    # that never returns ends the run instead of hanging it (#101).
+    BOOT_TIMEOUT = 600
+    # Bundler's marker for a variable that was unset before it activated.
+    BUNDLER_UNSET = "BUNDLER_ENVIRONMENT_PRESERVER_INTENTIONALLY_NIL"
 
     # @param boot [Hash] boot config sent to the daemon: project_root, boot,
     #   load_paths, framework, rails.
     # @param app_root [String] directory to spawn the daemon in (the app root).
-    # @param ruby_version [String, nil] RBENV_VERSION for the app's Ruby (nil = inherit).
+    # @param ruby_version [String, nil] RBENV_VERSION for the app's Ruby.
+    #   nil keeps a pin already in the environment. That pin is often the
+    #   Ruby that started the tool.
     # @param gemfile [String, nil] BUNDLE_GEMFILE for the app's bundle (nil = app_root/Gemfile).
     # @param errio [IO] where daemon stderr is drained.
     def initialize(boot:, app_root:, ruby_version: nil, gemfile: nil, errio: $stderr)
@@ -76,7 +96,7 @@ module Mutineer
       reply =
         begin
           send_line("id" => id, "worker" => worker, "payload" => payload, "tests" => tests, "timeout" => timeout)
-          read_line
+          read_line(timeout + REPLY_GRACE)
         rescue Errno::EPIPE, IOError
           nil
         end
@@ -110,32 +130,181 @@ module Mutineer
       return unless @stdin
 
       send_line("cmd" => "quit") rescue nil # rubocop:disable Style/RescueModifier
-      @wait_thr&.join
+      @wait_thr&.join(REPLY_GRACE) # a wedged daemon is killed by close_io
     ensure
       close_io
     end
 
     private
 
-    # Cleaned environment for the app bundle: strip the gem's bundler/Ruby context
-    # so `bundle exec` resolves the APP's Gemfile under the requested Ruby.
+    # Full environment for the daemon child. Spawn with +unsetenv_others+ so a
+    # key deleted here does not leak back from the parent. Undo only what
+    # Bundler saved before it activated, then point the child at the target app.
+    #
+    # @api private
+    # @return [Hash{String => String}]
     def app_env
-      env = ENV.to_h.reject { |k, _| k.start_with?("BUNDLE_", "RUBY", "GEM_") }
+      env = restored_user_env
+      scrub_managed_ruby_bins!(env)
+      env.delete("BUNDLER_SETUP")
       env["BUNDLE_GEMFILE"] = @gemfile
       env["RBENV_VERSION"] = @ruby_version if @ruby_version
-      env["RAILS_ENV"] ||= "test" if @boot[:rails] || @boot["rails"]
+      env["RAILS_ENV"] = "test" if rails_boot? && !env.key?("RAILS_ENV")
+      env
+    end
+
+    # Environment with every <tt>BUNDLER_ORIG_*</tt> value put back. Bundler
+    # writes one saved key per variable it replaced. Restoring all of them
+    # undoes the tool bundle without a copied key list.
+    #
+    # @api private
+    # @return [Hash{String => String}]
+    def restored_user_env
+      env = ENV.to_h
+      restored = []
+      env.keys.each do |saved_key|
+        next unless saved_key.start_with?("BUNDLER_ORIG_")
+
+        key = saved_key.delete_prefix("BUNDLER_ORIG_")
+        saved = env.delete(saved_key)
+        restored << key
+        if saved == BUNDLER_UNSET
+          env.delete(key)
+        else
+          env[key] = saved
+        end
+      end
+      # A parent require must not run in the app, even when Bundler saved it.
+      %w[RUBYOPT RUBYLIB].each { |key| env.delete(key) }
+      # No saved original means the value belongs to the tool process.
+      %w[GEM_HOME GEM_PATH BUNDLE_PATH BUNDLE_WITHOUT BUNDLER_VERSION].each do |key|
+        env.delete(key) unless restored.include?(key)
+      end
       env
     end
 
+    # Drop rbenv and asdf version bins so shims can select the app Ruby.
+    # Leave those bins in place when no shim directory exists, so +bundle+
+    # stays on PATH. Leave chruby bins in place.
+    #
+    # @api private
+    # @param env [Hash{String => String}]
+    # @return [void]
+    def scrub_managed_ruby_bins!(env)
+      parts = env["PATH"].to_s.split(File::PATH_SEPARATOR)
+      rbenv = rbenv_shim_dirs(env)
+      asdf = asdf_shim_dirs(env)
+      dropped_rbenv = false
+      dropped_asdf = false
+      kept = parts.reject do |part|
+        if rbenv_version_bin?(part, env) && rbenv.any?
+          dropped_rbenv = true
+        elsif asdf_version_bin?(part, env) && asdf.any?
+          dropped_asdf = true
+        else
+          false
+        end
+      end
+      return if kept.size == parts.size
+
+      shims = []
+      shims.concat(rbenv) if dropped_rbenv
+      shims.concat(asdf) if dropped_asdf
+      env["PATH"] = (shims + kept).uniq.join(File::PATH_SEPARATOR)
+    end
+
+    # Existing rbenv shim directory. +RBENV_ROOT+ wins. Otherwise the home
+    # directory's <tt>.rbenv</tt> is used.
+    #
+    # @api private
+    # @param env [Hash{String => String}]
+    # @return [Array<String>]
+    def rbenv_shim_dirs(env)
+      root = rbenv_root(env)
+      return [] if root.nil?
+
+      dir = File.join(root, "shims")
+      File.directory?(dir) ? [dir] : []
+    end
+
+    # rbenv install root. +RBENV_ROOT+ wins over <tt>~/.rbenv</tt>.
+    #
+    # @api private
+    # @param env [Hash{String => String}]
+    # @return [String, nil]
+    def rbenv_root(env)
+      root = env["RBENV_ROOT"]
+      return File.expand_path(root) if root && !root.empty?
+
+      home = env["HOME"]
+      return nil if home.nil? || home.empty?
+
+      File.join(home, ".rbenv")
+    end
+
+    # Existing asdf shim directories. A custom install uses +ASDF_DATA_DIR+.
+    #
+    # @api private
+    # @param env [Hash{String => String}]
+    # @return [Array<String>]
+    def asdf_shim_dirs(env)
+      dirs = []
+      data = env["ASDF_DATA_DIR"]
+      dirs << File.join(data, "shims") if data && !data.empty?
+      home = env["HOME"]
+      dirs << File.join(home, ".asdf", "shims") if home && !home.empty?
+      dirs.select { |dir| File.directory?(dir) }.uniq
+    end
+
+    # True when `part` is a version bin under the active rbenv root.
+    #
+    # @api private
+    # @param part [String] one PATH entry.
+    # @param env [Hash{String => String}]
+    # @return [Boolean]
+    def rbenv_version_bin?(part, env)
+      root = rbenv_root(env)
+      return false if root.nil?
+
+      prefix = File.join(root, "versions")
+      File.expand_path(part).match?(%r{\A#{Regexp.escape(prefix)}/[^/]+/bin/?\z})
+    end
+
+    # True when `part` is an asdf Ruby version bin under ~/.asdf or +ASDF_DATA_DIR+.
+    #
+    # @api private
+    # @param part [String] one PATH entry.
+    # @param env [Hash{String => String}]
+    # @return [Boolean]
+    def asdf_version_bin?(part, env)
+      return true if part.match?(%r{/\.asdf/installs/ruby/[^/]+/bin/?\z})
+
+      data = env["ASDF_DATA_DIR"]
+      return false if data.nil? || data.empty?
+
+      prefix = File.join(File.expand_path(data), "installs", "ruby")
+      File.expand_path(part).match?(%r{\A#{Regexp.escape(prefix)}/[^/]+/bin/?\z})
+    end
+
+    # True when this daemon boots a Rails app.
+    #
+    # @api private
+    # @return [Boolean]
+    def rails_boot?
+      @boot[:rails] || @boot["rails"]
+    end
+
     # Spawn the daemon under the app bundle and complete the ready handshake.
     #
     # @return [void]
     # @raise [Mutineer::DaemonBootError] when the daemon fails to boot.
     def spawn_daemon
       # Plain `bundle exec ruby`, NOT `rbenv exec`, which would break CI and any
-      # non-rbenv setup. When bundler/ruby are rbenv shims, the RBENV_VERSION
-      # carried in app_env still selects the app's Ruby; otherwise the active
-      # Ruby is used.
+      # non-rbenv setup. An explicit ruby_version replaces RBENV_VERSION.
+      # With no argument, a pin already in the environment stays. That pin is
+      # often the Ruby that started the tool. `.ruby-version` applies only
+      # when no pin is set. rbenv and asdf version bins leave PATH when a shim
+      # directory exists. chruby bins stay.
       # Everything up to the handshake is terminal, not one mutant's problem: a spawn
       # the OS refuses (EMFILE/ENOMEM under --jobs N, ENOENT when `bundle` does not
       # resolve) and a daemon that dies before accepting the boot payload (EPIPE on
@@ -145,7 +314,8 @@ module Mutineer
         begin
           @stdin, @stdout, @stderr, @wait_thr = Open3.popen3(
             app_env, "bundle", "exec", "ruby",
-            "-r", DAEMON_PATH, "-e", "Mutineer::DaemonServer.run", chdir: @app_root
+            "-r", DAEMON_PATH, "-e", "Mutineer::DaemonServer.run",
+            unsetenv_others: true, chdir: @app_root
           )
           # Drain daemon stderr to the tool's stderr so child/boot errors are visible.
           # Tracked (not fire-and-forget) so close_io can reclaim it on quit/respawn;
@@ -158,16 +328,21 @@ module Mutineer
           end
 
           send_line(@boot)
-          read_line
+          read_line(BOOT_TIMEOUT)
         rescue SystemCallError, IOError => e
           close_io
           raise DaemonBootError, "daemon could not be started: #{e.class}: #{e.message}"
         end
 
       unless ready && ready["ready"]
-        detail = ready && ready["error"] ? ready["error"] : "daemon exited before the handshake"
+        detail =
+          if ready && ready["error"] then ready["error"]
+          elsif @timed_out then "the daemon did not finish booting within #{BOOT_TIMEOUT}s"
+          else "daemon exited before the handshake"
+          end
+        timed_out = @timed_out
         close_io
-        raise DaemonBootError, "daemon failed to boot under the app bundle: #{detail}"
+        raise (timed_out ? DaemonBootTimeout : DaemonBootError), "daemon failed to boot under the app bundle: #{detail}"
       end
     end
 
@@ -176,10 +351,11 @@ module Mutineer
       close_io
       @restarts += 1
       if @restarts > MAX_RESTARTS
-        raise DaemonBootError, "daemon crashed #{@restarts} times; aborting the run"
+        raise DaemonBootError, "daemon crashed or stopped answering #{@restarts} times; aborting the run"
       end
 
-      @errio.puts("[mutineer] daemon crashed — respawning (#{@restarts}/#{MAX_RESTARTS})")
+      cause = @timed_out ? "stopped answering" : "crashed"
+      @errio.puts("[mutineer] daemon #{cause} — respawning (#{@restarts}/#{MAX_RESTARTS})")
       spawn_daemon
     end
 
@@ -192,8 +368,15 @@ module Mutineer
       @stdin.flush
     end
 
-    # Read one JSON reply line; nil on EOF/dead pipe (caller treats as a crash).
-    def read_line
+    # Read one JSON reply line; nil on EOF/dead pipe, or when no line arrives
+    # within `timeout` seconds (caller treats either as a crash).
+    #
+    # @param timeout [Numeric, nil] seconds to wait; nil waits for the reply.
+    # @return [Hash, nil]
+    def read_line(timeout = nil)
+      @timed_out = timeout && !@stdout.wait_readable(timeout)
+      return nil if @timed_out
+
       line = @stdout.gets
       line && JSON.parse(line.strip)
     rescue IOError, Errno::EPIPE, JSON::ParserError
@@ -205,6 +388,13 @@ module Mutineer
     #
     # @return [void]
     def close_io
+      # A wedged daemon may never read the closed stdin, so the reap below would
+      # wait forever: kill it first. An exited daemon makes this a no-op.
+      begin
+        Process.kill(:KILL, @wait_thr.pid) if @wait_thr&.alive?
+      rescue Errno::ESRCH
+        nil # it exited between the check and the kill
+      end
       @drain&.kill # stop the drain BEFORE closing its fd (avoids a copy_stream EBADF)
       [@stdin, @stdout, @stderr].each { |io| io&.close rescue nil } # rubocop:disable Style/RescueModifier
       @wait_thr&.join # reap the exited daemon so respawn/quit leaves no zombie