mutineer 1.5.0 → 1.7.0

data/lib/mutineer/daemon_backend.rb

@@ -5,10 +5,8 @@ require_relative "result"
 require_relative "coverage_map"
 require_relative "daemon_client"
 require_relative "progress"
-# No require_relative "runner" on purpose: runner.rb requires this file, and the
-# reverse edge makes Ruby warn "circular require considered harmful" on every -w
-# load. Runner is loaded first on every real path; requiring this file alone leaves
-# it undefined. Rationale and the real fix: #75.
+require_relative "job_plan"
+require_relative "orphan_guard"
 
 module Mutineer
   # Daemon execution backend. Boots the app ONCE in a persistent subprocess under
@@ -19,12 +17,12 @@ module Mutineer
   # When jobs > 1 each worker runs against its OWN database, which is what makes
   # `--jobs N` safe under Rails (#26): parallel verdicts are identical to serial.
   #
-  # Job collection, `--since` filtering and coverage selection stay on {Runner} and
-  # are called from here, so the daemon path can never drift from the in-process
-  # path on which mutants run or which tests narrow a mutant (score parity).
+  # Job collection, `--since` filtering and coverage selection live in {JobPlan},
+  # which the in-process path calls too, so the daemon path can never drift from
+  # it on which mutants run or which tests narrow a mutant (score parity).
   #
   # Unlike {ExternalBackend}, which is a leaf {Runner} calls into, this module owns
-  # its orchestration and calls back for that shared vocabulary.
+  # its orchestration and takes that shared vocabulary from {JobPlan}.
   module DaemonBackend
     # Default per-mutant timeout on the daemon path (seconds), overridden by
     # config.daemon_timeout. Coverage narrowing usually keeps each job short; this
@@ -34,8 +32,13 @@ module Mutineer
     DEFAULT_TIMEOUT = 60
 
     # The daemon's per-mutant tempfile, written into the source dir so
-    # require_relative resolves. Kept in step with DaemonServer#sweep_temps.
-    DAEMON_TEMP_GLOB = "mutineer_daemon*.rb"
+    # require_relative resolves. The leading dot hides it from Zeitwerk (#253);
+    # older versions wrote it without one. Kept in step with DaemonServer#sweep_temps.
+    DAEMON_TEMP_GLOB = "{.,}mutineer_daemon*.rb"
+
+    # Seconds the end-of-run sweep waits for an orphaned child to die. Its
+    # watchdog notices a dead daemon within {OrphanGuard::POLL}.
+    SWEEP_GRACE = (OrphanGuard::POLL * 2) + 0.5
 
     # Full daemon run: collect jobs, build the coverage map once, then execute
     # serially or across N worker daemons. Fail-fast forces serial so the survivor
@@ -44,28 +47,17 @@ module Mutineer
     # @param config [Mutineer::Config] run configuration (daemon set).
     # @param operator_classes [Array<Class>] resolved operators.
     # @return [Array(Mutineer::AggregateResult, Hash<String,String>, Hash)] aggregate,
-    #   source map, and the {Runner.collect_jobs} extras.
+    #   source map, and the {JobPlan.collect_jobs} extras.
     def self.execute(config, operator_classes)
-      jobs, ignored_results, source_map, extras = Runner.collect_jobs(config, operator_classes)
-      jobs = Runner.filter_since(jobs, source_map, config) if config.since
+      jobs, ignored_results, source_map, extras = JobPlan.collect_jobs(config, operator_classes)
+      jobs, ignored_results, extras[:unscoped_mutants] = JobPlan.scope_since(jobs, ignored_results, source_map, config)
       abs_tests = config.tests.map { |t| File.expand_path(t, config.project_root) }
 
       # Nothing to mutate (`--since` matched no changed line, or every mutant is
       # suppressed). Return before booting anything: the coverage daemon and the
       # worker daemons below each boot the whole app, and README documents
       # `--since origin/<base>` for PR CI, where a docs-only PR is routine.
-      if jobs.empty?
-        # The daemon sweeps orphaned temps at boot and nothing boots here, so sweep
-        # tool-side. A file a hard-killed run left in app/models breaks the app's own
-        # Zeitwerk boot, not just Mutineer's next run.
-        Runner.sweep_orphans(Runner.source_dirs(config), DAEMON_TEMP_GLOB)
-        return [AggregateResult.new(ignored_results), source_map, extras]
-      end
-
-      # Build the coverage map once (app-side). nil when the build fails: runners
-      # fall back to the full --test set (and emit a stderr warning) rather than
-      # mis-scoring everything as no_coverage.
-      coverage_map = build_coverage_map(config, abs_tests)
+      return [AggregateResult.new(ignored_results), source_map, extras] if jobs.empty?
 
       # Worker count = resolved --jobs, capped at the job count (no idle daemons).
       # >1 → N concurrent daemon handles, each on its OWN worker DB (N-handles, the
@@ -74,25 +66,70 @@ module Mutineer
       # input index, so the verdict set would diverge from serial (a different,
       # non-deterministic survivor set/score). The "identical to --jobs 1" guarantee
       # below only holds when fail-fast cannot race.
+      # A second database config is only known after the app boots, so the first
+      # daemon can still lower this count before any worker starts.
       worker_count = [config.jobs || 1, 1].max
       worker_count = 1 if config.fail_fast
       worker_count = [worker_count, jobs.size].min
 
-      results =
-        if worker_count > 1
-          run_parallel(jobs, worker_count, config, abs_tests, coverage_map, source_map)
-        else
-          run_serial(jobs, config, abs_tests, coverage_map, source_map)
-        end
+      # One booted process copies worker databases, then builds the coverage map.
+      # A cached map still provisions: the copy has to happen while no worker is
+      # connected to the template database. That process keeps its database lock
+      # until the worker daemons have taken theirs.
+      prepare_client = nil
+      begin
+        coverage_map, worker_count, prepare_client = prepare_run(config, abs_tests, worker_count)
+
+        results =
+          if worker_count > 1
+            run_parallel(jobs, worker_count, config, abs_tests, coverage_map, source_map, prepare_client)
+          else
+            run_serial(jobs, config, abs_tests, coverage_map, source_map, prepare_client)
+          end
+      ensure
+        # run_serial and run_parallel quit this client once workers hold locks.
+        # Quit again here when they never reached that point. A second quit is a no-op.
+        prepare_client&.quit
+      end
 
       [AggregateResult.new(results + ignored_results), source_map, extras]
+    ensure
+      sweep_daemon_temps(config)
+    end
+
+    # Removes daemon mutant files left in the source dirs, on every exit path.
+    # The daemon sweeps at boot, but an empty run boots nothing, and a daemon
+    # that dies mid-mutant orphans a child that lives on until its watchdog
+    # fires (#250). The respawned daemon sweeps before that, and a run that
+    # ends or aborts right after the crash does too. So a file whose owner is
+    # still alive gets {SWEEP_GRACE} to go. A clean run leaves no file and pays
+    # nothing. A file a concurrent run still owns stays in place, but looks the
+    # same as a dying child's, so this run then waits the full grace.
+    #
+    # @param config [Mutineer::Config] the run config.
+    # @return [void]
+    def self.sweep_daemon_temps(config)
+      dirs = JobPlan.source_dirs(config)
+      deadline = Process.clock_gettime(Process::CLOCK_MONOTONIC) + SWEEP_GRACE
+      loop do
+        still_owned = JobPlan.sweep_orphans(dirs, DAEMON_TEMP_GLOB)
+        break if still_owned.empty?
+        break if Process.clock_gettime(Process::CLOCK_MONOTONIC) >= deadline
+
+        sleep 0.1
+      end
+    rescue SystemCallError
+      nil # never replace the run's own result or exception with a cleanup error
     end
 
     # Build the coverage map via a short-lived daemon (boots the app once, captures
     # per-test coverage app-side, ships the map back). Returns a query-only
     # CoverageMap, or nil when the build fails / returns empty. Callers then run the
-    # full --test set. Coverage-build IPC has no wall-clock (same limitation as
-    # in-process build_via_fork). A normal nonempty map scores like in-process;
+    # full --test set. An empty map that came with boot lines returns a map with
+    # those lines only: it narrows nothing, but still classifies `ran_at_load`.
+    # Each capture is bounded by capture_timeout (#101); the client's wait for
+    # the whole map is not, because its length grows with the number of test
+    # files. A normal nonempty map scores like in-process;
     # nil falls back to the full suite (more testing, not comparable).
     #
     # @param config [Mutineer::Config] the run config.
@@ -101,15 +138,68 @@ module Mutineer
     def self.build_coverage_map(config, abs_tests)
       client = DaemonClient.new(boot: boot_config(config, abs_tests, coverage: true),
                                 app_root: config.project_root).start
-      data = begin
-        client.coverage
+      begin
+        coverage_map_from_client(client, config)
       ensure
         client.quit
       end
+    rescue DaemonBootTimeout
+      raise
+    rescue DaemonBootError => e
+      warn_coverage_fallback("#{e.class}: #{e.message}")
+      nil
+    end
+
+    # Boot the first daemon, copy worker databases, then read the coverage map
+    # from that same process. A boot failure still falls back to the full test
+    # set. A provisioning failure does not: it stops the run.
+    #
+    # @param config [Mutineer::Config]
+    # @param abs_tests [Array<String>]
+    # @param worker_count [Integer] slots requested before the database count is known.
+    # @return [Array] the coverage map (or nil), the worker count to start, and
+    #   the prepare daemon. The caller quits that daemon after worker daemons
+    #   hold their locks. It is nil when this process could not boot one.
+    def self.prepare_run(config, abs_tests, worker_count)
+      client = nil
+      begin
+        client = DaemonClient.new(boot: boot_config(config, abs_tests, coverage: true),
+                                  app_root: config.project_root).start
+      rescue DaemonBootTimeout
+        raise
+      rescue DaemonBootError => e
+        warn_coverage_fallback("#{e.class}: #{e.message}")
+        return [nil, worker_count, nil]
+      end
+
+      begin
+        count = client.database_count.to_i
+        if count > 1 && worker_count > 1
+          warn "[mutineer] this app has #{count} databases; Mutineer runs one worker. " \
+               "Parallel runs support one database."
+          worker_count = 1
+        end
+        # Raises DaemonBootError. Callers must not score mutants after this.
+        client.provision(worker_count)
+        [coverage_map_from_client(client, config), worker_count, client]
+      rescue Exception # rubocop:disable Lint/RescueException
+        # An interrupt must also quit it, or the daemon keeps its database lock.
+        client.quit
+        raise
+      end
+    end
+
+    # Turn one daemon's coverage reply into a map. Does not start or stop the daemon.
+    #
+    # @param client [Mutineer::DaemonClient]
+    # @param config [Mutineer::Config]
+    # @return [Mutineer::CoverageMap, nil]
+    def self.coverage_map_from_client(client, config)
+      data = client.coverage
       # A red unmutated suite must abort, even when the shipped map is empty.
       # Falling back to the full --test set would treat those failures as kills.
       if data.is_a?(Hash) && Array(data["failed_clean_tests"]).any?
-        Runner.abort_if_unclean!(CoverageMap.from_data(
+        JobPlan.abort_if_unclean!(CoverageMap.from_data(
           map: data["map"] || {},
           failed_test_files: data["failed_test_files"] || [],
           project_root: config.project_root,
@@ -120,11 +210,21 @@ module Mutineer
       unless data && !(data["map"] || {}).empty?
         reason = data.is_a?(Hash) && data["error"] ? data["error"] : "empty map"
         warn_coverage_fallback(reason)
-        return nil
+        # No narrowing ({job_result} runs every test), but the lines and
+        # methods that ran at boot still classify a survivor on one of them.
+        load_lines = data.is_a?(Hash) ? Array(data["load_lines"]) : []
+        load_methods = data.is_a?(Hash) ? Array(data["load_methods"]) : []
+        return nil if load_lines.empty? && load_methods.empty?
+
+        return CoverageMap.from_data(map: {}, failed_test_files: [], project_root: config.project_root,
+                                     load_lines: load_lines, load_methods: load_methods)
       end
 
       CoverageMap.from_data(map: data["map"], failed_test_files: data["failed_test_files"] || [],
-                            project_root: config.project_root)
+                            project_root: config.project_root, load_lines: data["load_lines"] || [],
+                            load_methods: data["load_methods"] || [])
+    rescue DaemonBootTimeout
+      raise # a second daemon for the mutant runs would hang just as long
     rescue DaemonBootError => e
       warn_coverage_fallback("#{e.class}: #{e.message}")
       nil
@@ -145,59 +245,69 @@ module Mutineer
     # (stop at the first survivor).
     #
     # @api private
+    # @param prepare_client [Mutineer::DaemonClient, nil] the daemon that copied
+    #   the databases. Quit once this worker has booted and taken its lock.
     # @return [Array<Mutineer::Result>] results in input order.
-    def self.run_serial(jobs, config, abs_tests, coverage_map, source_map)
-      client = DaemonClient.new(boot: boot_config(config, abs_tests),
-                                app_root: config.project_root).start
-      results = []
-      progress = Progress.new(jobs.size)
+    def self.run_serial(jobs, config, abs_tests, coverage_map, source_map, prepare_client = nil)
+      client = nil
       begin
+        client = DaemonClient.new(boot: boot_config(config, abs_tests),
+                                  app_root: config.project_root).start
+        prepare_client&.quit
+        prepare_client = nil
+        results = []
+        progress = Progress.new(jobs.size)
         jobs.each_with_index do |job, i|
           r = job_result(job, i, client, 0, config, coverage_map, abs_tests, source_map)
           results << r
           progress.tick
           break if config.fail_fast && r.survived?
         end
+        results
       ensure
-        client.quit
+        prepare_client&.quit
+        client&.quit
       end
-      results
     end
 
     # Parallel path: N daemon handles, each pinned to its own worker slot (own DB).
     # A shared queue of job indices feeds N tool-side threads; results are placed by
     # input index so the verdict set matches serial. Callers must not pass fail_fast
     # here ({execute} forces serial for fail-fast). Per-mutant crashes are classified
-    # in {job_result}, shared with the serial path; a {DaemonBootError} ends the run
-    # here rather than scoring the remainder against a daemon that has given up.
+    # in {job_result}, shared with the serial path. Any error that escapes a worker
+    # (a {DaemonBootError}, or a defect) clears the queue, and the run ends once
+    # every worker has joined, rather than scoring the remainder first.
     #
     # @api private
+    # @param prepare_client [Mutineer::DaemonClient, nil] the daemon that copied
+    #   the databases. Quit once every worker has booted and taken its lock.
     # @return [Array<Mutineer::Result>] one result per input job, in input order.
-    def self.run_parallel(jobs, worker_count, config, abs_tests, coverage_map, source_map)
+    def self.run_parallel(jobs, worker_count, config, abs_tests, coverage_map, source_map, prepare_client = nil)
       results  = Array.new(jobs.size)
       progress = Progress.new(jobs.size)
       queue    = Queue.new
+      errors   = Queue.new
       jobs.each_index { |i| queue << i }
 
-      # Built one at a time so a refused spawn part-way (EMFILE under a high --jobs)
-      # can still quit the daemons already up. Array.new would lose every reference.
+      # Both lists grow one entry at a time, so a refused spawn part-way (EMFILE
+      # under a high --jobs) or a Ctrl-C leaves the ensure below every daemon
+      # already up. Each client joins its list before it starts, so one
+      # interrupted mid-boot is quit too. The prepare daemon stays up through
+      # this loop so its lock still covers the databases until each worker
+      # holds one.
       clients = []
+      threads = []
       begin
         worker_count.times do
-          clients << DaemonClient.new(boot: boot_config(config, abs_tests),
-                                      app_root: config.project_root).start
+          clients << DaemonClient.new(boot: boot_config(config, abs_tests), app_root: config.project_root)
+          clients.last.start
         end
-      rescue StandardError
-        clients.each(&:quit)
-        raise
+      ensure
+        prepare_client&.quit
       end
 
-      clients.each_with_index.map do |client, worker|
-        Thread.new do
-          # The abort below is re-raised by join and reported once there; without
-          # this Ruby also dumps the thread's backtrace, which the serial path never
-          # does. Same fault, same output, whatever --jobs is set to.
-          Thread.current.report_on_exception = false
+      clients.each_with_index do |client, worker|
+        threads << Thread.new do
           loop do
             i = begin
               queue.pop(true)
@@ -207,20 +317,35 @@ module Mutineer
             results[i] = job_result(jobs[i], i, client, worker, config, coverage_map, abs_tests, source_map)
             progress.tick
           end
-        rescue DaemonBootError
-          # The daemon gave up for good. Stop feeding the other workers rather
-          # than letting them score the rest of the run against a dead client;
-          # Thread#join re-raises this and ends the run.
+        rescue Exception => e # rubocop:disable Lint/RescueException
+          # The daemon gave up for good, or a defect escaped. Stop feeding the
+          # other workers, so they stop after their current mutant, and hand the
+          # error to the main thread. The thread itself ends cleanly.
           queue.clear
-          raise
+          errors << e
         ensure
           client.quit
         end
-      end.each(&:join)
+      end
+      # Join every worker before raising, so all daemons have quit and the
+      # end-of-run sweep sees each child gone.
+      threads.each(&:join)
+      raise errors.pop unless errors.empty?
 
       # Every job was popped by some worker and every pop assigns, so no slot can
-      # be nil here: an escaping exception aborts the run via join instead.
+      # be nil here: a worker error is raised above instead.
       results
+    ensure
+      # An exception from outside (Ctrl-C, a caller's Timeout) skips the joins
+      # above. Kill every worker first, so their daemons quit at the same time,
+      # then wait for them. A second Ctrl-C during that wait still exits.
+      queue&.clear
+      threads&.each(&:kill)
+      threads&.each(&:join)
+      # Quit every client too. One may have no thread yet (startup failed or
+      # was interrupted), and a thread killed before its block began never runs
+      # its own ensure. A second quit of a client is a no-op.
+      clients&.each(&:quit)
     end
 
     # Build the payload for one job, run it on the given daemon/worker, and attach
@@ -246,51 +371,66 @@ module Mutineer
       # Skip an invalid mutant tool-side: never ship a payload that would fail to
       # load and read as a false `killed`.
       # Narrow to covering tests (shared with the in-process path via
-      # Runner.coverage_selection, so scores match). :verdict = no_coverage/uncapturable,
-      # no fork. No map (build failed) → run the full --test set (fallback, not
-      # narrowed).
-      sel = coverage_map && Runner.coverage_selection(subject.file, mutation, subject, source, coverage_map)
+      # JobPlan.coverage_selection, so scores match). :verdict = no_coverage/uncapturable,
+      # no fork. No map, or an empty one (build failed) → run the full --test set
+      # (fallback, not narrowed).
+      sel = coverage_map && !coverage_map.map.empty? && JobPlan.coverage_selection(subject.file, mutation, subject, source, coverage_map)
       r =
         if Parser.parse_string(mutated).errors.any?
           Result.skipped
         elsif sel && sel[0] == :verdict
           sel[1]
+        elsif subject.unplaceable?("reload") # the daemon always reloads the whole file
+          # A :maybe mutant that cannot be placed stays no_coverage, as in-process.
+          sel && sel[0] == :maybe ? Result.no_coverage : Result.unplaceable
         else
           verdict = client.request(
             id: req_id, worker: worker, timeout: config.daemon_timeout || DEFAULT_TIMEOUT,
             payload: { "code" => mutated, "source_file" => File.expand_path(subject.file, config.project_root) },
             tests: sel ? sel[1] : abs_tests
           )
-          result_for(verdict)
+          # A survivor whose line ran at load, as in-process (Runner.run).
+          loaded = JobPlan.load_verdict(result_for(verdict), subject.file, mutation, subject, source, coverage_map)
+          # Tests that may not have run the mutant, as in-process (#254).
+          JobPlan.selection_verdict(sel && sel[0], loaded)
         end
       r.with(subject: subject, mutation: mutation, id: id)
     end
 
-    # The boot config the daemon needs to boot the app once: where to boot, the test
-    # load roots (so `require "test_helper"` resolves in every fork), framework, and
-    # whether this is Rails.
+    # The boot config the daemon needs to boot the app once: where to boot, the
+    # --require files to load after it, the test load roots (so
+    # `require "test_helper"` resolves in every fork), framework, and whether this
+    # is Rails.
     #
     # @param config [Mutineer::Config] the run config.
     # @param abs_tests [Array<String>] absolute --test paths.
     # @param coverage [Boolean] whether this daemon builds the coverage map.
+    # @param db_role [String, nil] "prepare" (copy databases on command),
+    #   "worker" (lock only), or nil to pick from `coverage`.
     # @return [Hash] the boot config shipped to the daemon.
-    def self.boot_config(config, abs_tests, coverage: false)
+    def self.boot_config(config, abs_tests, coverage: false, db_role: nil)
       {
         project_root: config.project_root,
         boot: File.expand_path(config.boot || "config/environment", config.project_root),
-        load_paths: Runner.test_load_roots(abs_tests),
+        # Required after the boot, as in-process (Runner.execute); also part of
+        # the coverage digest.
+        require_paths: config.require_paths.map { |f| File.expand_path(f, config.project_root) },
+        load_paths: JobPlan.test_load_roots(abs_tests),
         cache_dir: File.expand_path(config.cache_dir, config.project_root),
-        source_dirs: Runner.source_dirs(config), # so the daemon can sweep orphan mutant temps
+        source_dirs: JobPlan.source_dirs(config), # so the daemon can sweep orphan mutant temps
         framework: config.framework,
         rails: config.rails,
-        # Schema for per-worker DB isolation. Sent when present; the daemon
-        # skips worker-DB schema loading if the path is absent (e.g. structure.sql apps).
+        # Schema for per-worker DB isolation. Sent when present; the daemon loads
+        # it over a worker's copy of the test DB only when that copy is out of date.
         schema: schema_path(config),
         # Coverage narrowing. Only the short-lived map-building daemon starts
         # Coverage (before boot); worker daemons boot with it OFF (no wasted
         # instrumentation/memory across every mutant fork). `sources`/`tests` are the
         # map-build inputs.
         coverage: coverage,
+        # "prepare" copies worker databases when asked. "worker" only locks.
+        # A direct client that omits this provisions at boot instead.
+        db_role: db_role || (coverage ? "prepare" : "worker"),
         # The map-building daemon's capture limit (--capture-timeout); nil = default.
         capture_timeout: config.capture_timeout,
         sources: config.sources.map { |s| File.expand_path(s, config.project_root) },
@@ -298,10 +438,10 @@ module Mutineer
       }
     end
 
-    # Absolute path to the app's `db/schema.rb` if it exists, else nil. Used by the
-    # daemon to schema-load each fork's isolated worker database. Only `schema.rb`
-    # is supported this pass; `structure.sql` apps get nil and fall back to
-    # whatever the worker DB already holds.
+    # Absolute path to the app's `db/schema.rb` if it exists, else nil. Each worker
+    # DB starts as a copy of the test DB; the daemon loads this file over the copy
+    # when the copy's schema differs. `structure.sql` apps get nil and keep the
+    # copy as it is.
     #
     # @param config [Mutineer::Config] the run config.
     # @api private
@@ -331,6 +471,7 @@ module Mutineer
     # tests drive directly: {boot_config} from the zero-dep suite and
     # {build_coverage_map} from the daemon suite. Everything else is daemon-pipeline
     # internals with no caller outside this file.
-    private_class_method :run_serial, :run_parallel, :job_result, :schema_path, :result_for
+    private_class_method :sweep_daemon_temps, :run_serial, :run_parallel, :job_result, :schema_path, :result_for,
+                         :prepare_run, :coverage_map_from_client
   end
 end

data/lib/mutineer/daemon_client.rb

@@ -1,8 +1,10 @@
 # frozen_string_literal: true
 
 require "json"
+require "io/wait"
 require "open3"
 require_relative "external_backend"
+require_relative "rails_run_env"
 
 module Mutineer
   # Raised when the daemon cannot be booted or is gone for good: a bad boot path, an
@@ -12,6 +14,11 @@ module Mutineer
   # to a runtime error (exit 1).
   class DaemonBootError < StandardError; end
 
+  # A boot that has not answered the handshake within DaemonClient::BOOT_TIMEOUT
+  # (#101). Unlike other boot errors it is not worth retrying with a second
+  # daemon, which would wait just as long.
+  class DaemonBootTimeout < DaemonBootError; end
+
   # Tool-side handle for the app-side daemon.
   #
   # Spawns `daemon_server.rb` UNDER THE APP'S BUNDLE/RUBY. The child gets the
@@ -29,6 +36,13 @@ module Mutineer
     DAEMON_PATH = File.expand_path("daemon_server.rb", __dir__)
     # How many times to respawn a crashing daemon before aborting the run.
     MAX_RESTARTS = 3
+    # Seconds a verdict may lag the request's own timeout. The daemon enforces
+    # that timeout on the mutant child, so a reply later than this means the
+    # daemon itself is wedged (#101): it is killed and respawned.
+    REPLY_GRACE = 30
+    # Seconds the daemon gets to boot the app and answer the handshake. A boot
+    # that never returns ends the run instead of hanging it (#101).
+    BOOT_TIMEOUT = 600
     # Bundler's marker for a variable that was unset before it activated.
     BUNDLER_UNSET = "BUNDLER_ENVIRONMENT_PRESERVER_INTENTIONALLY_NIL"
 
@@ -58,6 +72,32 @@ module Mutineer
       self
     end
 
+    # How many database configs the booted test environment reported.
+    # 1 when the daemon did not send a count.
+    #
+    # @return [Integer]
+    def database_count
+      @database_count || 1
+    end
+
+    # Ask the daemon to create `slots` worker databases. Raises when the copy
+    # fails or another run holds the database, so the caller stops before any
+    # mutant is scored.
+    #
+    # @param slots [Integer] worker slots to create.
+    # @return [void]
+    # @raise [Mutineer::DaemonBootError]
+    def provision(slots)
+      raise DaemonBootError, "daemon is not running" if @stdin.nil?
+
+      send_line("cmd" => "provision", "slots" => slots)
+      reply = read_line(BOOT_TIMEOUT)
+      return if reply && reply["ok"]
+
+      detail = reply && reply["error"] ? reply["error"] : "provision failed"
+      raise DaemonBootError, detail
+    end
+
     # Run one mutant: ship the payload + covering tests, return the verdict string.
     # On a daemon crash (EOF/dead pipe) respawn (bounded) and return `"error"` for
     # this mutant. Never a wrong verdict, never a wedged run.
@@ -83,7 +123,7 @@ module Mutineer
       reply =
         begin
           send_line("id" => id, "worker" => worker, "payload" => payload, "tests" => tests, "timeout" => timeout)
-          read_line
+          read_line(timeout + REPLY_GRACE)
         rescue Errno::EPIPE, IOError
           nil
         end
@@ -117,7 +157,7 @@ module Mutineer
       return unless @stdin
 
       send_line("cmd" => "quit") rescue nil # rubocop:disable Style/RescueModifier
-      @wait_thr&.join
+      @wait_thr&.join(REPLY_GRACE) # a wedged daemon is killed by close_io
     ensure
       close_io
     end
@@ -137,6 +177,8 @@ module Mutineer
       env["BUNDLE_GEMFILE"] = @gemfile
       env["RBENV_VERSION"] = @ruby_version if @ruby_version
       env["RAILS_ENV"] = "test" if rails_boot? && !env.key?("RAILS_ENV")
+      # The daemon loads tests after boot. Rails reads this when parallelize runs.
+      RailsRunEnv.pin_child!(env)
       env
     end
 
@@ -315,17 +357,23 @@ module Mutineer
           end
 
           send_line(@boot)
-          read_line
+          read_line(BOOT_TIMEOUT)
         rescue SystemCallError, IOError => e
           close_io
           raise DaemonBootError, "daemon could not be started: #{e.class}: #{e.message}"
         end
 
       unless ready && ready["ready"]
-        detail = ready && ready["error"] ? ready["error"] : "daemon exited before the handshake"
+        detail =
+          if ready && ready["error"] then ready["error"]
+          elsif @timed_out then "the daemon did not finish booting within #{BOOT_TIMEOUT}s"
+          else "daemon exited before the handshake"
+          end
+        timed_out = @timed_out
         close_io
-        raise DaemonBootError, "daemon failed to boot under the app bundle: #{detail}"
+        raise (timed_out ? DaemonBootTimeout : DaemonBootError), "daemon failed to boot under the app bundle: #{detail}"
       end
+      @database_count = ready["database_count"] || 1
     end
 
     # Respawn after a crash, up to MAX_RESTARTS, then hard-fail loudly.
@@ -333,10 +381,11 @@ module Mutineer
       close_io
       @restarts += 1
       if @restarts > MAX_RESTARTS
-        raise DaemonBootError, "daemon crashed #{@restarts} times; aborting the run"
+        raise DaemonBootError, "daemon crashed or stopped answering #{@restarts} times; aborting the run"
       end
 
-      @errio.puts("[mutineer] daemon crashed — respawning (#{@restarts}/#{MAX_RESTARTS})")
+      cause = @timed_out ? "stopped answering" : "crashed"
+      @errio.puts("[mutineer] daemon #{cause} — respawning (#{@restarts}/#{MAX_RESTARTS})")
       spawn_daemon
     end
 
@@ -349,8 +398,15 @@ module Mutineer
       @stdin.flush
     end
 
-    # Read one JSON reply line; nil on EOF/dead pipe (caller treats as a crash).
-    def read_line
+    # Read one JSON reply line; nil on EOF/dead pipe, or when no line arrives
+    # within `timeout` seconds (caller treats either as a crash).
+    #
+    # @param timeout [Numeric, nil] seconds to wait; nil waits for the reply.
+    # @return [Hash, nil]
+    def read_line(timeout = nil)
+      @timed_out = timeout && !@stdout.wait_readable(timeout)
+      return nil if @timed_out
+
       line = @stdout.gets
       line && JSON.parse(line.strip)
     rescue IOError, Errno::EPIPE, JSON::ParserError
@@ -362,6 +418,13 @@ module Mutineer
     #
     # @return [void]
     def close_io
+      # A wedged daemon may never read the closed stdin, so the reap below would
+      # wait forever: kill it first. An exited daemon makes this a no-op.
+      begin
+        Process.kill(:KILL, @wait_thr.pid) if @wait_thr&.alive?
+      rescue Errno::ESRCH
+        nil # it exited between the check and the kill
+      end
       @drain&.kill # stop the drain BEFORE closing its fd (avoids a copy_stream EBADF)
       [@stdin, @stdout, @stderr].each { |io| io&.close rescue nil } # rubocop:disable Style/RescueModifier
       @wait_thr&.join # reap the exited daemon so respawn/quit leaves no zombie