mutineer 1.6.0 → 1.7.0

data/lib/mutineer/daemon_backend.rb

@@ -6,6 +6,7 @@ require_relative "coverage_map"
 require_relative "daemon_client"
 require_relative "progress"
 require_relative "job_plan"
+require_relative "orphan_guard"
 
 module Mutineer
   # Daemon execution backend. Boots the app ONCE in a persistent subprocess under
@@ -31,8 +32,13 @@ module Mutineer
     DEFAULT_TIMEOUT = 60
 
     # The daemon's per-mutant tempfile, written into the source dir so
-    # require_relative resolves. Kept in step with DaemonServer#sweep_temps.
-    DAEMON_TEMP_GLOB = "mutineer_daemon*.rb"
+    # require_relative resolves. The leading dot hides it from Zeitwerk (#253);
+    # older versions wrote it without one. Kept in step with DaemonServer#sweep_temps.
+    DAEMON_TEMP_GLOB = "{.,}mutineer_daemon*.rb"
+
+    # Seconds the end-of-run sweep waits for an orphaned child to die. Its
+    # watchdog notices a dead daemon within {OrphanGuard::POLL}.
+    SWEEP_GRACE = (OrphanGuard::POLL * 2) + 0.5
 
     # Full daemon run: collect jobs, build the coverage map once, then execute
     # serially or across N worker daemons. Fail-fast forces serial so the survivor
@@ -44,25 +50,14 @@ module Mutineer
     #   source map, and the {JobPlan.collect_jobs} extras.
     def self.execute(config, operator_classes)
       jobs, ignored_results, source_map, extras = JobPlan.collect_jobs(config, operator_classes)
-      jobs = JobPlan.filter_since(jobs, source_map, config) if config.since
+      jobs, ignored_results, extras[:unscoped_mutants] = JobPlan.scope_since(jobs, ignored_results, source_map, config)
       abs_tests = config.tests.map { |t| File.expand_path(t, config.project_root) }
 
       # Nothing to mutate (`--since` matched no changed line, or every mutant is
       # suppressed). Return before booting anything: the coverage daemon and the
       # worker daemons below each boot the whole app, and README documents
       # `--since origin/<base>` for PR CI, where a docs-only PR is routine.
-      if jobs.empty?
-        # The daemon sweeps orphaned temps at boot and nothing boots here, so sweep
-        # tool-side. A file a hard-killed run left in app/models breaks the app's own
-        # Zeitwerk boot, not just Mutineer's next run.
-        JobPlan.sweep_orphans(JobPlan.source_dirs(config), DAEMON_TEMP_GLOB)
-        return [AggregateResult.new(ignored_results), source_map, extras]
-      end
-
-      # Build the coverage map once (app-side). nil when the build fails: runners
-      # fall back to the full --test set (and emit a stderr warning) rather than
-      # mis-scoring everything as no_coverage.
-      coverage_map = build_coverage_map(config, abs_tests)
+      return [AggregateResult.new(ignored_results), source_map, extras] if jobs.empty?
 
       # Worker count = resolved --jobs, capped at the job count (no idle daemons).
       # >1 → N concurrent daemon handles, each on its OWN worker DB (N-handles, the
@@ -71,18 +66,60 @@ module Mutineer
       # input index, so the verdict set would diverge from serial (a different,
       # non-deterministic survivor set/score). The "identical to --jobs 1" guarantee
       # below only holds when fail-fast cannot race.
+      # A second database config is only known after the app boots, so the first
+      # daemon can still lower this count before any worker starts.
       worker_count = [config.jobs || 1, 1].max
       worker_count = 1 if config.fail_fast
       worker_count = [worker_count, jobs.size].min
 
-      results =
-        if worker_count > 1
-          run_parallel(jobs, worker_count, config, abs_tests, coverage_map, source_map)
-        else
-          run_serial(jobs, config, abs_tests, coverage_map, source_map)
-        end
+      # One booted process copies worker databases, then builds the coverage map.
+      # A cached map still provisions: the copy has to happen while no worker is
+      # connected to the template database. That process keeps its database lock
+      # until the worker daemons have taken theirs.
+      prepare_client = nil
+      begin
+        coverage_map, worker_count, prepare_client = prepare_run(config, abs_tests, worker_count)
+
+        results =
+          if worker_count > 1
+            run_parallel(jobs, worker_count, config, abs_tests, coverage_map, source_map, prepare_client)
+          else
+            run_serial(jobs, config, abs_tests, coverage_map, source_map, prepare_client)
+          end
+      ensure
+        # run_serial and run_parallel quit this client once workers hold locks.
+        # Quit again here when they never reached that point. A second quit is a no-op.
+        prepare_client&.quit
+      end
 
       [AggregateResult.new(results + ignored_results), source_map, extras]
+    ensure
+      sweep_daemon_temps(config)
+    end
+
+    # Removes daemon mutant files left in the source dirs, on every exit path.
+    # The daemon sweeps at boot, but an empty run boots nothing, and a daemon
+    # that dies mid-mutant orphans a child that lives on until its watchdog
+    # fires (#250). The respawned daemon sweeps before that, and a run that
+    # ends or aborts right after the crash does too. So a file whose owner is
+    # still alive gets {SWEEP_GRACE} to go. A clean run leaves no file and pays
+    # nothing. A file a concurrent run still owns stays in place, but looks the
+    # same as a dying child's, so this run then waits the full grace.
+    #
+    # @param config [Mutineer::Config] the run config.
+    # @return [void]
+    def self.sweep_daemon_temps(config)
+      dirs = JobPlan.source_dirs(config)
+      deadline = Process.clock_gettime(Process::CLOCK_MONOTONIC) + SWEEP_GRACE
+      loop do
+        still_owned = JobPlan.sweep_orphans(dirs, DAEMON_TEMP_GLOB)
+        break if still_owned.empty?
+        break if Process.clock_gettime(Process::CLOCK_MONOTONIC) >= deadline
+
+        sleep 0.1
+      end
+    rescue SystemCallError
+      nil # never replace the run's own result or exception with a cleanup error
     end
 
     # Build the coverage map via a short-lived daemon (boots the app once, captures
@@ -101,11 +138,64 @@ module Mutineer
     def self.build_coverage_map(config, abs_tests)
       client = DaemonClient.new(boot: boot_config(config, abs_tests, coverage: true),
                                 app_root: config.project_root).start
-      data = begin
-        client.coverage
+      begin
+        coverage_map_from_client(client, config)
       ensure
         client.quit
       end
+    rescue DaemonBootTimeout
+      raise
+    rescue DaemonBootError => e
+      warn_coverage_fallback("#{e.class}: #{e.message}")
+      nil
+    end
+
+    # Boot the first daemon, copy worker databases, then read the coverage map
+    # from that same process. A boot failure still falls back to the full test
+    # set. A provisioning failure does not: it stops the run.
+    #
+    # @param config [Mutineer::Config]
+    # @param abs_tests [Array<String>]
+    # @param worker_count [Integer] slots requested before the database count is known.
+    # @return [Array] the coverage map (or nil), the worker count to start, and
+    #   the prepare daemon. The caller quits that daemon after worker daemons
+    #   hold their locks. It is nil when this process could not boot one.
+    def self.prepare_run(config, abs_tests, worker_count)
+      client = nil
+      begin
+        client = DaemonClient.new(boot: boot_config(config, abs_tests, coverage: true),
+                                  app_root: config.project_root).start
+      rescue DaemonBootTimeout
+        raise
+      rescue DaemonBootError => e
+        warn_coverage_fallback("#{e.class}: #{e.message}")
+        return [nil, worker_count, nil]
+      end
+
+      begin
+        count = client.database_count.to_i
+        if count > 1 && worker_count > 1
+          warn "[mutineer] this app has #{count} databases; Mutineer runs one worker. " \
+               "Parallel runs support one database."
+          worker_count = 1
+        end
+        # Raises DaemonBootError. Callers must not score mutants after this.
+        client.provision(worker_count)
+        [coverage_map_from_client(client, config), worker_count, client]
+      rescue Exception # rubocop:disable Lint/RescueException
+        # An interrupt must also quit it, or the daemon keeps its database lock.
+        client.quit
+        raise
+      end
+    end
+
+    # Turn one daemon's coverage reply into a map. Does not start or stop the daemon.
+    #
+    # @param client [Mutineer::DaemonClient]
+    # @param config [Mutineer::Config]
+    # @return [Mutineer::CoverageMap, nil]
+    def self.coverage_map_from_client(client, config)
+      data = client.coverage
       # A red unmutated suite must abort, even when the shipped map is empty.
       # Falling back to the full --test set would treat those failures as kills.
       if data.is_a?(Hash) && Array(data["failed_clean_tests"]).any?
@@ -155,59 +245,69 @@ module Mutineer
     # (stop at the first survivor).
     #
     # @api private
+    # @param prepare_client [Mutineer::DaemonClient, nil] the daemon that copied
+    #   the databases. Quit once this worker has booted and taken its lock.
     # @return [Array<Mutineer::Result>] results in input order.
-    def self.run_serial(jobs, config, abs_tests, coverage_map, source_map)
-      client = DaemonClient.new(boot: boot_config(config, abs_tests),
-                                app_root: config.project_root).start
-      results = []
-      progress = Progress.new(jobs.size)
+    def self.run_serial(jobs, config, abs_tests, coverage_map, source_map, prepare_client = nil)
+      client = nil
       begin
+        client = DaemonClient.new(boot: boot_config(config, abs_tests),
+                                  app_root: config.project_root).start
+        prepare_client&.quit
+        prepare_client = nil
+        results = []
+        progress = Progress.new(jobs.size)
         jobs.each_with_index do |job, i|
           r = job_result(job, i, client, 0, config, coverage_map, abs_tests, source_map)
           results << r
           progress.tick
           break if config.fail_fast && r.survived?
         end
+        results
       ensure
-        client.quit
+        prepare_client&.quit
+        client&.quit
       end
-      results
     end
 
     # Parallel path: N daemon handles, each pinned to its own worker slot (own DB).
     # A shared queue of job indices feeds N tool-side threads; results are placed by
     # input index so the verdict set matches serial. Callers must not pass fail_fast
     # here ({execute} forces serial for fail-fast). Per-mutant crashes are classified
-    # in {job_result}, shared with the serial path; a {DaemonBootError} ends the run
-    # here rather than scoring the remainder against a daemon that has given up.
+    # in {job_result}, shared with the serial path. Any error that escapes a worker
+    # (a {DaemonBootError}, or a defect) clears the queue, and the run ends once
+    # every worker has joined, rather than scoring the remainder first.
     #
     # @api private
+    # @param prepare_client [Mutineer::DaemonClient, nil] the daemon that copied
+    #   the databases. Quit once every worker has booted and taken its lock.
     # @return [Array<Mutineer::Result>] one result per input job, in input order.
-    def self.run_parallel(jobs, worker_count, config, abs_tests, coverage_map, source_map)
+    def self.run_parallel(jobs, worker_count, config, abs_tests, coverage_map, source_map, prepare_client = nil)
       results  = Array.new(jobs.size)
       progress = Progress.new(jobs.size)
       queue    = Queue.new
+      errors   = Queue.new
       jobs.each_index { |i| queue << i }
 
-      # Built one at a time so a refused spawn part-way (EMFILE under a high --jobs)
-      # can still quit the daemons already up. Array.new would lose every reference.
+      # Both lists grow one entry at a time, so a refused spawn part-way (EMFILE
+      # under a high --jobs) or a Ctrl-C leaves the ensure below every daemon
+      # already up. Each client joins its list before it starts, so one
+      # interrupted mid-boot is quit too. The prepare daemon stays up through
+      # this loop so its lock still covers the databases until each worker
+      # holds one.
       clients = []
+      threads = []
       begin
         worker_count.times do
-          clients << DaemonClient.new(boot: boot_config(config, abs_tests),
-                                      app_root: config.project_root).start
+          clients << DaemonClient.new(boot: boot_config(config, abs_tests), app_root: config.project_root)
+          clients.last.start
         end
-      rescue StandardError
-        clients.each(&:quit)
-        raise
+      ensure
+        prepare_client&.quit
       end
 
-      clients.each_with_index.map do |client, worker|
-        Thread.new do
-          # The abort below is re-raised by join and reported once there; without
-          # this Ruby also dumps the thread's backtrace, which the serial path never
-          # does. Same fault, same output, whatever --jobs is set to.
-          Thread.current.report_on_exception = false
+      clients.each_with_index do |client, worker|
+        threads << Thread.new do
           loop do
             i = begin
               queue.pop(true)
@@ -217,20 +317,35 @@ module Mutineer
             results[i] = job_result(jobs[i], i, client, worker, config, coverage_map, abs_tests, source_map)
             progress.tick
           end
-        rescue DaemonBootError
-          # The daemon gave up for good. Stop feeding the other workers rather
-          # than letting them score the rest of the run against a dead client;
-          # Thread#join re-raises this and ends the run.
+        rescue Exception => e # rubocop:disable Lint/RescueException
+          # The daemon gave up for good, or a defect escaped. Stop feeding the
+          # other workers, so they stop after their current mutant, and hand the
+          # error to the main thread. The thread itself ends cleanly.
           queue.clear
-          raise
+          errors << e
         ensure
           client.quit
         end
-      end.each(&:join)
+      end
+      # Join every worker before raising, so all daemons have quit and the
+      # end-of-run sweep sees each child gone.
+      threads.each(&:join)
+      raise errors.pop unless errors.empty?
 
       # Every job was popped by some worker and every pop assigns, so no slot can
-      # be nil here: an escaping exception aborts the run via join instead.
+      # be nil here: a worker error is raised above instead.
       results
+    ensure
+      # An exception from outside (Ctrl-C, a caller's Timeout) skips the joins
+      # above. Kill every worker first, so their daemons quit at the same time,
+      # then wait for them. A second Ctrl-C during that wait still exits.
+      queue&.clear
+      threads&.each(&:kill)
+      threads&.each(&:join)
+      # Quit every client too. One may have no thread yet (startup failed or
+      # was interrupted), and a thread killed before its block began never runs
+      # its own ensure. A second quit of a client is a no-op.
+      clients&.each(&:quit)
     end
 
     # Build the payload for one job, run it on the given daemon/worker, and attach
@@ -265,6 +380,9 @@ module Mutineer
           Result.skipped
         elsif sel && sel[0] == :verdict
           sel[1]
+        elsif subject.unplaceable?("reload") # the daemon always reloads the whole file
+          # A :maybe mutant that cannot be placed stays no_coverage, as in-process.
+          sel && sel[0] == :maybe ? Result.no_coverage : Result.unplaceable
         else
           verdict = client.request(
             id: req_id, worker: worker, timeout: config.daemon_timeout || DEFAULT_TIMEOUT,
@@ -272,7 +390,9 @@ module Mutineer
             tests: sel ? sel[1] : abs_tests
           )
           # A survivor whose line ran at load, as in-process (Runner.run).
-          JobPlan.load_verdict(result_for(verdict), subject.file, mutation, subject, source, coverage_map)
+          loaded = JobPlan.load_verdict(result_for(verdict), subject.file, mutation, subject, source, coverage_map)
+          # Tests that may not have run the mutant, as in-process (#254).
+          JobPlan.selection_verdict(sel && sel[0], loaded)
         end
       r.with(subject: subject, mutation: mutation, id: id)
     end
@@ -285,8 +405,10 @@ module Mutineer
     # @param config [Mutineer::Config] the run config.
     # @param abs_tests [Array<String>] absolute --test paths.
     # @param coverage [Boolean] whether this daemon builds the coverage map.
+    # @param db_role [String, nil] "prepare" (copy databases on command),
+    #   "worker" (lock only), or nil to pick from `coverage`.
     # @return [Hash] the boot config shipped to the daemon.
-    def self.boot_config(config, abs_tests, coverage: false)
+    def self.boot_config(config, abs_tests, coverage: false, db_role: nil)
       {
         project_root: config.project_root,
         boot: File.expand_path(config.boot || "config/environment", config.project_root),
@@ -306,6 +428,9 @@ module Mutineer
         # instrumentation/memory across every mutant fork). `sources`/`tests` are the
         # map-build inputs.
         coverage: coverage,
+        # "prepare" copies worker databases when asked. "worker" only locks.
+        # A direct client that omits this provisions at boot instead.
+        db_role: db_role || (coverage ? "prepare" : "worker"),
         # The map-building daemon's capture limit (--capture-timeout); nil = default.
         capture_timeout: config.capture_timeout,
         sources: config.sources.map { |s| File.expand_path(s, config.project_root) },
@@ -346,6 +471,7 @@ module Mutineer
     # tests drive directly: {boot_config} from the zero-dep suite and
     # {build_coverage_map} from the daemon suite. Everything else is daemon-pipeline
     # internals with no caller outside this file.
-    private_class_method :run_serial, :run_parallel, :job_result, :schema_path, :result_for
+    private_class_method :sweep_daemon_temps, :run_serial, :run_parallel, :job_result, :schema_path, :result_for,
+                         :prepare_run, :coverage_map_from_client
   end
 end

data/lib/mutineer/daemon_client.rb

@@ -4,6 +4,7 @@ require "json"
 require "io/wait"
 require "open3"
 require_relative "external_backend"
+require_relative "rails_run_env"
 
 module Mutineer
   # Raised when the daemon cannot be booted or is gone for good: a bad boot path, an
@@ -71,6 +72,32 @@ module Mutineer
       self
     end
 
+    # How many database configs the booted test environment reported.
+    # 1 when the daemon did not send a count.
+    #
+    # @return [Integer]
+    def database_count
+      @database_count || 1
+    end
+
+    # Ask the daemon to create `slots` worker databases. Raises when the copy
+    # fails or another run holds the database, so the caller stops before any
+    # mutant is scored.
+    #
+    # @param slots [Integer] worker slots to create.
+    # @return [void]
+    # @raise [Mutineer::DaemonBootError]
+    def provision(slots)
+      raise DaemonBootError, "daemon is not running" if @stdin.nil?
+
+      send_line("cmd" => "provision", "slots" => slots)
+      reply = read_line(BOOT_TIMEOUT)
+      return if reply && reply["ok"]
+
+      detail = reply && reply["error"] ? reply["error"] : "provision failed"
+      raise DaemonBootError, detail
+    end
+
     # Run one mutant: ship the payload + covering tests, return the verdict string.
     # On a daemon crash (EOF/dead pipe) respawn (bounded) and return `"error"` for
     # this mutant. Never a wrong verdict, never a wedged run.
@@ -150,6 +177,8 @@ module Mutineer
       env["BUNDLE_GEMFILE"] = @gemfile
       env["RBENV_VERSION"] = @ruby_version if @ruby_version
       env["RAILS_ENV"] = "test" if rails_boot? && !env.key?("RAILS_ENV")
+      # The daemon loads tests after boot. Rails reads this when parallelize runs.
+      RailsRunEnv.pin_child!(env)
       env
     end
 
@@ -344,6 +373,7 @@ module Mutineer
         close_io
         raise (timed_out ? DaemonBootTimeout : DaemonBootError), "daemon failed to boot under the app bundle: #{detail}"
       end
+      @database_count = ready["database_count"] || 1
     end
 
     # Respawn after a crash, up to MAX_RESTARTS, then hard-fail loudly.