mutineer 0.10.0 → 0.11.1

data/lib/mutineer/daemon_server.rb

@@ -20,17 +20,23 @@ module Mutineer
   #
   # Protocol (one JSON object per line, both directions):
   #   boot in  : {"cmd":"boot","project_root":"...","boot":"config/environment",
-  #               "load_paths":["test"],"framework":"minitest","rails":true}
+  #               "load_paths":["test"],"framework":"minitest","rails":true,"schema":"db/schema.rb"}
   #   ready out: {"ready":true,"ruby":"3.3.6"}   (or {"ready":false,"error":"..."} then exit)
-  #   run  in  : {"id":N,"payload":{"code":"<ruby>","source_file":"app/models/order.rb"},
+  #   run  in  : {"id":N,"worker":I,"payload":{"code":"<ruby>","source_file":"app/models/order.rb"},
   #               "tests":["test/models/order_test.rb"],"timeout":30}
   #   verdict  : {"id":N,"verdict":"survived"|"killed"|"error"|"timeout"}
   #   quit in  : {"cmd":"quit"}
   #
+  # Worker isolation (#26/U5): when the app is Rails, each fork is routed to its own
+  # database `<db>-<worker>` via {RailsWorkerDb} BEFORE any test loads, so concurrent
+  # workers can't clobber each other's transactional fixtures. `worker` defaults to 0
+  # (serial). SQLite this pass; Postgres provisioning is U10.
+  #
   # Verdict mapping (KTD-5, Phase-2a honest limit): child exit 0=survived (suite
-  # passed), 1=killed (suite failed), 2=error (child raised AROUND the test — load or
-  # boot failure); parent-detected timeout. Tagging an in-TEST DB error as `error`
-  # (vs killed) is a Phase-2b concern (needs the after_fork adapter's re-raise).
+  # passed), 1=killed (suite failed), 2=error (child raised AROUND the test — load,
+  # boot, or worker-DB routing failure); parent-detected timeout. Tagging an in-TEST DB
+  # error (one fired inside a test body, vs at routing time) as `error` rather than
+  # `killed` is a U6 concern — only observable under the concurrent gate.
   module DaemonServer
     # Poll interval (seconds) for the per-fork deadline wait loop.
     POLL = 0.02
@@ -67,6 +73,14 @@ module Mutineer
           end
           break if req["cmd"] == "quit"
 
+          # #26/U7: build the coverage map app-side and ship it to the tool, which
+          # then selects covering tests per mutant. One-shot control message.
+          if req["cmd"] == "coverage"
+            output.puts(JSON.generate(build_coverage_map))
+            output.flush
+            next
+          end
+
           output.puts(JSON.generate(run_mutant(req)))
           output.flush
         end
@@ -77,16 +91,24 @@ module Mutineer
       # BOOT ONCE. chdir + require the app's boot file so the whole app is loaded and
       # inherited by every fork. Never requires mutineer.
       def boot!(cfg)
+        @cfg = cfg
         @framework = cfg.fetch("framework", "minitest")
         @source_dirs = Array(cfg["source_dirs"]).map { |d| File.expand_path(d) }
         Dir.chdir(cfg["project_root"]) if cfg["project_root"]
         ENV["RAILS_ENV"] ||= "test" if cfg["rails"]
         Array(cfg["load_paths"]).each { |d| $LOAD_PATH.unshift(File.expand_path(d)) }
+        # #26/U7: start Coverage BEFORE the app loads, so booted source lines are
+        # instrumented — the map build (build_via_fork) forks this booted parent.
+        if cfg["coverage"]
+          require "coverage"
+          Coverage.start(lines: true)
+        end
         # Clear any mutant tempfile a prior SIGKILLed timeout child orphaned in a
         # source dir BEFORE the app boots — Zeitwerk would otherwise choke on the
         # tempfile's non-constant name during autoload setup.
         sweep_temps
         require File.expand_path(cfg["boot"]) if cfg["boot"]
+        setup_worker_db(cfg) if cfg["rails"]
       rescue Exception => e # rubocop:disable Lint/RescueException
         # Boot failed (bad boot path, app error) — tell the client and exit so it can
         # surface a clean error rather than hang on the handshake.
@@ -95,17 +117,68 @@ module Mutineer
         exit!(1)
       end
 
+      # Load the per-worker DB adapter app-side (sibling gem file, by relative path so
+      # it bypasses the app bundle — like this daemon itself). No-op unless the app has
+      # ActiveRecord. Records the adapter + schema path so each fork can route to its
+      # own database (#26 isolation, U5). SQLite-only this pass; a non-SQLite config
+      # raises in the fork and reads as `error`, never a mis-routed verdict.
+      def setup_worker_db(cfg)
+        require_relative "rails_worker_db"
+        @worker_db = RailsWorkerDb.available? ? RailsWorkerDb : nil
+        schema = cfg["schema"] && File.expand_path(cfg["schema"])
+        @schema_path = schema if schema && File.exist?(schema)
+        # Schema is loaded once per worker slot on first use (not every mutant fork).
+        @schema_ready = {}
+      rescue LoadError => e
+        @errio.puts("[daemon] worker-DB routing unavailable: #{e.message}")
+        @worker_db = nil
+      end
+
+      # #26/U7: build the coverage map app-side (Coverage was started at boot) and
+      # return it as `{map, failed_test_files}` for the tool to select covering tests.
+      # Capture forks route to worker 0's DB (isolated, serial). On any failure return
+      # an empty map + an error string — the tool then falls back to the full test set
+      # rather than mis-scoring everything as no_coverage.
+      def build_coverage_map
+        require_relative "coverage_map"
+        root = @cfg["project_root"] || Dir.pwd
+        cmap = CoverageMap.new(
+          source_paths: Array(@cfg["sources"]), test_paths: Array(@cfg["tests"]),
+          load_paths: Array(@cfg["load_paths"]), project_root: root,
+          boot_path: @cfg["boot"], framework: @framework, cache_dir: File.join(root, ".mutineer")
+        ).build_via_fork(after_fork: coverage_after_fork)
+        { "map" => cmap.map, "failed_test_files" => cmap.failed_test_files }
+      rescue Exception => e # rubocop:disable Lint/RescueException
+        @errio.puts("[daemon] coverage build failed: #{e.class}: #{e.message}")
+        { "map" => {}, "failed_test_files" => [], "error" => "#{e.class}: #{e.message}" }
+      end
+
+      # Fork-safety hook for coverage capture: route each capture fork to worker 0's
+      # isolated DB (captures run serially, so one worker is enough). Nil when the app
+      # has no worker-DB adapter (non-Rails) — capture then runs as before.
+      def coverage_after_fork
+        return nil unless @worker_db
+
+        schema = @schema_path
+        -> { @worker_db.after_fork(0, schema) }
+      end
+
       # Fork a child to run one mutant in isolation; decode its exit into a verdict.
       def run_mutant(req)
         timeout = req.fetch("timeout", 30)
+        worker  = req.fetch("worker", 0)
+        # Load schema until the first killed/survived fork for this worker slot.
+        schema_for_fork = (@worker_db && @schema_path && !@schema_ready[worker]) ? @schema_path : nil
         pid = fork do
-          # New process group so a per-fork timeout can SIGKILL the whole subtree
-          # (carries the Phase-1 pgroup discipline), and silence the child's stdout so
-          # test-framework output can never corrupt the IPC pipe (KTD-6).
+          # New process group so a per-fork timeout can SIGKILL the whole subtree,
+          # and silence the child's stdout so test output never corrupts the IPC pipe.
           Process.setpgid(0, 0) rescue nil # rubocop:disable Style/RescueModifier
           $stdout.reopen(File::NULL, "w")
           code =
             begin
+              # Route THIS fork at its own worker database before any test loads.
+              # A routing failure raises here and is scored `error`, never a false verdict.
+              @worker_db&.after_fork(worker, schema_for_fork)
               apply_payload(req["payload"])
               run_tests(Array(req["tests"]))
             rescue Exception => e # rubocop:disable Lint/RescueException
@@ -115,6 +188,10 @@ module Mutineer
           exit!(code)
         end
         verdict = wait_verdict(pid, timeout)
+        # Mark ready only when the child finished cleanly after schema load
+        # (killed/survived). Timeout can interrupt mid-load_schema; error is a
+        # routing failure — both leave the slot unready so the next fork reloads.
+        @schema_ready[worker] = true if schema_for_fork && %w[killed survived].include?(verdict)
         # A SIGKILLed timeout child skipped its Tempfile unlink — sweep the orphan so
         # it can't outlive the run or trip Zeitwerk on a later fork.
         sweep_temps if verdict == "timeout"
@@ -151,7 +228,14 @@ module Mutineer
             rescue Errno::ESRCH, Errno::EPERM
               Process.kill(:KILL, pid) rescue nil # rubocop:disable Style/RescueModifier
             end
-            Process.waitpid(pid) rescue nil # rubocop:disable Style/RescueModifier
+            begin
+              _reaped, status = Process.waitpid2(pid)
+              if status && status.exited? && !status.signaled?
+                return { 0 => "survived", 1 => "killed" }.fetch(status.exitstatus, "error")
+              end
+            rescue Errno::ECHILD
+              # already reaped
+            end
             return "timeout"
           end
           sleep POLL

data/lib/mutineer/external_backend.rb

@@ -62,6 +62,12 @@ module Mutineer
         Result.timeout
       else # :exited
         return Result.survived if code&.zero?
+        # Signal death (nil exitstatus) is infrastructure, not a suite assertion
+        # failure — match Isolation/daemon (error), do not inflate kill rate.
+        if code.nil?
+          warn output if verbose && !output.empty?
+          return Result.error("test-command terminated by signal")
+        end
 
         warn output if verbose && !output.empty?
         Result.killed
@@ -151,7 +157,13 @@ module Mutineer
           rescue Errno::ESRCH, Errno::EPERM
             Process.kill(:KILL, pid) rescue nil # rubocop:disable Style/RescueModifier
           end
-          Process.waitpid(pid) rescue nil # rubocop:disable Style/RescueModifier
+          begin
+            _reaped, status = Process.waitpid2(pid)
+            # Finished cleanly between poll and kill — honor real exit code.
+            return [:exited, status.exitstatus] if status && status.exited? && !status.signaled?
+          rescue Errno::ECHILD
+            # already reaped
+          end
           return [:timeout, nil]
         end
         sleep POLL

data/lib/mutineer/isolation.rb

@@ -31,6 +31,9 @@ module Mutineer
     # @return [Mutineer::Result] result from the child process.
     def self.run(timeout: DEFAULT_TIMEOUT)
       pid = fork do
+        # Own process group so a timeout kill can reap grandchildren (match
+        # daemon/external backends). Best-effort: if setpgid fails, kill the pid.
+        Process.setpgid(0, 0) rescue nil # rubocop:disable Style/RescueModifier
         code = 0
         begin
           result = yield
@@ -48,20 +51,27 @@ module Mutineer
         exit!(code)
       end
 
-      # Single-threaded deadline poll (R2): we are the ONLY caller of waitpid
-      # on this pid, so we never reap-then-kill. We SIGKILL only after WNOHANG
-      # shows the child is still alive past the deadline — so the kill can
-      # never hit a reaped/recycled pid. Timeout is a parent-side fact
-      # (deadline reached), not status.signaled? (which is true for ANY signal
-      # death, e.g. SIGSEGV).
+      # Single-threaded deadline poll: we are the ONLY caller of waitpid on this
+      # pid, so we never reap-then-kill. SIGKILL the process group only after
+      # WNOHANG shows the child is still alive past the deadline.
       deadline = Process.clock_gettime(Process::CLOCK_MONOTONIC) + timeout
       loop do
         reaped, status = Process.waitpid2(pid, Process::WNOHANG)
         return decode(status) if reaped
 
         if Process.clock_gettime(Process::CLOCK_MONOTONIC) >= deadline
-          Process.kill(:KILL, pid) rescue nil # rubocop:disable Style/RescueModifier
-          Process.waitpid(pid) rescue nil # rubocop:disable Style/RescueModifier
+          begin
+            Process.kill(:KILL, -pid)
+          rescue Errno::ESRCH, Errno::EPERM
+            Process.kill(:KILL, pid) rescue nil # rubocop:disable Style/RescueModifier
+          end
+          begin
+            _reaped, status = Process.waitpid2(pid)
+            # Child may have finished cleanly between WNOHANG and kill; honor it.
+            return decode(status) if status && status.exited? && !status.signaled?
+          rescue Errno::ECHILD
+            # already reaped
+          end
           return Result.timeout
         end
         sleep 0.005

data/lib/mutineer/mutators/statement_removal.rb

@@ -14,17 +14,19 @@ module Mutineer
       # @return [void]
       def visit_statements_node(node)
         stmts = node.body
-        return if stmts.length < 2
-
-        stmts[0...-1].each do |stmt|
-          loc = stmt.location
-          @mutations << Mutation.new(
-            start_offset: loc.start_offset,
-            end_offset: loc.end_offset,
-            replacement: "nil",
-            operator: :statement_removal
-          )
+        if stmts.length >= 2
+          stmts[0...-1].each do |stmt|
+            loc = stmt.location
+            @mutations << Mutation.new(
+              start_offset: loc.start_offset,
+              end_offset: loc.end_offset,
+              replacement: "nil",
+              operator: :statement_removal
+            )
+          end
         end
+        # Nested statement lists (if/begin/while bodies) are separate StatementsNodes.
+        super
       end
     end
   end

data/lib/mutineer/rails_worker_db.rb

@@ -0,0 +1,137 @@
+# frozen_string_literal: true
+
+module Mutineer
+  # #26/#27 Phase 2b (U5) — per-worker database isolation for the daemon path.
+  #
+  # Loaded APP-SIDE by {DaemonServer} (a sibling gem file, pulled in by absolute path
+  # so it bypasses the app bundle — the same trick {DaemonClient} uses to run
+  # `daemon_server.rb` under a bundle that has no mutineer). It uses the app's OWN
+  # already-booted ActiveRecord and NEVER `require "active_record"` (R10/KTD-8): every
+  # method that touches AR first confirms {available?}, so the daemon core stays
+  # framework-agnostic and the gem keeps its zero-runtime-dependency promise.
+  #
+  # Isolation model (KTD-7): each parallel worker gets its OWN database so concurrent
+  # forks can't clobber each other's transactional fixtures (the measured #26
+  # corruption). {after_fork} runs inside a freshly-forked child and points that
+  # child's connection at the worker's database BEFORE any test loads; transactional
+  # fixtures then repopulate that isolated database per test.
+  #
+  # Scope: this pass ships the **SQLite** adapter (per-worker file, hermetic,
+  # spike-proven). Postgres per-worker DBs (`CREATE DATABASE <db>-<worker>`) are not
+  # implemented yet; a non-SQLite config raises a clear NotImplementedError rather
+  # than silently mis-routing.
+  #
+  # Honest limit (KTD-5): routing failures surface as `error` via {verify_connection!}.
+  # Re-raising an AR error that fires *inside a test body* past Minitest (so an in-test
+  # DB failure is `error`, not `killed`) is only observable under concurrent load and
+  # is deferred to U6 with the parallel gate — noted, not silently skipped.
+  module RailsWorkerDb
+    # True when the app has ActiveRecord loaded — the only condition under which any
+    # other method here may touch AR. Never triggers an autoload/require of AR itself.
+    #
+    # @return [Boolean]
+    def self.available?
+      defined?(ActiveRecord::Base) ? true : false
+    end
+
+    # Derive a per-worker database path from a base path by inserting `-<worker>`
+    # before the extension. Pure string transform (no AR) so it is unit-testable in
+    # the zero-dep suite. `storage/test.sqlite3`, worker 1 -> `storage/test-1.sqlite3`.
+    #
+    # @param database [String] the base database path.
+    # @param worker [Integer] the worker slot index (0..N-1).
+    # @return [String] the per-worker database path.
+    def self.worker_database_path(database, worker)
+      ext = File.extname(database)
+      "#{database.delete_suffix(ext)}-#{worker}#{ext}"
+    end
+
+    # Build the AR connection config for one worker by copying the app's current
+    # (default test) config and swapping in the per-worker database path. SQLite only
+    # this pass — a non-SQLite adapter raises so the SQLite-first scope fails loud
+    # instead of mis-routing (Postgres is U10).
+    #
+    # @param worker [Integer] the worker slot index.
+    # @return [Hash] a symbol-keyed AR configuration hash for the worker database.
+    # @raise [NotImplementedError] when the app's database is non-SQLite or in-memory.
+    def self.worker_db_config(worker)
+      hash    = ActiveRecord::Base.connection_db_config.configuration_hash
+      adapter = hash[:adapter].to_s
+      # U10 seam: the config-SHAPING below (per_worker_config) is already
+      # adapter-general — it derives correct SQLite *and* Postgres worker-DB names.
+      # What's gated is runtime PROVISIONING: SQLite files are created on connect,
+      # but Postgres needs an explicit `CREATE DATABASE` per worker (U10). Until that
+      # lands, refuse non-SQLite loudly rather than route to a database that doesn't
+      # exist until Postgres worker creation is implemented.
+      unless adapter.start_with?("sqlite")
+        raise NotImplementedError,
+              "worker-DB isolation currently provisions SQLite only (got adapter #{adapter.inspect}); " \
+              "Postgres per-worker provisioning is U10 (#26/#35) — use a SQLite test DB, or drop --jobs."
+      end
+
+      per_worker_config(hash, worker)
+    end
+
+    # Pure config-shaping (no AR): given a connection config hash, return the
+    # per-worker variant with its database swapped to the worker's own name.
+    # Adapter-general — SQLite (`storage/test.sqlite3` → `storage/test-<w>.sqlite3`)
+    # and Postgres (`myapp_test` → `myapp_test-<w>`, Rails `parallelize` naming) both
+    # fall out of {worker_database_path}. Extracted + unit-tested so the Postgres
+    # SHAPE is proven ready for U10 without a live database.
+    #
+    # @param config_hash [Hash] a connection config hash (symbol or string keys).
+    # @param worker [Integer] the worker slot index.
+    # @return [Hash] the per-worker config (symbol keys), database swapped.
+    # @raise [NotImplementedError] for an in-memory or empty database (no per-worker split).
+    def self.per_worker_config(config_hash, worker)
+      hash     = config_hash.transform_keys(&:to_sym)
+      database = hash[:database].to_s
+      if database.empty? || database == ":memory:"
+        raise NotImplementedError,
+              "worker-DB isolation needs a file/name-backed database (got #{database.inspect})."
+      end
+
+      hash.merge(database: worker_database_path(database, worker))
+    end
+
+    # Child-side (after fork): route this process's ActiveRecord at the worker's own
+    # database and confirm it is reachable, so a routing failure reads as `error`
+    # (via the daemon's child rescue) rather than a false verdict. Loads the schema
+    # into the worker database when a schema path is given (idempotent — schema.rb
+    # runs with `force: true`), covering a fresh worker file.
+    #
+    # @param worker [Integer] the worker slot index.
+    # @param schema_path [String, nil] absolute path to `db/schema.rb`, or nil to skip.
+    # @return [void]
+    def self.after_fork(worker, schema_path = nil)
+      return unless available?
+
+      ActiveRecord::Base.establish_connection(worker_db_config(worker))
+      load_schema(schema_path) if schema_path
+      verify_connection!
+    end
+
+    # Load a Rails `schema.rb` into the current connection with output silenced
+    # (fork child stdout is already File::NULL; this is belt-and-braces).
+    #
+    # @param schema_path [String] absolute path to `db/schema.rb`.
+    # @return [void]
+    def self.load_schema(schema_path)
+      ActiveRecord::Migration.verbose = false if defined?(ActiveRecord::Migration)
+      original = $stdout
+      $stdout = File.open(File::NULL, "w")
+      load schema_path
+    ensure
+      $stdout.close unless $stdout.equal?(original)
+      $stdout = original
+    end
+
+    # Force a round-trip to the freshly-routed connection so a broken route fails HERE
+    # (→ `error`) instead of later masquerading as a test failure (→ false `killed`).
+    #
+    # @return [void]
+    def self.verify_connection!
+      ActiveRecord::Base.connection.execute("SELECT 1")
+    end
+  end
+end

data/lib/mutineer/reporter.rb

@@ -69,12 +69,21 @@ module Mutineer
       verdict(out, threshold) if threshold && threshold.positive?
     end
 
-    # 0 pass / 1 below threshold. Usage errors (exit 2) are the CLI's job.
+    # 0 pass / 1 below threshold or untestable-with-errors. Usage errors (exit 2)
+    # are the CLI's job. When the score is nil (nothing killed or survived), pure
+    # no_coverage / all-ignored / empty still skip the gate; if any mutant was
+    # errored, timed out, or uncapturable, fail the gate so a broken harness
+    # cannot green CI under --threshold.
     def exit_code(threshold:)
       return 0 if threshold.nil? || threshold <= 0
 
       score = @agg.mutation_score
-      return 0 if score.nil? # no testable mutants — gate skipped (warning already emitted)
+      if score.nil?
+        broken = @agg.errored_count + @agg.timeout_count + @agg.uncapturable_count
+        return 1 if @agg.total.positive? && broken.positive?
+
+        return 0 # pure no_coverage / ignored / empty — gate skipped
+      end
 
       score >= threshold ? 0 : 1
     end
@@ -380,12 +389,38 @@ module Mutineer
                  "#{@agg.ignored_count} ignored excluded"
       if score.nil?
         out.puts "Mutation score: N/A  (no covered mutants)"
-        err.puts "[mutineer] no covered mutations; mutation score is N/A and the threshold check is skipped."
+        if broken_nil_score?
+          err.puts "[mutineer] no covered mutations (#{broken_nil_score_detail}); " \
+                   "threshold gate fails under a positive --threshold (broken harness)."
+        else
+          err.puts "[mutineer] no covered mutations; mutation score is N/A and the threshold check is skipped."
+        end
       else
         out.puts "Mutation score: #{score}%  (killed / (killed + survived); #{excluded})"
       end
     end
 
+    # True when nil score is due to errors/timeouts/uncapturable (gate must fail).
+    #
+    # @api private
+    # @return [Boolean]
+    def broken_nil_score?
+      @agg.total.positive? &&
+        (@agg.errored_count + @agg.timeout_count + @agg.uncapturable_count).positive?
+    end
+
+    # Human-readable counts for a broken nil-score run.
+    #
+    # @api private
+    # @return [String]
+    def broken_nil_score_detail
+      parts = []
+      parts << "#{@agg.errored_count} errored" if @agg.errored_count.positive?
+      parts << "#{@agg.timeout_count} timeout" if @agg.timeout_count.positive?
+      parts << "#{@agg.uncapturable_count} uncapturable" if @agg.uncapturable_count.positive?
+      parts.join(", ")
+    end
+
     # #11: one line per source after the global summary, so a multi-source run
     # shows which file is weak. Omitted for a single-source run — the global
     # summary already says everything (ponytail: no redundant one-line block).
@@ -470,7 +505,13 @@ module Mutineer
     # @return [void]
     def verdict(out, threshold)
       score = @agg.mutation_score
-      return if score.nil?
+      if score.nil?
+        if broken_nil_score?
+          out.puts "FAILED: no covered mutants (#{broken_nil_score_detail}); " \
+                   "threshold #{threshold}% cannot pass with a broken harness"
+        end
+        return
+      end
 
       if score >= threshold
         out.puts "PASSED: #{score}% >= threshold #{threshold}%"