mutineer 1.6.0 → 1.7.0

data/lib/mutineer/daemon_server.rb

@@ -3,7 +3,10 @@
 require "json"
 require "tempfile"
 require_relative "child_stdout"
+require_relative "child_coverage"
 require_relative "orphan_guard"
+require_relative "concern_blocks"
+require_relative "redaction"
 
 module Mutineer
   # App-side daemon (persistent worker).
@@ -26,17 +29,19 @@ module Mutineer
   # Protocol (one JSON object per line, both directions):
   #   boot in  : {"cmd":"boot","project_root":"...","boot":"config/environment","require_paths":[...],
   #               "load_paths":["test"],"framework":"minitest","rails":true,"schema":"db/schema.rb"}
-  #   ready out: {"ready":true,"ruby":"3.3.6"}   (or {"ready":false,"error":"..."} then exit)
+  #   ready out: {"ready":true,"ruby":"3.3.6","database_count":1}
+  #              (or {"ready":false,"error":"..."} then exit)
+  #   provision: {"cmd":"provision","slots":N} -> {"ok":true} or {"ok":false,"error":"..."}
   #   run  in  : {"id":N,"worker":I,"payload":{"code":"<ruby>","source_file":"app/models/order.rb"},
   #               "tests":["test/models/order_test.rb"],"timeout":30}
   #   verdict  : {"id":N,"verdict":"survived"|"killed"|"error"|"timeout"}
   #   quit in  : {"cmd":"quit"}
   #
   # Worker isolation: when the app is Rails, each fork is routed to its own
-  # database `<db>-<worker>` via {RailsWorkerDb} BEFORE any test loads, so
-  # concurrent workers cannot clobber each other's transactional fixtures.
-  # `worker` defaults to 0 (serial). SQLite this pass; Postgres provisioning is
-  # not yet implemented.
+  # database via {RailsWorkerDb} BEFORE any test loads, so concurrent workers
+  # cannot clobber each other's transactional fixtures. `worker` defaults to 0
+  # (serial). SQLite copies a file per slot. Postgres and MySQL use
+  # `<base>-mutineer-<w>`, created once before the worker daemons start.
   #
   # Verdict mapping: child exit 0=survived (suite passed), 1=killed (suite
   # failed), 2=error (child raised AROUND the test: load, boot, or worker-DB
@@ -47,6 +52,26 @@ module Mutineer
   module DaemonServer
     # Poll interval (seconds) for the per-fork deadline wait loop.
     POLL = 0.02
+    # Basename glob for a mutant file written beside a source file.
+    # Tempfile puts hyphens in the name, so it is not a constant name. The
+    # leading dot hides a new file from Zeitwerk (#253); a file an older
+    # version left behind has none, so the glob matches both.
+    DAEMON_TEMP_BASENAME = "{.,}mutineer_daemon*.rb"
+
+    # Skips daemon mutant files when Zeitwerk scans an autoload directory.
+    # The boot sweep keeps a file another process still holds. Zeitwerk skips a
+    # hidden one (#253), but one an older version wrote has no dot, and it would
+    # stop the next boot, because its name is not a constant.
+    module DaemonTempIgnore
+      # @return [void]
+      def setup
+        dirs = Array(Mutineer::DaemonServer.instance_variable_get(:@source_dirs))
+        dirs.each do |dir|
+          ignore(File.join(dir, Mutineer::DaemonServer::DAEMON_TEMP_BASENAME))
+        end
+        super
+      end
+    end
 
     class << self
       # Serve the protocol on the given IO pair (defaults to stdio). Returns on quit.
@@ -66,7 +91,7 @@ module Mutineer
         return if boot_line.nil? # client vanished before boot
 
         boot!(JSON.parse(boot_line.strip))
-        output.puts(JSON.generate("ready" => true, "ruby" => RUBY_VERSION))
+        output.puts(JSON.generate("ready" => true, "ruby" => RUBY_VERSION, "database_count" => database_count))
         output.flush
 
         input.each_line do |line|
@@ -84,6 +109,14 @@ module Mutineer
           end
           break if req["cmd"] == "quit"
 
+          # Create worker databases before coverage forks or mutant forks.
+          # The first daemon does this. A restarted worker daemon does not.
+          if req["cmd"] == "provision"
+            output.puts(JSON.generate(provision_slots(req["slots"])))
+            output.flush
+            next
+          end
+
           # Build the coverage map app-side and ship it to the tool, which then
           # selects covering tests per mutant. One-shot control message.
           if req["cmd"] == "coverage"
@@ -129,11 +162,17 @@ module Mutineer
           require "coverage"
           Coverage.start(lines: true, methods: true)
         end
-        # Clear any mutant tempfile a prior SIGKILLed timeout child orphaned in a
-        # source dir BEFORE the app boots. Zeitwerk would otherwise choke on the
-        # tempfile's non-constant name during autoload setup.
+        # Clear a mutant tempfile a prior SIGKILLed child orphaned in a source
+        # dir BEFORE the app boots. Zeitwerk skips a hidden one, but it would
+        # choke on the non-constant name of one an older version wrote without
+        # the dot (#253). A file another
+        # live process still owns stays: this runs before the database lock,
+        # so a refused boot must not delete that run's mutant. The kept file
+        # is hidden from Zeitwerk instead.
         sweep_temps
+        install_zeitwerk_ignore! if cfg["rails"]
         require File.expand_path(cfg["boot"]) if cfg["boot"]
+        @zeitwerk_trace&.disable
         # The --require files, after the boot as in-process (Runner.execute), so
         # the coverage peek sees what they ran at load.
         Array(cfg["require_paths"]).each { |f| require File.expand_path(f) }
@@ -141,7 +180,7 @@ module Mutineer
       rescue Exception => e # rubocop:disable Lint/RescueException
         # Boot failed (bad boot path, app error). Tell the client and exit so it
         # can surface a clean error rather than hang on the handshake.
-        @output.puts(JSON.generate("ready" => false, "error" => "#{e.class}: #{e.message}"))
+        @output.puts(JSON.generate("ready" => false, "error" => describe_error(e)))
         @output.flush
         exit!(1)
       end
@@ -149,20 +188,59 @@ module Mutineer
       # Load the per-worker DB adapter app-side (sibling gem file, by relative path
       # so it bypasses the app bundle, like this daemon itself). No-op unless the
       # app has ActiveRecord. Records the adapter + schema path so each fork can
-      # route to its own database. SQLite-only this pass; a non-SQLite config
-      # raises in the fork and reads as `error`, never a mis-routed verdict.
+      # route to its own database.
+      #
+      # `db_role` "prepare" waits for a provision command (the first daemon).
+      # "worker" only takes the shared lock (a worker daemon, including a
+      # respawn). Anything else provisions now, which is what a direct client
+      # that never sends the command needs. SQLite provisioning does nothing.
       def setup_worker_db(cfg)
         require_relative "rails_worker_db"
         @worker_db = RailsWorkerDb.available? ? RailsWorkerDb : nil
         schema = cfg["schema"] && File.expand_path(cfg["schema"])
         @schema_path = schema if schema && File.exist?(schema)
-        # Each worker slot is seeded once, on first use (not every mutant fork).
+        # Each SQLite slot is seeded once, on first use (not every mutant fork).
+        # Postgres and MySQL slots are copied at provision time, so this flag is unused there.
         @slot_ready = {}
+        return unless @worker_db
+
+        case cfg["db_role"]
+        when "prepare"
+          nil
+        when "worker"
+          @worker_db.prepare_worker!
+        else
+          @worker_db.provision(cfg.fetch("slots", 2).to_i)
+        end
       rescue LoadError => e
-        @errio.puts("[daemon] worker-DB routing unavailable: #{e.message}")
+        @errio.puts("[daemon] worker-DB routing unavailable: #{redact(e.message)}")
         @worker_db = nil
       end
 
+      # Database configs in the booted test environment. 1 when the app has no
+      # worker-DB adapter. The tool uses this to force a single worker.
+      #
+      # @return [Integer]
+      def database_count
+        return 1 unless @worker_db
+
+        @worker_db.database_count
+      rescue StandardError
+        1
+      end
+
+      # Run {RailsWorkerDb.provision} and report success or the cause. A failure
+      # stays on this reply. It does not become a mutant verdict.
+      #
+      # @param slots [Integer, String]
+      # @return [Hash]
+      def provision_slots(slots)
+        @worker_db.provision(Integer(slots)) if @worker_db
+        { "ok" => true }
+      rescue Exception => e # rubocop:disable Lint/RescueException
+        { "ok" => false, "error" => describe_error(e) }
+      end
+
       # Build the coverage map app-side (Coverage was started at boot) and return
       # it as `{map, failed_test_files, load_lines, load_methods}` for the tool to select covering tests.
       # Capture forks route to worker 0's DB (isolated, serial). On any failure
@@ -185,8 +263,26 @@ module Mutineer
           "failed_clean_tests" => cmap.failed_clean_tests, "load_lines" => cmap.load_lines.to_a,
           "load_methods" => cmap.load_methods.to_a }
       rescue Exception => e # rubocop:disable Lint/RescueException
-        @errio.puts("[daemon] coverage build failed: #{e.class}: #{e.message}")
-        { "map" => {}, "failed_test_files" => [], "error" => "#{e.class}: #{e.message}" }
+        error = describe_error(e)
+        @errio.puts("[daemon] coverage build failed: #{error}")
+        { "map" => {}, "failed_test_files" => [], "error" => error }
+      end
+
+      # `Class: message` for an error the daemon reports, with credentials hidden.
+      # Rails parses DATABASE_URL itself, so its URI error can carry the password.
+      #
+      # @param error [Exception]
+      # @return [String]
+      def describe_error(error)
+        redact("#{error.class}: #{error.message}")
+      end
+
+      # Hide database credentials in text bound for the client or stderr.
+      #
+      # @param text [String]
+      # @return [String]
+      def redact(text)
+        Redaction.text(text)
       end
 
       # Fork-safety hook for coverage capture. Each capture fork drops its copy
@@ -225,6 +321,7 @@ module Mutineer
           code =
             begin
               ChildStdout.silence
+              ChildCoverage.allow_start
               close_protocol
               # Route THIS fork at its own worker database before any test loads.
               # A routing failure raises here and is scored `error`, never a false verdict.
@@ -232,7 +329,7 @@ module Mutineer
               apply_payload(req["payload"])
               run_tests(Array(req["tests"]))
             rescue Exception => e # rubocop:disable Lint/RescueException
-              @errio.puts("[daemon-child] #{e.class}: #{e.message}")
+              @errio.puts("[daemon-child] #{describe_error(e)}")
               2
             end
           exit!(code)
@@ -243,17 +340,46 @@ module Mutineer
         # failure. Both leave the slot unready so the next fork seeds again.
         @slot_ready[worker] = true if first_use && %w[killed survived].include?(verdict)
         # A SIGKILLed timeout child skipped its Tempfile unlink. Sweep the orphan
-        # so it cannot outlive the run or trip Zeitwerk on a later fork.
+        # so it cannot outlive the run.
         sweep_temps if verdict == "timeout"
         { "id" => req["id"], "verdict" => verdict }
       end
 
+      # Hide kept mutant files from Zeitwerk before the app builds its loaders.
+      # The loader class may not exist yet. Watch for it, then stop watching.
+      #
+      # @return [void]
+      def install_zeitwerk_ignore!
+        if defined?(Zeitwerk::Loader)
+          prepend_daemon_temp_ignore
+          return
+        end
+
+        @zeitwerk_trace = TracePoint.new(:end) do |event|
+          next unless event.self.is_a?(Class) && event.self.name == "Zeitwerk::Loader"
+
+          @zeitwerk_trace.disable
+          prepend_daemon_temp_ignore
+        end
+        @zeitwerk_trace.enable
+      end
+
+      # @return [void]
+      def prepend_daemon_temp_ignore
+        return if Zeitwerk::Loader.ancestors.include?(DaemonTempIgnore)
+
+        Zeitwerk::Loader.prepend(DaemonTempIgnore)
+      end
+
       # Remove orphaned mutant tempfiles from the source dirs (parent-side; the
       # SIGKILL path cannot run the child's ensure). Mirrors JobPlan.sweep_orphans.
+      # Skips a file another process still owns ({OrphanGuard.mutant_file_in_use?}).
       def sweep_temps
         @source_dirs.to_a.each do |dir|
-          Dir.glob(File.join(dir, "mutineer_daemon*.rb")).each do |f|
-            File.unlink(f) rescue nil # rubocop:disable Style/RescueModifier
+          Dir.glob(File.join(dir, DAEMON_TEMP_BASENAME)).each do |path|
+            next if OrphanGuard.mutant_file_in_use?(path)
+
+            File.unlink(path) rescue nil # rubocop:disable Style/RescueModifier
           end
         end
       end
@@ -300,15 +426,21 @@ module Mutineer
       # `require_relative` in the mutated source resolves against its real
       # neighbours. Writing it to the tmpdir would LoadError on such files and
       # score a spurious `error` that diverges from the in-process path. The
-      # Zeitwerk hazard (a stray `.rb` in an autoload dir) is handled by the
-      # boot/timeout `sweep_temps`, not by relocating the file. Same path for
-      # reload (whole file) and redefine (wrapped snippet).
+      # leading dot keeps the file away from Zeitwerk, which skips hidden files:
+      # loading it can be the first time Zeitwerk scans a namespace directory,
+      # and a parallel worker's file may sit there too (#253). `sweep_temps` only
+      # removes orphans and never touches a live file. Same path for reload
+      # (whole file) and redefine (wrapped snippet). The backend sends the whole
+      # file, so the concern blocks it stores replace the forgotten ones (#256).
       def apply_payload(payload)
         dir = File.dirname(File.expand_path(payload.fetch("source_file")))
-        Tempfile.create(["mutineer_daemon", ".rb"], dir) do |f|
+        Tempfile.create([".mutineer_daemon", ".rb"], dir) do |f|
+          # Held until this block ends, including SIGKILL (the kernel drops it).
+          # Another run's sweep sees the lock and does not delete the file.
+          f.flock(File::LOCK_EX) rescue nil # rubocop:disable Style/RescueModifier
           f.write(payload.fetch("code"))
           f.flush
-          load f.path
+          ConcernBlocks.replace(payload.fetch("source_file")) { load f.path }
         end
       end
 

data/lib/mutineer/external_backend.rb

@@ -3,6 +3,7 @@
 require "shellwords"
 require "tempfile"
 require_relative "result"
+require_relative "rails_run_env"
 
 module Mutineer
   # Raised when the smoke check (the unmutated suite) is not green, so the run
@@ -25,6 +26,7 @@ module Mutineer
   # Child environment: bundler/gem env injected by Mutineer's Ruby is stripped, and
   # version-manager concrete version bins (e.g. `~/.rbenv/versions/3.4.9/bin`) are
   # removed from PATH so shims / `.ruby-version` can select the app's Ruby (#32).
+  # PARALLEL_WORKERS and DISABLE_SPRING are set to 1 ({RailsRunEnv}).
   module ExternalBackend
     # Generous ceiling for the one-off smoke/calibration run (a cold app boot plus
     # the full suite). The per-mutant timeout is derived from how long this took.
@@ -32,6 +34,9 @@ module Mutineer
     # Poll interval for the deadline wait loop. Independent of Isolation's loop —
     # this backend waits on an external process TREE, not an in-process fork.
     POLL = 0.02
+    # Exit codes a shell or wrapper uses when the command could not run at all
+    # (126: not executable, 127: not found). Scored `error`, never `killed`.
+    CANNOT_RUN_CODES = [126, 127].freeze
 
     # PATH entries that pin a concrete Ruby under a version manager (ahead of
     # shims). Optional trailing slash; normal bins are left alone.
@@ -67,7 +72,8 @@ module Mutineer
     # onto the parent env: set a key to +nil+ to unset it. Scrubs Mutineer/bundler
     # injection and version-manager PATH pins so shims / `.ruby-version` can win.
     # Keeps other vars (e.g. `RAILS_ENV`) so setting them on the Mutineer command
-    # still reaches the suite.
+    # still reaches the suite. Also sets +PARALLEL_WORKERS+ to 1 and
+    # +DISABLE_SPRING+ to 1. A wrapper that sets either variable again wins.
     #
     # @api private
     # @return [Hash{String => String, nil}] env for Process.spawn.
@@ -90,6 +96,7 @@ module Mutineer
                kept
              end
       env["PATH"] = path.join(File::PATH_SEPARATOR)
+      RailsRunEnv.pin_child!(env, spring: true)
       env
     end
 
@@ -145,6 +152,10 @@ module Mutineer
         end
 
         warn output if verbose && !output.empty?
+        # 126 (cannot execute) and 127 (command not found) come from a wrapper
+        # such as `bundle exec` that could not start the suite: no test ran.
+        return Result.error("test-command could not start (exit #{code})") if CANNOT_RUN_CODES.include?(code)
+
         Result.killed
       end
     end

data/lib/mutineer/init.rb

@@ -0,0 +1,124 @@
+# frozen_string_literal: true
+
+require "optparse"
+require_relative "config"
+require_relative "mutator_registry"
+
+module Mutineer
+  # Writes a starter `.mutineer.yml` and prints the first run command.
+  # It does not replace an existing file unless `--force` is set.
+  module Init
+    # Usage line for a bad invocation.
+    USAGE = "Usage: mutineer init [--rails] [--force]"
+
+    # Writes the config and prints the first command.
+    # 0 is success. 2 is a usage error, a refused overwrite, or a missing directory.
+    #
+    # @param argv [Array<String>] arguments after `init`.
+    # @param out [IO] where the command goes.
+    # @param err [IO] where errors go.
+    # @param dir [String] directory that receives `.mutineer.yml`.
+    # @return [Integer] 0 or 2.
+    def self.run(argv, out: $stdout, err: $stderr, dir: Dir.pwd)
+      rails = false
+      force = false
+      help = false
+      parser = OptionParser.new do |o|
+        o.banner = USAGE
+        o.on("--rails") { rails = true }
+        o.on("--force") { force = true }
+        o.on("--help") { help = true }
+      end
+      begin
+        parser.parse!(argv)
+      rescue OptionParser::InvalidOption, OptionParser::MissingArgument => e
+        err.puts "mutineer: #{e.message}"
+        return 2
+      end
+      if help
+        out.puts USAGE
+        return 0
+      end
+      unless argv.empty?
+        err.puts "mutineer: init takes no source paths"
+        return 2
+      end
+
+      path = File.join(dir, CONFIG_FILE)
+      if File.directory?(path)
+        err.puts "mutineer: #{CONFIG_FILE} is a directory"
+        return 2
+      end
+      if File.exist?(path) && !force
+        err.puts "mutineer: #{CONFIG_FILE} already exists. Pass --force to replace it."
+        return 2
+      end
+      # The command writes one file. It does not create the project directory.
+      unless File.directory?(dir)
+        err.puts "mutineer: cannot write #{CONFIG_FILE}: no such directory"
+        return 2
+      end
+
+      begin
+        File.write(path, template(rails: rails))
+      rescue SystemCallError => e
+        err.puts "mutineer: cannot write #{CONFIG_FILE}: #{e.message}"
+        return 2
+      end
+      command = command_for(dir, rails: rails)
+      if command
+        out.puts command
+      else
+        err.puts "mutineer: no source folder found. Create app/models or lib, then run mutineer on that folder."
+      end
+      0
+    end
+
+    # Starter config. `since` stays commented so the first run is a full scan.
+    # `--rails` also sets `rails: true` and leaves `daemon` commented.
+    #
+    # @param rails [Boolean] when true, set `rails: true`.
+    # @return [String] YAML text.
+    def self.template(rails:)
+      operators = MutatorRegistry::DEFAULT_NAMES.map { |name| "  - #{name}" }.join("\n")
+      head = <<~YAML
+        # Mutineer config. Command-line flags override this file.
+        # These operators are the default set.
+        operators:
+      YAML
+      tail = <<~YAML
+        # since: origin/main
+        # Leave this commented so the first run is a full scan.
+      YAML
+      body = "#{head}#{operators}\n\n#{tail}"
+      return body unless rails
+
+      extra = <<~YAML
+        rails: true
+
+        # daemon: true
+        # Leave this commented until the support matrix says the daemon fits.
+        # The matrix is on https://davidteren.github.io/mutineer/rails.html#daemon
+      YAML
+      "#{body.rstrip}\n\n#{extra}"
+    end
+
+    # The first run command. Plain init names `lib`.
+    # `--rails` names each existing folder among `app/models`, `app/services`, and `lib`.
+    # A missing folder is left out: `mutineer run` rejects a path that is not there.
+    #
+    # @param dir [String] project directory.
+    # @param rails [Boolean] when true, print the Rails source folders.
+    # @return [String, nil] the command, or nil when no Rails source folder exists.
+    def self.command_for(dir, rails:)
+      return "mutineer run lib" unless rails
+
+      folders = ["app/models", "app/services", "lib"].select do |folder|
+        File.directory?(File.join(dir, folder))
+      end
+      return if folders.empty?
+
+      "mutineer run #{folders.join(" ")}"
+    end
+  end
+end

data/lib/mutineer/isolation.rb

@@ -4,7 +4,9 @@ require "tempfile"
 require_relative "result"
 require_relative "parser"
 require_relative "child_stdout"
+require_relative "child_coverage"
 require_relative "kill_channel"
+require_relative "concern_blocks"
 
 module Mutineer
   # Fork-based isolation for running one mutant. The block runs in a child
@@ -56,6 +58,7 @@ module Mutineer
         code = 0
         begin
           ChildStdout.silence
+          ChildCoverage.allow_start
           # A block that takes no channel (a zero-arity lambda) still runs.
           result = channel ? yield(wr) : yield
           code = result.is_a?(Integer) ? result : 0
@@ -179,15 +182,24 @@ module Mutineer
     # "base"`). Writing it elsewhere makes those requires resolve to the temp
     # dir and raise LoadError.
     #
+    # Loading the file reopens its namespace. With lazy loading, that can be the
+    # first time Zeitwerk scans the namespace's directory. Zeitwerk skips hidden
+    # files, so the leading dot keeps it from rejecting the tempfile's name as a
+    # constant, in this child and in any other running at the same time (#253).
+    #
+    # A concern's stored `included` and `prepended` blocks from the source file
+    # are forgotten around the load, so ActiveSupport accepts the blocks the
+    # temp file stores (#256).
+    #
     # @api private
     # @param mutated [String] mutated source text.
     # @param source_file [String] original source file path.
     # @return [Object] whatever `load` returns.
     def self.apply_whole_file(mutated, source_file)
-      Tempfile.create(["mutineer_mutant", ".rb"], File.dirname(File.expand_path(source_file))) do |f|
+      Tempfile.create([".mutineer_mutant", ".rb"], File.dirname(File.expand_path(source_file))) do |f|
         f.write(mutated)
         f.flush
-        load f.path
+        ConcernBlocks.replace(source_file) { load f.path }
       end
     end