saboteur 0.3.0 → 0.5.0

data/lib/saboteur/frameworks/rails.rb

@@ -1,11 +1,15 @@
 # SPDX-License-Identifier: MIT
 # frozen_string_literal: true
 
+require "coverage"
 require "fileutils"
+require "pathname"
+require "English"
+require "tmpdir"
 
 module Saboteur
   module Frameworks
-    # Opt-in Rails support (`rails: true` in .saboteur.yml).
+    # Opt-in Rails support (`c.rails = true` in config/saboteur.rb).
     #
     # Refuses to run unless RAILS_ENV is test and every configured database is
     # local, eager loads the app so every subject's owner is defined before
@@ -14,9 +18,23 @@ module Saboteur
     class Rails
       LOCAL_HOSTS = ["localhost", "127.0.0.1", "::1", "[::1]"].freeze
 
+      # How many blocking session names the template-copy error lists.
+      MAX_BLOCKER_NAMES = 5
+
+      # sqlite3-ruby's Backup#step returns raw SQLite status codes instead
+      # of raising, and finish discards its own: without an explicit check
+      # a partial destination would pass as a copy.
+      SQLITE_DONE = 101
+      SQLITE_BUSY = 5
+      SQLITE_LOCKED = 6
+      BACKUP_ATTEMPTS = 5
+      BACKUP_RETRY_DELAY = 0.05
+
       def initialize(config, hooks)
         @config = config
         @hooks = hooks
+        @worker_copies = []
+        @memory_snapshots = {}
       end
 
       def before_boot
@@ -27,18 +45,31 @@ module Saboteur
         # on macOS that goes through XPC, which crashes a forked process. The
         # databases are local only (check_databases), so it buys nothing.
         ENV["PGGSSENCMODE"] ||= "disable"
+
+        # Bytecode from Bootsnap's compile cache carries no coverage counters,
+        # so a file served from the cache never appears in Coverage.result.
+        # Turn the cache off for this process before config/boot loads
+        # Bootsnap; the load-path cache still works. Bootsnap checks
+        # DISABLE_<feature> by presence, so a value the user set disables the
+        # cache too. Files loaded during the eager load are also covered by
+        # eager_load_with_coverage_running.
+        ENV["DISABLE_BOOTSNAP_COMPILE_CACHE"] ||= "1"
       end
 
       def after_boot
         unless defined?(::Rails.env)
           raise Error,
-                "rails: true but Rails did not load (add config/environment to require)"
+                "rails: true but Rails did not load (add config/environment to require - for an engine, " \
+                "its dummy app's environment, such as test/dummy/config/environment.rb; only a dynamically " \
+                "created dummy app has no environment file, and there you require the helper that calls " \
+                "setup instead, such as DummyApp.setup in spec/rails_helper)"
         end
         raise Error, "rails integration refuses to run: Rails.env is #{::Rails.env}" unless ::Rails.env.test?
 
         check_databases
-        ::Rails.application.eager_load!
+        eager_load_with_coverage_running
         disconnect
+        snapshot_memory_databases
         @hooks.register(:before_fork) { |_mutation| disconnect }
         @hooks.register(:after_fork) { reconnect }
       end
@@ -54,16 +85,58 @@ module Saboteur
       end
 
       # Parent, before workers start: one copy of each test database per
-      # extra worker, made from the prepared test database.
+      # extra worker, made from the prepared test database. Every attempted
+      # copy is tracked before it runs, so cleanup_workers drops them all,
+      # including the partial set when a copy failed.
       def prepare_workers(jobs)
         return unless defined?(::ActiveRecord::Base) && jobs > 1
 
+        @worker_copies = []
         test_configs.each do |db|
-          (1...jobs).each { |number| copy_database(db, self.class.worker_database(db.database, number)) }
+          # An in-memory SQLite database lives in the parent's connection;
+          # workers inherit it through fork, so there is nothing to copy.
+          next if self.class.memory_database?(db)
+
+          (1...jobs).each do |number|
+            name = self.class.worker_database(db.database, number)
+            # Track the target before the copy, not after it: an attempt
+            # that created the database and then failed (a post-CREATE
+            # error, a partial SQLite copy) must still be dropped at the
+            # end of the run. Dropping is idempotent (DROP IF EXISTS,
+            # rm -f), so tracking a copy that never happened costs nothing.
+            @worker_copies << [db, name]
+            copy_database(db, name)
+          end
         end
         disconnect
       end
 
+      # Parent, after the run: drop every copy prepare_workers made, so runs
+      # do not collect one database per worker. Every copy is attempted even
+      # when a drop fails; the failures are reported together at the end.
+      def cleanup_workers
+        return if @worker_copies.empty? && @memory_snapshot_dir.nil?
+
+        failures = @worker_copies.filter_map do |db, name|
+          drop_database(db, name)
+          nil
+        rescue StandardError => e
+          "#{name}: #{e.message}"
+        end
+        @worker_copies = []
+        if @memory_snapshot_dir
+          FileUtils.rm_rf(@memory_snapshot_dir)
+          @memory_snapshot_dir = nil
+          @memory_snapshots = {}
+        end
+        begin
+          disconnect
+        rescue StandardError => e
+          failures << "disconnect: #{e.message}"
+        end
+        raise Error, "could not drop worker database copies: #{failures.join('; ')}" if failures.any?
+      end
+
       # Worker: point every connection pool (primary, and any cache, queue or
       # connects_to databases, for every role and shard) at this worker's
       # copy of its database.
@@ -74,11 +147,16 @@ module Saboteur
         handler.connection_pool_list(:all).to_a.each do |pool|
           config = pool.pool_config
           db = pool.db_config
+          # An in-memory SQLite database would be discarded by
+          # re-establishing: the worker keeps its own connection.
+          next if self.class.memory_database?(db)
+
           handler.establish_connection(db.configuration_hash.merge(database: self.class.worker_database(db.database,
                                                                                                         number)),
                                        owner_name: self.class.owner_of(config), role: config.role,
                                        shard: config.shard)
         end
+        restore_memory_databases
       end
 
       def self.owner_of(pool_config)
@@ -87,12 +165,35 @@ module Saboteur
         Object.const_get(pool_config.connection_descriptor.name)
       end
 
+      # Only SQLite can be in-memory; every other adapter names a server.
+      def self.memory_database?(db)
+        db.adapter.to_s.match?(/sqlite/) && db.database.to_s == ":memory:"
+      end
+
       def self.local_host?(host)
         host.nil? || host.to_s.empty? || host.to_s.start_with?("/") || LOCAL_HOSTS.include?(host.to_s.downcase)
       end
 
       private
 
+      # Bootsnap before 1.25 skips its bytecode cache only while Coverage is
+      # running (1.25 and later skip it whenever Coverage is set up, suspended
+      # or not), and Coverage is suspended while the app boots (exe/saboteur
+      # sets it up suspended). Cached bytecode carries no coverage counters.
+      # Resume Coverage just for the eager load, so every app file compiles
+      # with counters. The coverage recorder clears the counts before the
+      # suite runs, so load-time hits are never credited to a test.
+      def eager_load_with_coverage_running
+        return ::Rails.application.eager_load! unless Coverage.state == :suspended
+
+        Coverage.resume
+        begin
+          ::Rails.application.eager_load!
+        ensure
+          Coverage.suspend
+        end
+      end
+
       def check_databases
         return unless defined?(::ActiveRecord::Base)
 
@@ -118,29 +219,157 @@ module Saboteur
           disconnect
           copy_postgres(db, name)
         when /sqlite/
-          # A database in WAL mode keeps recent writes in -wal/-shm files.
-          ["", "-wal", "-shm"].each do |suffix|
-            FileUtils.cp("#{db.database}#{suffix}", "#{name}#{suffix}") if File.exist?("#{db.database}#{suffix}")
-          end
+          copy_sqlite(db, name)
         else
           raise Error, "--jobs > 1 with Rails needs Postgres or SQLite (got #{db.adapter})"
         end
       end
 
+      # The mirror of copy_database. Adapters copy_database rejects never get
+      # copies, so there is nothing to drop for them.
+      def drop_database(db, name)
+        case db.adapter
+        when /postg/
+          disconnect
+          drop_postgres(db, name)
+        when /sqlite/
+          target = sqlite_absolute(name)
+          ["", "-wal", "-shm"].each { |suffix| FileUtils.rm_f("#{target}#{suffix}") }
+        end
+      end
+
+      # A relative sqlite path means something only to Rails: 7.1+ resolves
+      # it against Rails.root, not the process working directory. Resolve
+      # the source the same way, checkpoint so a live WAL database holds
+      # every committed write in its main file, then snapshot it through
+      # the backup API. A missing source is an error, not a skip: otherwise
+      # the worker starts from a fresh empty database and silently tests
+      # nothing.
+      def copy_sqlite(db, name)
+        source = sqlite_absolute(db.database)
+        unless File.exist?(source)
+          raise Error, "test database #{source} does not exist; run the test suite once to create it"
+        end
+
+        checkpoint_sqlite(source)
+        backup_sqlite(source, sqlite_absolute(name))
+      end
+
+      def sqlite_absolute(database)
+        return database if Pathname.new(database).absolute?
+        return database unless defined?(::Rails)
+
+        File.expand_path(database, ::Rails.root)
+      end
+
+      # wal_checkpoint(TRUNCATE) reports instead of raising: [busy, frames
+      # in the WAL, frames checkpointed]. A busy reader or an incomplete
+      # pass leaves committed writes outside the main file, and copying on
+      # would silently miss them - fail closed instead. Through a
+      # connection of saboteur's own, so the app's connections never point
+      # at the database being checkpointed.
+      def checkpoint_sqlite(source)
+        with_sqlite(source) do |database|
+          busy, frames, checkpointed = database.execute("PRAGMA wal_checkpoint(TRUNCATE)").first
+          return if busy.zero? && frames == checkpointed
+
+          raise Error, "could not checkpoint #{source} before copying it " \
+                       "(busy: #{busy}, frames: #{frames}, checkpointed: #{checkpointed}); " \
+                       "close the connections holding it and run again"
+        end
+      rescue SQLite3::BusyException => e
+        raise Error, "could not checkpoint #{source} before copying it (#{e.message}); " \
+                     "close the connections holding it and run again"
+      end
+
+      # A byte copy of main + WAL + SHM is no snapshot: a write landing
+      # between the file copies leaves the set malformed. The backup API
+      # copies a consistent image and restarts when the source changes
+      # under it. The target starts clean so stale sidecars from an
+      # earlier byte copy cannot shadow the fresh image.
+      def backup_sqlite(source, target)
+        ["", "-wal", "-shm"].each { |suffix| FileUtils.rm_f("#{target}#{suffix}") }
+        with_sqlite(source, readonly: true) do |src|
+          with_sqlite(target) do |dst|
+            backup = SQLite3::Backup.new(dst, "main", src, "main")
+            begin
+              step_backup(backup, target)
+            ensure
+              backup.finish
+            end
+          end
+        end
+      end
+
+      # BUSY and LOCKED earn a bounded retry; DONE means the target holds
+      # every page; anything else fails the copy.
+      def step_backup(backup, target)
+        BACKUP_ATTEMPTS.times do
+          case (status = backup.step(-1))
+          when SQLITE_DONE then return
+          when SQLITE_BUSY, SQLITE_LOCKED
+            sleep(BACKUP_RETRY_DELAY)
+          else
+            raise Error, "could not copy the database to #{target}: sqlite backup failed with status #{status}"
+          end
+        end
+        raise Error, "could not copy the database to #{target}: still busy after #{BACKUP_ATTEMPTS} attempts"
+      end
+
+      def with_sqlite(path, **)
+        require "sqlite3"
+
+        database = SQLite3::Database.new(path, **)
+        begin
+          yield database
+        ensure
+          database.close
+        end
+      end
+
       # Through a connection class of saboteur's own, so the app's
       # connections never point at the maintenance database.
       def copy_postgres(db, name)
-        admin = maintenance_record
-        admin.establish_connection(db.configuration_hash.merge(database: "postgres"))
-        admin.connection_pool.with_connection do |connection|
+        with_maintenance_connection(db) do |connection|
           connection.execute("DROP DATABASE IF EXISTS #{connection.quote_table_name(name)}")
+          refuse_busy_template(connection, db.database)
           connection.execute("CREATE DATABASE #{connection.quote_table_name(name)} " \
                              "TEMPLATE #{connection.quote_table_name(db.database)}")
         end
+      end
+
+      def drop_postgres(db, name)
+        with_maintenance_connection(db) do |connection|
+          connection.execute("DROP DATABASE IF EXISTS #{connection.quote_table_name(name)}")
+        end
+      end
+
+      # Through a connection class of saboteur's own, so the app's
+      # connections never point at the maintenance database.
+      def with_maintenance_connection(db, &)
+        admin = maintenance_record
+        admin.establish_connection(db.configuration_hash.merge(database: "postgres"))
+        admin.connection_pool.with_connection(&)
       ensure
         admin&.remove_connection
       end
 
+      # Postgres refuses CREATE DATABASE ... TEMPLATE while any other session
+      # is connected to the template, so name the blockers instead of dying
+      # mid-run with the raw error. Never terminates other sessions.
+      def refuse_busy_template(connection, database)
+        blockers = connection.select_values(<<~SQL)
+          SELECT application_name FROM pg_stat_activity
+          WHERE datname = #{connection.quote(database)} AND pid <> pg_backend_pid()
+        SQL
+        return if blockers.empty?
+
+        names = blockers.map { |app| app.to_s.empty? ? "unnamed" : app }.first(MAX_BLOCKER_NAMES)
+        raise Error, "cannot copy the test database #{database}: #{blockers.size} other session(s) connected " \
+                     "(#{names.join(', ')})#{', ...' if blockers.size > names.size}. " \
+                     "Stop other test runs, consoles or Spring, then run saboteur again."
+      end
+
       # ActiveRecord names a connection's owner by class name, so the class
       # needs a constant.
       def maintenance_record
@@ -149,10 +378,15 @@ module Saboteur
         self.class.const_set(:MaintenanceRecord, Class.new(::ActiveRecord::Base) { self.abstract_class = true })
       end
 
+      # Disconnecting an in-memory SQLite pool destroys the database, so
+      # those pools stay connected everywhere saboteur disconnects: the
+      # schema loaded at boot survives, and workers inherit it via fork.
       def disconnect
         return unless defined?(::ActiveRecord::Base)
 
-        ::ActiveRecord::Base.connection_handler.connection_pool_list(:all).each(&:disconnect!)
+        ::ActiveRecord::Base.connection_handler.connection_pool_list(:all).each do |pool|
+          pool.disconnect! unless self.class.memory_database?(pool.db_config)
+        end
       end
 
       # The parent disconnected before fork, so inherited pools hold no sockets.
@@ -160,6 +394,88 @@ module Saboteur
       # checks out a fresh connection of its own on first use.
       def reconnect
         disconnect
+        restore_memory_databases
+      end
+
+      # Parent, once the app booted: an in-memory SQLite database survives
+      # only inside its own connection, and a forked child cannot keep it -
+      # ActiveRecord discards inherited pools and the sqlite3 gem closes
+      # writable handles in the child, so the next access there opens an
+      # empty database. Snapshot each in-memory database to a file the
+      # children can restore from.
+      def snapshot_memory_databases
+        return unless defined?(::ActiveRecord::Base)
+
+        pools = ::ActiveRecord::Base.connection_handler.connection_pool_list(:all)
+        memory_pools = pools.select { |pool| self.class.memory_database?(pool.db_config) }
+        return if memory_pools.empty?
+
+        @memory_snapshot_dir = Dir.mktmpdir("saboteur-memory-")
+        own_memory_snapshots
+        memory_pools.each_with_index do |pool, index|
+          path = File.join(@memory_snapshot_dir, "#{index}.sqlite3")
+          pool.with_connection do |connection|
+            file = SQLite3::Database.new(path)
+            with_backup(file, connection.raw_connection, close: file) { |backup| step_backup(backup, path) }
+          end
+          @memory_snapshots[pool.db_config] = path
+        end
+      end
+
+      # The safety net for an abrupt process exit: Engine.call removes the
+      # snapshots at the run boundary on every normal path, so this handler
+      # fires only when the owner process dies first. Forked children
+      # inherit the handler but not the pid.
+      def own_memory_snapshots
+        owner = Process.pid
+        at_exit { cleanup_memory_snapshots(owner) }
+      end
+
+      def cleanup_memory_snapshots(owner)
+        FileUtils.rm_rf(@memory_snapshot_dir) if Process.pid == owner && @memory_snapshot_dir
+      end
+
+      # Yields a backup and always returns both handles: exactly one finish
+      # per init (skipped only when backup_init itself raised, leaving no
+      # handle), and the database closes even when finish fails.
+      def with_backup(destination, source, close:)
+        backup = SQLite3::Backup.new(destination, "main", source, "main")
+        yield backup
+      ensure
+        begin
+          backup&.finish
+        ensure
+          close_without_masking(close)
+        end
+      end
+
+      # A close can fail too, and a close that fails on the way out must
+      # not hide the error already in flight.
+      def close_without_masking(database)
+        in_flight = $ERROR_INFO
+        database.close
+      rescue SQLite3::Exception
+        raise unless in_flight
+      end
+
+      # Child: load the parent's snapshot into this process's own fresh
+      # in-memory database. The backup overwrites the destination, so
+      # restoring more than once is harmless.
+      def restore_memory_databases
+        return unless defined?(::ActiveRecord::Base)
+        return if @memory_snapshots.empty?
+
+        ::ActiveRecord::Base.connection_handler.connection_pool_list(:all).each do |pool|
+          snapshot = @memory_snapshots[pool.db_config]
+          next unless snapshot
+
+          pool.with_connection do |connection|
+            source = SQLite3::Database.new(snapshot, readonly: true)
+            with_backup(connection.raw_connection, source, close: source) do |backup|
+              step_backup(backup, pool.db_config.database)
+            end
+          end
+        end
       end
     end
   end

data/lib/saboteur/gates.rb

@@ -0,0 +1,83 @@
+# SPDX-License-Identifier: MIT
+# frozen_string_literal: true
+
+module Saboteur
+  # CI gates on how many mutants ended as a timeout or an error. A limit is a
+  # count ("3") or a percentage of the executed mutants ("5%"); the gate fails
+  # only when the count is above it. Unset means no gate.
+  class Gates
+    Violation = Data.define(:cause, :message)
+
+    # config key => [mutant status, exit cause recorded in the JSON report, flag]
+    SETTINGS = {
+      "max_timeouts" => [:timeout, "maxTimeouts", "--max-timeouts"],
+      "max_errors" => [:error, "maxErrors", "--max-errors"]
+    }.freeze
+    LIMIT = /\A(\d+)(%?)\z/
+
+    # The normalized limit: nil, an Integer count, or a String ending in "%".
+    # A malformed value is a ConfigError that names the setting.
+    def self.limit(value, key)
+      return nil if value.nil?
+
+      match = LIMIT.match(value.to_s.strip)
+      raise ConfigError, "#{key} must be a count (3) or a percentage (5%), got #{value.inspect}" unless match
+
+      digits, suffix = match.captures
+      number = digits.to_i
+      raise ConfigError, "#{key} percentage must be at most 100" if suffix == "%" && number > 100
+
+      suffix.empty? ? number : "#{number}%"
+    end
+
+    # The --max-timeouts and --max-errors flags; each value goes to the block with its config key.
+    def self.define_options(parser)
+      SETTINGS.each do |key, (status, _cause, flag)|
+        parser.on("#{flag} N", "Exit 1 when more than N mutants end as #{status} (N or N%)") do |value|
+          yield key.to_sym, value
+        end
+      end
+    end
+
+    def initialize(config)
+      @limits = SETTINGS.keys.to_h { |key| [key, self.class.limit(config[key], key)] }
+    end
+
+    attr_reader :cause
+
+    def any?
+      @limits.values.any?
+    end
+
+    # --max-errors judges errored mutants, so a threshold run tolerates them.
+    def judges_errors?
+      !@limits.fetch("max_errors").nil?
+    end
+
+    # Prints each failed gate to err and remembers the first one's cause for the JSON report.
+    def failed?(report, err)
+      found = violations(report)
+      found.each { |violation| err.puts("saboteur: #{violation.message}") }
+      @cause = found.first&.cause
+      !@cause.nil?
+    end
+
+    def violations(report)
+      SETTINGS.filter_map do |key, (status, cause, flag)|
+        limit = @limits.fetch(key)
+        count = report.counts.fetch(status)
+        next if limit.nil? || !over?(count, limit, report.executed.size)
+
+        Violation.new(cause: cause, message: "#{count} #{status} mutants, above #{flag} #{limit}")
+      end
+    end
+
+    private
+
+    def over?(count, limit, executed)
+      return count > limit if limit.is_a?(Integer)
+
+      count * 100 > limit.to_i * executed
+    end
+  end
+end