saboteur 0.2.0

data/lib/saboteur/execution/worker_pool.rb

@@ -0,0 +1,249 @@
+# SPDX-License-Identifier: MIT
+# frozen_string_literal: true
+
+require "json"
+require_relative "process_tree"
+require_relative "supervisor"
+
+module Saboteur
+  module Execution
+    # Runs a block over items on several worker processes forked from the
+    # booted parent, and returns the block's results in item order.
+    #
+    # Workers are processes, not threads: forking a mutant child from a
+    # threaded parent is unsafe, and processes keep each worker's state
+    # (and database connection) apart. Each worker takes one item at a time
+    # from the parent, so a slow item does not hold up the rest; biggest
+    # items go first when weights are given. Results travel back as JSON,
+    # so the block must return plain data.
+    #
+    # Workers are numbered from 0; TEST_ENV_NUMBER follows the parallel test
+    # convention Rails apps use for one database per worker ("", "2", "3").
+    # A worker that dies has its item reported as an error, with the reason
+    # (what it raised, or the signal), and is replaced. When workers keep
+    # dying before finishing a single item, the pool stops with that reason.
+    class WorkerPool
+      # done: items finished; status: the Process::Status once reaped;
+      # last_words: what the worker reported before dying, if anything.
+      Worker = Struct.new(:number, :pid, :commands, :results, :current, :done, :status, :last_words)
+      START_FAILURES = 3
+
+      # on_start: called in each worker with its number before its first
+      # item (per-worker database connections, for example).
+      def initialize(jobs:, on_start: nil)
+        @jobs = jobs
+        @on_start = on_start
+        @start_failures = 0
+      end
+
+      def map(items, weights: nil, &work)
+        order = items.each_index.sort_by { |index| [-(weights ? weights[index] : 0), index] }
+        return inline(items, order, &work) if @jobs <= 1
+
+        distribute(items, order, &work)
+      end
+
+      private
+
+      def inline(items, order, &work)
+        order.each_with_object(Array.new(items.size)) { |index, results| results[index] = work.call(items[index]) }
+      end
+
+      def distribute(items, order, &work)
+        results = Array.new(items.size)
+        queue = order.dup
+        workers = []
+        [@jobs, items.size].min.times { |number| workers << start(number, items, workers, &work) }
+        workers.each { |worker| assign(worker, queue) }
+        collect(workers, queue, results, items, &work) until workers.empty?
+        results
+      ensure
+        workers&.each { |worker| stop(worker) }
+      end
+
+      # Waits up to a second for results, then checks for workers that died
+      # without closing their pipe (a stray grandchild may still hold it).
+      def collect(workers, queue, results, items, &work)
+        ready = IO.select(workers.map(&:results), nil, nil, 1)&.first || []
+        ready.each do |io|
+          worker = workers.find { |candidate| candidate.results == io }
+          receive(worker, workers, queue, results, items, &work) if worker
+        end
+        workers.reject { |worker| alive?(worker) }.each do |worker|
+          replace(worker, workers, queue, results, items, &work)
+        end
+      end
+
+      # Reads one line from a worker whose pipe is ready, then gives it its
+      # next item, retires it when none is left, or replaces it if it died.
+      def receive(worker, workers, queue, results, items, &)
+        line = worker.results.gets
+        take(worker, line, results) if line
+        return replace(worker, workers, queue, results, items, &) if line.nil? || worker.last_words
+
+        assign(worker, queue)
+        retire(worker, workers) unless worker.current
+      end
+
+      def parse(line)
+        JSON.parse(line)
+      rescue JSON::ParserError => e
+        { "error" => "unreadable result from worker: #{e.message}" }
+      end
+
+      def alive?(worker)
+        !reaped?(worker)
+      end
+
+      # Stores a line from the worker: its item's result, or why it crashed.
+      def take(worker, line, results)
+        data = parse(line)
+        return worker.last_words = data["fatal"] if data.key?("fatal")
+        return unless worker.current
+
+        results[worker.current] = data
+        worker.done += 1
+        worker.current = nil
+      end
+
+      # Reads what a dead worker left in its pipe: a last result, or why it
+      # crashed.
+      def drain(worker, results)
+        pending = worker.results.read_nonblock(1 << 20, exception: false)
+        return unless pending.is_a?(String)
+
+        pending.each_line { |line| take(worker, line, results) if line.end_with?("\n") }
+      rescue IOError
+        nil
+      end
+
+      def replace(worker, workers, queue, results, items, &)
+        drain(worker, results)
+        retire(worker, workers)
+        reason = worker.last_words || "worker #{worker.number} died#{how(worker.status)}"
+        results[worker.current] = { "error" => "#{reason}, running this item" } if worker.current
+        give_up(reason) if worker.done.zero?
+        return if queue.empty?
+
+        fresh = start(worker.number, items, workers, &)
+        workers << fresh
+        assign(fresh, queue)
+      end
+
+      # Sends the worker the next item, if any; worker.current is nil after
+      # when there was none left.
+      def assign(worker, queue)
+        worker.current = queue.shift
+        return unless worker.current
+
+        worker.commands.puts(worker.current)
+        worker.commands.flush
+      rescue Errno::EPIPE, IOError
+        queue.unshift(worker.current) # the worker died before taking it
+        worker.current = nil
+      end
+
+      def how(status)
+        return "" unless status
+        return " (signal #{Signal.signame(status.termsig)})" if status.signaled?
+
+        " (exit #{status.exitstatus})"
+      end
+
+      # A worker that never finishes an item is a broken setup (on_start, a
+      # bug in the work itself), not bad luck: stop instead of failing every
+      # item one by one.
+      def give_up(reason)
+        @start_failures += 1
+        raise Error, "workers keep failing to start: #{reason}" if @start_failures >= START_FAILURES
+      end
+
+      def retire(worker, workers)
+        workers.delete(worker)
+        stop(worker)
+      end
+
+      # siblings: workers already running. The new worker closes its copies
+      # of their pipe ends, or a sibling would never see its command pipe
+      # close (end of input) when the parent is done with it.
+      def start(number, items, siblings, &work)
+        commands_read, commands_write = IO.pipe
+        results_read, results_write = IO.pipe
+        pid = fork do
+          siblings.each { |sibling| [sibling.commands, sibling.results].each(&:close) }
+          commands_write.close
+          results_read.close
+          serve(number, items, commands_read, results_write, &work)
+        end
+        commands_read.close
+        results_write.close
+        own_group(pid)
+        Supervisor.live[pid] = ProcessTree.new(pid)
+        Worker.new(number, pid, commands_write, results_read, nil, 0)
+      end
+
+      # Runs in the worker until the parent closes its command pipe. If it
+      # crashes, its last line says why.
+      def serve(number, items, commands, results, &)
+        Process.setpgid(0, 0)
+        Supervisor.reset_in_worker!
+        Supervisor.close_in_child.push(commands, results)
+        ENV["SABOTEUR_WORKER"] = number.to_s
+        ENV["TEST_ENV_NUMBER"] = number.zero? ? "" : (number + 1).to_s
+        @on_start&.call(number)
+        while (line = commands.gets)
+          results.puts(JSON.generate(run_one(items[Integer(line)], &)))
+          results.flush
+        end
+        exit!(0)
+      rescue Exception => e # rubocop:disable Lint/RescueException
+        last_words(results, "worker #{number} crashed: #{e.class}: #{e.message} (#{e.backtrace&.first})")
+        exit!(1)
+      end
+
+      def last_words(results, message)
+        results.puts(JSON.generate("fatal" => message))
+        results.flush
+      rescue StandardError
+        nil
+      end
+
+      def run_one(item, &work)
+        work.call(item)
+      rescue StandardError => e
+        { "error" => "#{e.class}: #{e.message}" }
+      end
+
+      # Closing the command pipe ends the worker's loop; anything it left
+      # running is killed with its tree.
+      def stop(worker)
+        worker.commands.close unless worker.commands.closed?
+        tree = Supervisor.live.delete(worker.pid)
+        deadline = Process.clock_gettime(Process::CLOCK_MONOTONIC) + 5
+        sleep 0.01 until reaped?(worker) || Process.clock_gettime(Process::CLOCK_MONOTONIC) > deadline
+        tree&.kill_all
+        worker.status ||= Process.wait2(worker.pid).last unless reaped?(worker)
+        worker.results.close unless worker.results.closed?
+      rescue Errno::ECHILD
+        nil
+      end
+
+      def own_group(pid)
+        Process.setpgid(pid, pid)
+      rescue Errno::EACCES, Errno::ESRCH, Errno::EPERM
+        nil
+      end
+
+      # Reaps the worker if it has exited, keeping its status.
+      def reaped?(worker)
+        return true if worker.status
+
+        pid, status = Process.waitpid2(worker.pid, Process::WNOHANG)
+        worker.status = status if pid
+        !pid.nil?
+      rescue Errno::ECHILD
+        true
+      end
+    end
+  end
+end

data/lib/saboteur/frameworks/rails.rb

@@ -0,0 +1,166 @@
+# SPDX-License-Identifier: MIT
+# frozen_string_literal: true
+
+require "fileutils"
+
+module Saboteur
+  module Frameworks
+    # Opt-in Rails support (`rails: true` in .saboteur.yml).
+    #
+    # Refuses to run unless RAILS_ENV is test and every configured database is
+    # local, eager loads the app so every subject's owner is defined before
+    # forking, and drops database connections in the parent so each child opens
+    # its own. Prints host names only, never credentials.
+    class Rails
+      LOCAL_HOSTS = ["localhost", "127.0.0.1", "::1", "[::1]"].freeze
+
+      def initialize(config, hooks)
+        @config = config
+        @hooks = hooks
+      end
+
+      def before_boot
+        env = ENV.fetch("RAILS_ENV", nil) || ENV.fetch("RACK_ENV", nil)
+        raise Error, "rails integration refuses to run unless RAILS_ENV=test (got #{env.inspect})" unless env == "test"
+
+        # libpq probes for Kerberos (GSS encryption) on every new connection;
+        # on macOS that goes through XPC, which crashes a forked process. The
+        # databases are local only (check_databases), so it buys nothing.
+        ENV["PGGSSENCMODE"] ||= "disable"
+      end
+
+      def after_boot
+        unless defined?(::Rails.env)
+          raise Error,
+                "rails: true but Rails did not load (add config/environment to require)"
+        end
+        raise Error, "rails integration refuses to run: Rails.env is #{::Rails.env}" unless ::Rails.env.test?
+
+        check_databases
+        ::Rails.application.eager_load!
+        disconnect
+        @hooks.register(:before_fork) { |_mutation| disconnect }
+        @hooks.register(:after_fork) { reconnect }
+      end
+
+      # The database a worker uses: the first keeps the test database, the
+      # others get copies named for saboteur, never a name parallel_tests or
+      # a developer might already use.
+      def self.worker_database(database, number)
+        return database if number.zero?
+
+        extension = File.extname(database)
+        "#{database.delete_suffix(extension)}_saboteur#{number + 1}#{extension}"
+      end
+
+      # Parent, before workers start: one copy of each test database per
+      # extra worker, made from the prepared test database.
+      def prepare_workers(jobs)
+        return unless defined?(::ActiveRecord::Base) && jobs > 1
+
+        test_configs.each do |db|
+          (1...jobs).each { |number| copy_database(db, self.class.worker_database(db.database, number)) }
+        end
+        disconnect
+      end
+
+      # Worker: point every connection pool (primary, and any cache, queue or
+      # connects_to databases, for every role and shard) at this worker's
+      # copy of its database.
+      def start_worker(number)
+        return unless defined?(::ActiveRecord::Base) && number.positive?
+
+        handler = ::ActiveRecord::Base.connection_handler
+        handler.connection_pool_list(:all).to_a.each do |pool|
+          config = pool.pool_config
+          db = pool.db_config
+          handler.establish_connection(db.configuration_hash.merge(database: self.class.worker_database(db.database,
+                                                                                                        number)),
+                                       owner_name: self.class.owner_of(config), role: config.role,
+                                       shard: config.shard)
+        end
+      end
+
+      def self.owner_of(pool_config)
+        return pool_config.connection_class if pool_config.respond_to?(:connection_class)
+
+        Object.const_get(pool_config.connection_descriptor.name)
+      end
+
+      def self.local_host?(host)
+        host.nil? || host.to_s.empty? || host.to_s.start_with?("/") || LOCAL_HOSTS.include?(host.to_s.downcase)
+      end
+
+      private
+
+      def check_databases
+        return unless defined?(::ActiveRecord::Base)
+
+        remote = ::ActiveRecord::Base.configurations.configs_for(env_name: "test").filter_map do |db|
+          host = db.configuration_hash[:host]
+          "#{db.name} (host #{host})" unless self.class.local_host?(host)
+        end
+        return if remote.empty?
+
+        raise Error, "rails integration refuses to run against non-local databases: #{remote.join(', ')}"
+      end
+
+      def test_configs
+        ::ActiveRecord::Base.configurations.configs_for(env_name: "test")
+      end
+
+      # Postgres: CREATE DATABASE ... TEMPLATE (needs no open connections to
+      # the template, so it runs from the maintenance database). SQLite: a
+      # file copy. Other adapters are not supported for workers.
+      def copy_database(db, name)
+        case db.adapter
+        when /postg/
+          disconnect
+          copy_postgres(db, name)
+        when /sqlite/
+          # A database in WAL mode keeps recent writes in -wal/-shm files.
+          ["", "-wal", "-shm"].each do |suffix|
+            FileUtils.cp("#{db.database}#{suffix}", "#{name}#{suffix}") if File.exist?("#{db.database}#{suffix}")
+          end
+        else
+          raise Error, "--jobs > 1 with Rails needs Postgres or SQLite (got #{db.adapter})"
+        end
+      end
+
+      # Through a connection class of saboteur's own, so the app's
+      # connections never point at the maintenance database.
+      def copy_postgres(db, name)
+        admin = maintenance_record
+        admin.establish_connection(db.configuration_hash.merge(database: "postgres"))
+        admin.connection_pool.with_connection do |connection|
+          connection.execute("DROP DATABASE IF EXISTS #{connection.quote_table_name(name)}")
+          connection.execute("CREATE DATABASE #{connection.quote_table_name(name)} " \
+                             "TEMPLATE #{connection.quote_table_name(db.database)}")
+        end
+      ensure
+        admin&.remove_connection
+      end
+
+      # ActiveRecord names a connection's owner by class name, so the class
+      # needs a constant.
+      def maintenance_record
+        return self.class::MaintenanceRecord if self.class.const_defined?(:MaintenanceRecord, false)
+
+        self.class.const_set(:MaintenanceRecord, Class.new(::ActiveRecord::Base) { self.abstract_class = true })
+      end
+
+      def disconnect
+        return unless defined?(::ActiveRecord::Base)
+
+        ::ActiveRecord::Base.connection_handler.connection_pool_list(:all).each(&:disconnect!)
+      end
+
+      # The parent disconnected before fork, so inherited pools hold no sockets.
+      # Disconnecting again in the child is idempotent and guarantees the child
+      # checks out a fresh connection of its own on first use.
+      def reconnect
+        disconnect
+      end
+    end
+  end
+end

data/lib/saboteur/hooks.rb

@@ -0,0 +1,51 @@
+# SPDX-License-Identifier: MIT
+# frozen_string_literal: true
+
+module Saboteur
+  # Lifecycle callbacks declared in config.
+  #
+  #   before_boot  - parent, before the target app is loaded
+  #   after_boot   - parent, after the app and test files are loaded
+  #   before_fork  - parent, once per mutant, just before fork
+  #   after_fork   - child, right after fork (reconnect DBs, reseed RNG)
+  #   around_mutant - child, wraps applying the mutant and running its tests
+  #   worker_start - each parallel worker (--jobs N), with its number, before
+  #                  its first subject
+  class Hooks
+    NAMES = %i[before_boot after_boot before_fork after_fork worker_start].freeze
+    AROUND = :around_mutant
+
+    def initialize
+      @callbacks = Hash.new { |hash, key| hash[key] = [] }
+    end
+
+    def register(name, callable = nil, &block)
+      name = name.to_sym
+      raise ArgumentError, "unknown hook: #{name}" unless NAMES.include?(name) || name == AROUND
+
+      @callbacks[name] << (callable || block || raise(ArgumentError, "hook #{name} needs a callable"))
+      self
+    end
+
+    def run(name, *args)
+      raise ArgumentError, "use #around for #{AROUND}" if name.to_sym == AROUND
+
+      @callbacks[name.to_sym].each { |callback| callback.call(*args) }
+      nil
+    end
+
+    # Each around_mutant callback receives the mutation and a proc that runs
+    # the remaining chain. The first registered callback is the outermost.
+    # Returns the block's result whatever the callbacks return; if a callback
+    # never calls the chain, the block does not run and nil is returned.
+    def around(mutation, &inner)
+      result = nil
+      body = -> { result = inner.call }
+      chain = @callbacks[AROUND].reverse.reduce(body) do |next_step, callback|
+        -> { callback.call(mutation, next_step) }
+      end
+      chain.call
+      result
+    end
+  end
+end

data/lib/saboteur/ignore_list.rb

@@ -0,0 +1,62 @@
+# SPDX-License-Identifier: MIT
+# frozen_string_literal: true
+
+require "yaml"
+
+module Saboteur
+  # Survivors the team has accepted. YAML list of entries:
+  #
+  #   - subject: "Person#adult?"
+  #     mutation: "1a2b3c4d5e6f7a8b"   # digest shown next to each survivor
+  #     reason: "equivalent: age is always an integer"
+  #
+  # reason is mandatory; a file with a missing or blank reason does not load.
+  class IgnoreList
+    Entry = Data.define(:subject, :mutation, :reason)
+
+    attr_reader :entries
+
+    def self.load(path)
+      return new([]) unless path && File.file?(path)
+
+      data = YAML.safe_load_file(path) || []
+      raise ConfigError, "#{path}: expected a list of entries" unless data.is_a?(Array)
+
+      new(data.each_with_index.map { |raw, index| parse_entry(raw, path, index) })
+    rescue Psych::SyntaxError => e
+      raise ConfigError, "#{path}: #{e.message}"
+    end
+
+    def self.parse_entry(raw, path, index)
+      raise ConfigError, "#{path}: entry #{index + 1} is not a mapping" unless raw.is_a?(Hash)
+
+      values = %w[subject mutation reason].to_h { |key| [key, raw[key].to_s.strip] }
+      missing = values.select { |_, value| value.empty? }.keys
+      raise ConfigError, "#{path}: entry #{index + 1} is missing #{missing.join(', ')}" if missing.any?
+
+      Entry.new(subject: values["subject"], mutation: values["mutation"], reason: values["reason"])
+    end
+
+    def initialize(entries)
+      @entries = entries
+      @index = entries.to_h { |entry| [[entry.subject, entry.mutation], entry] }
+    end
+
+    # Marks matching survivors as ignored. Returns warnings for stale entries:
+    # entries whose subject was examined but whose mutation no longer exists.
+    def apply(results)
+      seen = {}
+      results.each do |result|
+        key = [result.subject.id, result.digest]
+        entry = @index[key]
+        next unless entry
+
+        seen[key] = true
+        result.ignored = entry.reason if result.survived?
+      end
+      examined = results.to_set { |result| result.subject.id }
+      entries.select { |entry| examined.include?(entry.subject) && !seen[[entry.subject, entry.mutation]] }
+             .map { |entry| "stale ignore entry #{entry.subject} #{entry.mutation}: no such mutation" }
+    end
+  end
+end

data/lib/saboteur/integration/base.rb

@@ -0,0 +1,72 @@
+# SPDX-License-Identifier: MIT
+# frozen_string_literal: true
+
+module Saboteur
+  module Integration
+    TestCase = Data.define(:id, :file)
+    # executed_ids: tests that actually ran, so the runner can tell a broken
+    # filter (nothing ran, every mutant "survives") from a real pass.
+    # failure_classes: the exception class name each failed test died with,
+    # so a kill by a failed assertion can be told from a kill by a crash.
+    # failure_messages: the first line of each failure, so a verdict that
+    # does not hold up can say why the test failed.
+    RunResult = Data.define(:passed, :failed_ids, :executed_ids, :runtime, :failure_classes, :failure_messages) do
+      def initialize(passed:, failed_ids:, executed_ids:, runtime:, failure_classes: {}, failure_messages: {})
+        super
+      end
+    end
+
+    REGISTRY = {} # rubocop:disable Style/MutableConstant
+
+    def self.register(name, klass)
+      REGISTRY[name.to_s] = klass
+    end
+
+    def self.fetch(name)
+      REGISTRY.fetch(name.to_s) { raise Saboteur::Error, "unknown integration: #{name}" }
+    end
+
+    # Contract every test framework adapter implements.
+    #
+    # Lifecycle: #setup runs once in the parent after boot and loads test files
+    # without running any test. #run and #run_all are called inside a forked
+    # child; they must not exit the process and must report by return value,
+    # not by printing.
+    class Base
+      attr_reader :config
+
+      def initialize(config)
+        @config = config
+      end
+
+      def setup
+        raise NotImplementedError
+      end
+
+      # @return [Array<TestCase>]
+      def all_tests
+        raise NotImplementedError
+      end
+
+      # Runs the given tests, stopping at the first failure.
+      # @param ids [Array<String>] ids from #all_tests
+      # @return [RunResult]
+      def run(_ids)
+        raise NotImplementedError
+      end
+
+      # Runs every test, calling recorder.test_started(id, file) before and
+      # recorder.test_finished(id) after each one. Used to build the coverage map.
+      def run_all(_recorder)
+        raise NotImplementedError
+      end
+
+      # Files whose content decides which tests exist (for the cache key).
+      def test_files
+        []
+      end
+    end
+  end
+end
+
+require_relative "rspec"