henitai 0.5.3 → 0.6.0

data/lib/henitai/fork_server/client.rb

@@ -0,0 +1,231 @@
+# frozen_string_literal: true
+
+module Henitai
+  module ForkServer
+    # The parent's end of the connection to the fork server. It sends `run`
+    # requests and collects the server's events; the parent cannot wait on the
+    # mutant processes (they are the server's children), so their ends arrive
+    # as `exited` events.
+    class Client
+      class ServerLost < StandardError; end
+
+      class BootFailed < StandardError; end
+
+      class SpawnFailed < StandardError; end
+
+      SHUTDOWN_GRACE = 5.0
+      SPAWN_TIMEOUT = 30.0
+      REAP_POLL_SECONDS = 0.05
+
+      # @param socket [IO] the parent end of the connection
+      # @param pid [Integer, nil] the server process, reaped on #shutdown
+      # @param shutdown_grace [Float] seconds the server may take to exit before it is killed
+      def initialize(socket:, pid:, shutdown_grace: SHUTDOWN_GRACE)
+        @socket = socket
+        @pid = pid
+        @shutdown_grace = shutdown_grace
+        @reader = Protocol::Reader.new(socket)
+        @next_request_id = 0
+        @boot = nil
+        @spawned = {}
+        @exits = {}
+        @outstanding = []
+      end
+
+      def lost?
+        @lost == true
+      end
+
+      # The connection the parent can select on to learn of relayed events.
+      def io
+        @socket
+      end
+
+      # @raise [BootFailed] when the server reports failure or stays silent
+      # @raise [ServerLost] when the server dies while booting
+      def wait_until_ready(timeout)
+        outcome = await(timeout) { @boot }
+        raise BootFailed, "the fork server did not become ready within #{timeout}s" unless outcome
+        raise BootFailed, outcome.last if outcome.first == :failed
+
+        nil
+      end
+
+      # @return [Integer] pid of the forked mutant process
+      # @raise [SpawnFailed] when the server could not fork this mutant
+      # @raise [ServerLost]
+      def spawn_mutant(mutant_id:, test_files:, log_paths:, env: {})
+        request_id = (@next_request_id += 1)
+        send_message("type" => "run", "request_id" => request_id, "mutant_id" => mutant_id,
+                     "test_files" => test_files, "log_paths" => log_paths, "env" => env)
+        outcome = await(SPAWN_TIMEOUT) { @spawned.delete(request_id) }
+        lose_server("the fork server did not answer a run request") unless outcome
+        raise SpawnFailed, outcome.last if outcome.first == :failed
+
+        @outstanding << outcome.last
+        outcome.last
+      end
+
+      # @return [Status, nil] how the process ended; nil when it is still
+      #   running after +timeout+ seconds
+      def wait_exit(pid, timeout)
+        await(timeout) { @exits.delete(pid) }
+      end
+
+      # Non-blocking variants for the parallel scheduler, which reaps
+      # completions from relayed events. A lost server completes every
+      # outstanding process with a {Status.lost} instead of raising, so no
+      # slot waits for an event that cannot arrive and no verdict is invented.
+      #
+      # @return [Status, nil]
+      def poll_exit(pid)
+        drain_available
+        @exits.delete(pid)
+      end
+
+      # @return [Array(Integer, Status), nil]
+      def poll_any_exit
+        drain_available
+        pid, status = @exits.first
+        return nil unless pid
+
+        @exits.delete(pid)
+        [pid, status]
+      end
+
+      # Whether an event is already in user space (read off the socket but not
+      # yet handed out), which the socket's readability no longer signals.
+      def pending_events?
+        !@exits.empty? || @reader.buffered_message?
+      end
+
+      # Stops tracking a pid whose result is no longer wanted, such as a child
+      # the caller gave up on after a timeout. Its exit, if one already
+      # arrived or arrives later, is dropped rather than left for a recycled pid.
+      def forget(pid)
+        drain_available
+        @outstanding.delete(pid)
+        @exits.delete(pid)
+      end
+
+      # Gives up on a server that is lost or stalled: it must not fork another
+      # mutant that a local run is about to repeat.
+      def abandon
+        @lost = true
+        kill_outstanding
+        close_socket
+        reap_server
+        @pid = nil
+      end
+
+      def shutdown
+        send_message("type" => "shutdown")
+      rescue ServerLost
+        nil
+      ensure
+        close_socket
+        reap_server
+      end
+
+      private
+
+      # The registry keeps fallback children from inheriting this connection.
+      def close_socket
+        InheritedFdRegistry.unregister(@socket)
+        @socket.close unless @socket.closed?
+      end
+
+      # Without its server a mutant process has no one to reap it, and a local
+      # rerun of the same mutant must not race it.
+      def kill_outstanding
+        @outstanding.each { |pid| GroupKiller.kill(pid) }
+      end
+
+      def await(timeout)
+        deadline = now + timeout
+        until (result = yield)
+          remaining = deadline - now
+          return nil unless remaining.positive?
+
+          pump(remaining)
+        end
+        result
+      end
+
+      def pump(timeout)
+        message = @reader.next_message(timeout)
+        return if message == Protocol::Reader::TIMEOUT
+        return lose_server if message.nil?
+
+        record(message)
+      end
+
+      def drain_available
+        loop do
+          message = @reader.next_message(0)
+          break if message == Protocol::Reader::TIMEOUT
+
+          message.nil? ? lose_server : record(message)
+        end
+      rescue ServerLost
+        nil
+      end
+
+      def lose_server(reason = "the fork server closed the connection")
+        @lost = true
+        kill_outstanding
+        @outstanding.each { |pid| @exits[pid] ||= Status.lost }
+        @outstanding.clear
+        raise ServerLost, reason
+      end
+
+      def record(message)
+        case message["type"]
+        when "ready" then @boot = [:ready]
+        when "boot_failed" then @boot = [:failed, message["reason"]]
+        when "spawned" then @spawned[message["request_id"]] = [:ok, message["pid"]]
+        when "spawn_failed" then @spawned[message["request_id"]] = [:failed, message["reason"]]
+        when "exited" then record_exit(message)
+        end
+      end
+
+      def record_exit(message)
+        pid = message["pid"]
+        return unless @outstanding.delete(pid)
+
+        @exits[pid] = Status.from_event(message)
+      end
+
+      def send_message(message)
+        @socket.write(Protocol.encode(message))
+      rescue IOError, SystemCallError => e
+        raise ServerLost, "the fork server connection is gone: #{e.message}"
+      end
+
+      def reap_server
+        return if @pid.nil? || exited_within?(@shutdown_grace)
+
+        Process.kill(:SIGKILL, @pid)
+        Process.wait(@pid)
+      rescue Errno::ECHILD, Errno::ESRCH
+        nil
+      end
+
+      def exited_within?(grace)
+        deadline = now + grace
+        loop do
+          return true if Process.wait(@pid, Process::WNOHANG)
+          return false if now > deadline
+
+          sleep(REAP_POLL_SECONDS)
+        end
+      rescue Errno::ECHILD
+        true
+      end
+
+      def now
+        Process.clock_gettime(Process::CLOCK_MONOTONIC)
+      end
+    end
+  end
+end

data/lib/henitai/fork_server/group_killer.rb

@@ -0,0 +1,48 @@
+# frozen_string_literal: true
+
+module Henitai
+  module ForkServer
+    # Signals a mutant process the fork server forked. Such a process leads
+    # its own process group only once its bootstrap ran +setpgid+, so until
+    # then the group does not exist and the process itself is signalled.
+    # Its parent is the server, so it is never ours to wait on.
+    module GroupKiller
+      # @param leader [Boolean] whether the pid itself may still be alive;
+      #   false once it was reaped, when only stragglers in its group remain
+      def self.kill(pid, signal = :SIGKILL, leader: true)
+        Process.kill(signal, -pid)
+      rescue Errno::ESRCH
+        kill_leader(pid, signal) if leader
+      rescue Errno::EPERM
+        nil
+      end
+
+      def self.kill_leader(pid, signal)
+        Process.kill(signal, pid)
+      rescue Errno::ESRCH, Errno::EPERM
+        nil
+      end
+      private_class_method :kill_leader
+
+      # Signal 0 probes existence only; a missing process or group is not alive.
+      def self.alive?(pid, leader: true)
+        Process.kill(0, -pid)
+        true
+      rescue Errno::ESRCH
+        leader && probe(pid)
+      rescue Errno::EPERM
+        true
+      end
+
+      def self.probe(pid)
+        Process.kill(0, pid)
+        true
+      rescue Errno::ESRCH
+        false
+      rescue Errno::EPERM
+        true
+      end
+      private_class_method :probe
+    end
+  end
+end

data/lib/henitai/fork_server/group_terminator.rb

@@ -0,0 +1,47 @@
+# frozen_string_literal: true
+
+module Henitai
+  module ForkServer
+    # Stops a mutant process of the fork server and everything it started.
+    #
+    # The process is the server's child, not ours: waiting on it raises
+    # ECHILD, so liveness is probed with signal 0 instead.
+    class GroupTerminator
+      GRACE_SECONDS = 2.0
+      KILL_SECONDS = 1.0
+      POLL_SECONDS = 0.02
+
+      def initialize(killer: GroupKiller, clock: Process.method(:clock_gettime), pauser: Kernel.method(:sleep))
+        @killer = killer
+        @clock = clock
+        @pauser = pauser
+      end
+
+      # @param leader [Boolean] false when the process already ended and only
+      #   members of its group are left
+      def terminate(pid, leader: true)
+        @killer.kill(pid, :SIGTERM, leader:)
+        return if gone_within?(pid, GRACE_SECONDS, leader)
+
+        @killer.kill(pid, :SIGKILL, leader:)
+        gone_within?(pid, KILL_SECONDS, leader)
+      end
+
+      private
+
+      def gone_within?(pid, seconds, leader)
+        deadline = now + seconds
+        while @killer.alive?(pid, leader:)
+          return false if now >= deadline
+
+          @pauser.call(POLL_SECONDS)
+        end
+        true
+      end
+
+      def now
+        @clock.call(Process::CLOCK_MONOTONIC)
+      end
+    end
+  end
+end

data/lib/henitai/fork_server/launcher.rb

@@ -0,0 +1,38 @@
+# frozen_string_literal: true
+
+module Henitai
+  module ForkServer
+    # Forks one mutant process from the booted server. The process then does
+    # what a per-mutant child does after its own boot: process-group and trap
+    # setup, output capture, activation, the tests, the child report.
+    class Launcher
+      # @param mutants [Hash{String => Mutant}] the run's mutants by id
+      def initialize(integration:, mutants:, socket:, server_pid: Process.pid)
+        @integration = integration
+        @mutants = mutants
+        @socket = socket
+        @server_pid = server_pid
+      end
+
+      # @param request [Hash] a `run` request
+      # @return [Integer] pid of the forked process
+      def call(request)
+        mutant = @mutants.fetch(request["mutant_id"]) { raise KeyError, "unknown mutant #{request['mutant_id']}" }
+        test_files = Array(request["test_files"])
+        log_paths = request.fetch("log_paths").transform_keys(&:to_sym)
+        env = request["env"] || {}
+        Process.fork { run_forked(mutant, test_files, log_paths, env) }
+      end
+
+      private
+
+      def run_forked(mutant, test_files, log_paths, env)
+        @socket.close
+        Integration::ChildBootstrap.after_fork!(parent_pid: @server_pid)
+        env.each { |key, value| ENV[key] = value }
+        ENV["HENITAI_MUTANT_ID"] = mutant.id
+        Process.exit(@integration.run_in_child(mutant:, test_files:, log_paths:))
+      end
+    end
+  end
+end

data/lib/henitai/fork_server/mutant_runner.rb

@@ -0,0 +1,141 @@
+# frozen_string_literal: true
+
+require "forwardable"
+
+module Henitai
+  module ForkServer
+    # Runs mutants through the fork server instead of booting the test
+    # environment in every mutant process. Offers the integration's
+    # #run_mutant, so the engine's retry and timeout logic is unchanged.
+    #
+    # Also stands in for the integration of a parallel run: #spawn_mutant hands
+    # the scheduler a handle for a process the server forked, and #runtime lets
+    # it learn of that process's exit from the server's events.
+    #
+    # Falls back to the integration's own per-mutant boot when the server is
+    # lost, or when it cannot fork one particular mutant (for example the
+    # control run's mutant, which the server never saw).
+    class MutantRunner
+      extend Forwardable
+
+      EXIT_GRACE_SECONDS = 5.0
+
+      def_delegators :@integration, :build_result, :select_tests, :test_files
+
+      # @param fallback_timeout [#call, nil] test files -> timeout for a mutant
+      #   the parent boots itself; the timeout handed to #run_mutant excludes
+      #   boot while the server is active and would be far too short for it.
+      # rubocop:disable Metrics/ParameterLists
+      def initialize(integration:, client:, env: {}, warner: nil, fallback_timeout: nil,
+                     report_store: Integration::ChildReportStore.new, terminator: GroupTerminator.new)
+        # rubocop:enable Metrics/ParameterLists
+        @integration = integration
+        @client = client
+        @env = env
+        @warner = warner || ->(message) { Kernel.warn(message) }
+        @fallback_timeout = fallback_timeout
+        @report_store = report_store
+        @terminator = terminator
+        @active = true
+      end
+
+      # False once the server is gone; a lost connection is noticed here even
+      # when no request is pending.
+      def active?
+        deactivate("the fork server closed the connection") if @active && @client.lost?
+        @active
+      end
+
+      def runtime
+        @runtime ||= Runtime.new(client: @client)
+      end
+
+      # @return [Integration::ChildHandle]
+      def spawn_mutant(mutant:, test_files:)
+        return spawn_locally(mutant, test_files) unless active?
+
+        spawn_through_server(mutant, test_files)
+      rescue Client::SpawnFailed
+        spawn_locally(mutant, test_files)
+      rescue Client::ServerLost => e
+        deactivate(e.message)
+        spawn_locally(mutant, test_files)
+      end
+
+      def run_mutant(mutant:, test_files:, timeout:)
+        return fallback(mutant, test_files, timeout) unless active?
+
+        run_through_server(mutant, test_files, timeout)
+      rescue Client::SpawnFailed
+        fallback(mutant, test_files, timeout)
+      rescue Client::ServerLost => e
+        deactivate(e.message)
+        fallback(mutant, test_files, timeout)
+      end
+
+      def stop
+        @active = false
+        @client.shutdown
+      end
+
+      private
+
+      def spawn_through_server(mutant, test_files)
+        log_paths = @integration.mutant_log_paths(mutant)
+        @report_store.clear(log_paths[:report_path])
+        pid = @client.spawn_mutant(mutant_id: mutant.id, test_files:, log_paths:, env: spawn_env)
+        runtime.served(pid)
+        Integration::ChildHandle.new(pid:, log_paths:)
+      end
+
+      def spawn_locally(mutant, test_files)
+        handle = @integration.spawn_mutant(mutant:, test_files:)
+        runtime.local(handle.pid)
+        Integration::ChildHandle.new(pid: handle.pid, log_paths: handle.log_paths,
+                                     timeout: fallback_timeout(test_files))
+      end
+
+      # The scheduler exports the slot of the process it is about to spawn.
+      def spawn_env
+        slot = ENV.fetch(SlotScheduler::WORKER_SLOT_ENV, nil)
+        slot ? @env.merge(SlotScheduler::WORKER_SLOT_ENV => slot) : @env
+      end
+
+      def run_through_server(mutant, test_files, timeout)
+        log_paths = @integration.mutant_log_paths(mutant)
+        @report_store.clear(log_paths[:report_path])
+        pid = @client.spawn_mutant(mutant_id: mutant.id, test_files:, log_paths:, env: @env)
+        wait_result = wait_for(pid, timeout)
+        @integration.build_result(wait_result, log_paths)
+      ensure
+        @terminator.terminate(pid, leader: false) if pid && wait_result != :timeout
+      end
+
+      def wait_for(pid, timeout)
+        status = @client.wait_exit(pid, timeout)
+        return status if status
+
+        @terminator.terminate(pid)
+        @client.wait_exit(pid, EXIT_GRACE_SECONDS)
+        @client.forget(pid)
+        :timeout
+      end
+
+      def fallback(mutant, test_files, timeout)
+        @integration.run_mutant(mutant:, test_files:, timeout: fallback_timeout(test_files) || timeout)
+      end
+
+      def fallback_timeout(test_files)
+        @fallback_timeout&.call(test_files)
+      end
+
+      def deactivate(reason)
+        return unless @active
+
+        @active = false
+        @client.abandon
+        @warner.call("henitai: the fork server stopped (#{reason}); falling back to per-mutant boot")
+      end
+    end
+  end
+end

data/lib/henitai/fork_server/protocol.rb

@@ -0,0 +1,73 @@
+# frozen_string_literal: true
+
+require "json"
+
+module Henitai
+  module ForkServer
+    # Line-delimited JSON messages between the henitai parent and its fork
+    # server. One message per line keeps both ends independent of how the
+    # socket chunks the stream.
+    #
+    # Requests (parent -> server):
+    #   run:      request_id, mutant_id, test_files, log_paths, env
+    #   shutdown
+    # Events (server -> parent):
+    #   ready | boot_failed(reason)
+    #   spawned(request_id, pid) | spawn_failed(request_id, reason)
+    #   exited(pid, exitstatus | termsig)
+    module Protocol
+      MAX_READ_BYTES = 65_536
+
+      def self.encode(message)
+        "#{JSON.generate(message)}\n"
+      end
+
+      # Buffers a stream and yields whole messages. Returns nil at end of
+      # stream, so a dead peer is distinguishable from a quiet one.
+      class Reader
+        TIMEOUT = :timeout
+
+        def initialize(io)
+          @io = io
+          @buffer = +""
+          @eof = false
+        end
+
+        # @param timeout [Float, nil] seconds to wait for a complete message
+        # @return [Hash, Symbol, nil] the message, TIMEOUT, or nil at end of stream
+        def next_message(timeout)
+          deadline = timeout && (Process.clock_gettime(Process::CLOCK_MONOTONIC) + timeout)
+          loop do
+            line = @buffer.slice!(/\A.*\n/)
+            return JSON.parse(line) if line
+            return nil if @eof
+
+            fill(deadline) or return TIMEOUT
+          end
+        end
+
+        # True when a whole message is already read off the socket, so the
+        # socket itself will not signal it as readable.
+        def buffered_message?
+          @buffer.include?("\n")
+        end
+
+        private
+
+        def fill(deadline)
+          remaining = deadline && [deadline - Process.clock_gettime(Process::CLOCK_MONOTONIC), 0].max
+          return false unless @io.wait_readable(remaining)
+
+          chunk = @io.read_nonblock(MAX_READ_BYTES, exception: false)
+          return true if chunk == :wait_readable
+
+          @eof = true if chunk.nil?
+          @buffer << chunk unless chunk.nil?
+          true
+        rescue EOFError, Errno::ECONNRESET
+          @eof = true
+        end
+      end
+    end
+  end
+end

data/lib/henitai/fork_server/reaper.rb

@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+
+module Henitai
+  module ForkServer
+    # Collects the processes that have ended since the last call, without
+    # blocking. The fork server is the only process that can wait on its
+    # mutant processes.
+    module Reaper
+      # @return [Array<Array(Integer, Status)>] pid and status per ended child
+      def self.reap
+        exits = [] # : Array[[Integer, Status]]
+        loop do
+          pid, status = Process.wait2(-1, Process::WNOHANG)
+          break unless pid
+
+          exits << [pid, Status.from_process_status(status)]
+        end
+        exits
+      rescue Errno::ECHILD
+        exits
+      end
+    end
+  end
+end