henitai 0.5.1 → 0.6.0

data/lib/henitai/fork_server/launcher.rb

@@ -0,0 +1,38 @@
+# frozen_string_literal: true
+
+module Henitai
+  module ForkServer
+    # Forks one mutant process from the booted server. The process then does
+    # what a per-mutant child does after its own boot: process-group and trap
+    # setup, output capture, activation, the tests, the child report.
+    class Launcher
+      # @param mutants [Hash{String => Mutant}] the run's mutants by id
+      def initialize(integration:, mutants:, socket:, server_pid: Process.pid)
+        @integration = integration
+        @mutants = mutants
+        @socket = socket
+        @server_pid = server_pid
+      end
+
+      # @param request [Hash] a `run` request
+      # @return [Integer] pid of the forked process
+      def call(request)
+        mutant = @mutants.fetch(request["mutant_id"]) { raise KeyError, "unknown mutant #{request['mutant_id']}" }
+        test_files = Array(request["test_files"])
+        log_paths = request.fetch("log_paths").transform_keys(&:to_sym)
+        env = request["env"] || {}
+        Process.fork { run_forked(mutant, test_files, log_paths, env) }
+      end
+
+      private
+
+      def run_forked(mutant, test_files, log_paths, env)
+        @socket.close
+        Integration::ChildBootstrap.after_fork!(parent_pid: @server_pid)
+        env.each { |key, value| ENV[key] = value }
+        ENV["HENITAI_MUTANT_ID"] = mutant.id
+        Process.exit(@integration.run_in_child(mutant:, test_files:, log_paths:))
+      end
+    end
+  end
+end

data/lib/henitai/fork_server/mutant_runner.rb

@@ -0,0 +1,141 @@
+# frozen_string_literal: true
+
+require "forwardable"
+
+module Henitai
+  module ForkServer
+    # Runs mutants through the fork server instead of booting the test
+    # environment in every mutant process. Offers the integration's
+    # #run_mutant, so the engine's retry and timeout logic is unchanged.
+    #
+    # Also stands in for the integration of a parallel run: #spawn_mutant hands
+    # the scheduler a handle for a process the server forked, and #runtime lets
+    # it learn of that process's exit from the server's events.
+    #
+    # Falls back to the integration's own per-mutant boot when the server is
+    # lost, or when it cannot fork one particular mutant (for example the
+    # control run's mutant, which the server never saw).
+    class MutantRunner
+      extend Forwardable
+
+      EXIT_GRACE_SECONDS = 5.0
+
+      def_delegators :@integration, :build_result, :select_tests, :test_files
+
+      # @param fallback_timeout [#call, nil] test files -> timeout for a mutant
+      #   the parent boots itself; the timeout handed to #run_mutant excludes
+      #   boot while the server is active and would be far too short for it.
+      # rubocop:disable Metrics/ParameterLists
+      def initialize(integration:, client:, env: {}, warner: nil, fallback_timeout: nil,
+                     report_store: Integration::ChildReportStore.new, terminator: GroupTerminator.new)
+        # rubocop:enable Metrics/ParameterLists
+        @integration = integration
+        @client = client
+        @env = env
+        @warner = warner || ->(message) { Kernel.warn(message) }
+        @fallback_timeout = fallback_timeout
+        @report_store = report_store
+        @terminator = terminator
+        @active = true
+      end
+
+      # False once the server is gone; a lost connection is noticed here even
+      # when no request is pending.
+      def active?
+        deactivate("the fork server closed the connection") if @active && @client.lost?
+        @active
+      end
+
+      def runtime
+        @runtime ||= Runtime.new(client: @client)
+      end
+
+      # @return [Integration::ChildHandle]
+      def spawn_mutant(mutant:, test_files:)
+        return spawn_locally(mutant, test_files) unless active?
+
+        spawn_through_server(mutant, test_files)
+      rescue Client::SpawnFailed
+        spawn_locally(mutant, test_files)
+      rescue Client::ServerLost => e
+        deactivate(e.message)
+        spawn_locally(mutant, test_files)
+      end
+
+      def run_mutant(mutant:, test_files:, timeout:)
+        return fallback(mutant, test_files, timeout) unless active?
+
+        run_through_server(mutant, test_files, timeout)
+      rescue Client::SpawnFailed
+        fallback(mutant, test_files, timeout)
+      rescue Client::ServerLost => e
+        deactivate(e.message)
+        fallback(mutant, test_files, timeout)
+      end
+
+      def stop
+        @active = false
+        @client.shutdown
+      end
+
+      private
+
+      def spawn_through_server(mutant, test_files)
+        log_paths = @integration.mutant_log_paths(mutant)
+        @report_store.clear(log_paths[:report_path])
+        pid = @client.spawn_mutant(mutant_id: mutant.id, test_files:, log_paths:, env: spawn_env)
+        runtime.served(pid)
+        Integration::ChildHandle.new(pid:, log_paths:)
+      end
+
+      def spawn_locally(mutant, test_files)
+        handle = @integration.spawn_mutant(mutant:, test_files:)
+        runtime.local(handle.pid)
+        Integration::ChildHandle.new(pid: handle.pid, log_paths: handle.log_paths,
+                                     timeout: fallback_timeout(test_files))
+      end
+
+      # The scheduler exports the slot of the process it is about to spawn.
+      def spawn_env
+        slot = ENV.fetch(SlotScheduler::WORKER_SLOT_ENV, nil)
+        slot ? @env.merge(SlotScheduler::WORKER_SLOT_ENV => slot) : @env
+      end
+
+      def run_through_server(mutant, test_files, timeout)
+        log_paths = @integration.mutant_log_paths(mutant)
+        @report_store.clear(log_paths[:report_path])
+        pid = @client.spawn_mutant(mutant_id: mutant.id, test_files:, log_paths:, env: @env)
+        wait_result = wait_for(pid, timeout)
+        @integration.build_result(wait_result, log_paths)
+      ensure
+        @terminator.terminate(pid, leader: false) if pid && wait_result != :timeout
+      end
+
+      def wait_for(pid, timeout)
+        status = @client.wait_exit(pid, timeout)
+        return status if status
+
+        @terminator.terminate(pid)
+        @client.wait_exit(pid, EXIT_GRACE_SECONDS)
+        @client.forget(pid)
+        :timeout
+      end
+
+      def fallback(mutant, test_files, timeout)
+        @integration.run_mutant(mutant:, test_files:, timeout: fallback_timeout(test_files) || timeout)
+      end
+
+      def fallback_timeout(test_files)
+        @fallback_timeout&.call(test_files)
+      end
+
+      def deactivate(reason)
+        return unless @active
+
+        @active = false
+        @client.abandon
+        @warner.call("henitai: the fork server stopped (#{reason}); falling back to per-mutant boot")
+      end
+    end
+  end
+end

data/lib/henitai/fork_server/protocol.rb

@@ -0,0 +1,73 @@
+# frozen_string_literal: true
+
+require "json"
+
+module Henitai
+  module ForkServer
+    # Line-delimited JSON messages between the henitai parent and its fork
+    # server. One message per line keeps both ends independent of how the
+    # socket chunks the stream.
+    #
+    # Requests (parent -> server):
+    #   run:      request_id, mutant_id, test_files, log_paths, env
+    #   shutdown
+    # Events (server -> parent):
+    #   ready | boot_failed(reason)
+    #   spawned(request_id, pid) | spawn_failed(request_id, reason)
+    #   exited(pid, exitstatus | termsig)
+    module Protocol
+      MAX_READ_BYTES = 65_536
+
+      def self.encode(message)
+        "#{JSON.generate(message)}\n"
+      end
+
+      # Buffers a stream and yields whole messages. Returns nil at end of
+      # stream, so a dead peer is distinguishable from a quiet one.
+      class Reader
+        TIMEOUT = :timeout
+
+        def initialize(io)
+          @io = io
+          @buffer = +""
+          @eof = false
+        end
+
+        # @param timeout [Float, nil] seconds to wait for a complete message
+        # @return [Hash, Symbol, nil] the message, TIMEOUT, or nil at end of stream
+        def next_message(timeout)
+          deadline = timeout && (Process.clock_gettime(Process::CLOCK_MONOTONIC) + timeout)
+          loop do
+            line = @buffer.slice!(/\A.*\n/)
+            return JSON.parse(line) if line
+            return nil if @eof
+
+            fill(deadline) or return TIMEOUT
+          end
+        end
+
+        # True when a whole message is already read off the socket, so the
+        # socket itself will not signal it as readable.
+        def buffered_message?
+          @buffer.include?("\n")
+        end
+
+        private
+
+        def fill(deadline)
+          remaining = deadline && [deadline - Process.clock_gettime(Process::CLOCK_MONOTONIC), 0].max
+          return false unless @io.wait_readable(remaining)
+
+          chunk = @io.read_nonblock(MAX_READ_BYTES, exception: false)
+          return true if chunk == :wait_readable
+
+          @eof = true if chunk.nil?
+          @buffer << chunk unless chunk.nil?
+          true
+        rescue EOFError, Errno::ECONNRESET
+          @eof = true
+        end
+      end
+    end
+  end
+end

data/lib/henitai/fork_server/reaper.rb

@@ -0,0 +1,24 @@
+# frozen_string_literal: true
+
+module Henitai
+  module ForkServer
+    # Collects the processes that have ended since the last call, without
+    # blocking. The fork server is the only process that can wait on its
+    # mutant processes.
+    module Reaper
+      # @return [Array<Array(Integer, Status)>] pid and status per ended child
+      def self.reap
+        exits = [] # : Array[[Integer, Status]]
+        loop do
+          pid, status = Process.wait2(-1, Process::WNOHANG)
+          break unless pid
+
+          exits << [pid, Status.from_process_status(status)]
+        end
+        exits
+      rescue Errno::ECHILD
+        exits
+      end
+    end
+  end
+end

data/lib/henitai/fork_server/runtime.rb

@@ -0,0 +1,109 @@
+# frozen_string_literal: true
+
+module Henitai
+  module ForkServer
+    # The process primitives {SlotScheduler} drives, for a parallel run whose
+    # mutant processes belong to the fork server (ADR-15).
+    #
+    # The parent cannot wait on a grandchild, so a served pid completes when
+    # the server relays its +exited+ event. A pid forked locally (the per-mutant
+    # fallback) is waited on as usual. Signals go straight to the OS: every
+    # mutant process leads its own process group.
+    class Runtime
+      WAIT_SECONDS = 5.0
+      ANY_PID = -1
+
+      # @param client [Client]
+      # @param base [#clock_gettime, #wait2, #kill, #wait, #pause, #trap]
+      def initialize(client:, base: ProcessWorkerRunner::Runtime.new)
+        @client = client
+        @base = base
+        @served = []
+        @local = []
+      end
+
+      def served(pid)
+        @served << pid
+      end
+
+      def local(pid)
+        @local << pid
+      end
+
+      # Sockets the event loop must wake on besides its signal pipe.
+      def wakeup_ios
+        @client.lost? ? [] : [@client.io]
+      end
+
+      # An event already read off the socket does not wake a select on it.
+      def pending_events?
+        @client.pending_events?
+      end
+
+      def wait2(pid, flags = nil)
+        return any_exit(flags) if pid == ANY_PID
+        return served_exit(pid) if @served.include?(pid)
+
+        local_exit(pid, flags)
+      end
+
+      def wait(pid)
+        return @base.wait(pid) unless @served.include?(pid)
+
+        status = await_exit(pid)
+        @served.delete(pid) if status
+        status
+      end
+
+      # A served process leads no group until its bootstrap ran +setpgid+;
+      # until then the signal goes to the process itself.
+      def kill(signal, pid)
+        @base.kill(signal, pid)
+      rescue Errno::ESRCH
+        raise unless pid.negative? && @served.include?(-pid)
+
+        @base.kill(signal, -pid)
+      end
+
+      def clock_gettime(clock_id) = @base.clock_gettime(clock_id)
+      def pause(seconds) = @base.pause(seconds)
+      def trap(signal, handler = nil, &) = @base.trap(signal, handler, &)
+
+      private
+
+      def any_exit(flags)
+        finished = @client.poll_any_exit
+        if finished
+          @served.delete(finished.first)
+          return finished
+        end
+
+        @local.each do |pid|
+          result = local_exit(pid, flags)
+          return result if result
+        end
+        nil
+      end
+
+      def served_exit(pid)
+        status = @client.poll_exit(pid)
+        return nil unless status
+
+        @served.delete(pid)
+        [pid, status]
+      end
+
+      def local_exit(pid, flags)
+        result = @base.wait2(pid, flags)
+        @local.delete(pid) if result
+        result
+      end
+
+      def await_exit(pid)
+        @client.wait_exit(pid, WAIT_SECONDS)
+      rescue Client::ServerLost
+        @client.poll_exit(pid)
+      end
+    end
+  end
+end

data/lib/henitai/fork_server/server.rb

@@ -0,0 +1,108 @@
+# frozen_string_literal: true
+
+module Henitai
+  module ForkServer
+    # The request loop of the fork server process. It boots once (the boot
+    # callable loads the integration environment), then forks one mutant
+    # process per `run` request through the launcher and relays how each one
+    # ended, because only this process can wait on them.
+    class Server
+      POLL_SECONDS = 0.5
+
+      # @param socket [IO] the server end of the connection to the parent
+      # @param boot [#call] loads the environment; raising reports `boot_failed`
+      # @param launcher [#call] request Hash -> pid of the forked mutant process
+      # @param reaper [#call] -> Array of [pid, Status] for ended children
+      # @param wake_reader [IO, nil] readable when a child ended (CHLD wakeup)
+      # @param killer [#call] pid -> kills that pid's process group
+      # rubocop:disable Metrics/ParameterLists
+      def initialize(socket:, boot:, launcher:, reaper: Reaper.method(:reap), wake_reader: nil,
+                     killer: GroupKiller.method(:kill))
+        # rubocop:enable Metrics/ParameterLists
+        @socket = socket
+        @reader = Protocol::Reader.new(socket)
+        @boot = boot
+        @launcher = launcher
+        @reaper = reaper
+        @wake_reader = wake_reader
+        @killer = killer
+        @live = []
+        @shutdown = false
+      end
+
+      def serve
+        return unless announce_boot
+
+        until @shutdown
+          wait_for_activity
+          relay_exits
+          handle_requests
+        end
+      ensure
+        @live.each { |pid| @killer.call(pid) }
+      end
+
+      private
+
+      def announce_boot
+        @boot.call
+        send_event("type" => "ready")
+        true
+      rescue StandardError, ScriptError, SystemExit => e
+        send_event("type" => "boot_failed", "reason" => "#{e.class}: #{e.message}")
+        false
+      end
+
+      def wait_for_activity
+        ready = IO.select([@socket, @wake_reader].compact, nil, nil, POLL_SECONDS)
+        drain_wake_reader if @wake_reader && ready&.first&.include?(@wake_reader)
+      end
+
+      def drain_wake_reader
+        loop { @wake_reader.read_nonblock(4096) }
+      rescue IO::WaitReadable, EOFError
+        nil
+      end
+
+      def relay_exits
+        @reaper.call.each do |pid, status|
+          next unless @live.delete(pid)
+
+          send_event({ "type" => "exited", "pid" => pid }.merge(status.to_event))
+        end
+      end
+
+      def handle_requests
+        loop do
+          message = @reader.next_message(0)
+          return @shutdown = true if message.nil?
+          return unless message.is_a?(Hash)
+
+          dispatch(message)
+        end
+      end
+
+      def dispatch(message)
+        case message["type"]
+        when "run" then launch(message)
+        when "shutdown" then @shutdown = true
+        end
+      end
+
+      def launch(request)
+        pid = @launcher.call(request)
+        @live << pid
+        send_event("type" => "spawned", "request_id" => request["request_id"], "pid" => pid)
+      rescue StandardError => e
+        send_event("type" => "spawn_failed", "request_id" => request["request_id"],
+                   "reason" => "#{e.class}: #{e.message}")
+      end
+
+      def send_event(event)
+        @socket.write(Protocol.encode(event))
+      rescue IOError, SystemCallError
+        @shutdown = true
+      end
+    end
+  end
+end

data/lib/henitai/fork_server/server_process.rb

@@ -0,0 +1,73 @@
+# frozen_string_literal: true
+
+require "socket"
+require "fileutils"
+
+module Henitai
+  module ForkServer
+    # Forks the fork server from the parent, after the mutants exist, and
+    # returns the parent's Client for it. In the server process: set up like
+    # any mutant child, send boot output to a log file, boot the integration
+    # environment once, then serve requests until told to stop.
+    class ServerProcess
+      def self.start(integration:, mutants:, test_files:, log_path:)
+        new(integration:, mutants:, test_files:, log_path:).start
+      end
+
+      def initialize(integration:, mutants:, test_files:, log_path:)
+        @integration = integration
+        @mutants = mutants.to_h { |mutant| [mutant.id, mutant] }
+        @test_files = test_files
+        @log_path = log_path
+      end
+
+      # @return [Client]
+      def start
+        FileUtils.mkdir_p(File.dirname(@log_path))
+        parent_socket, server_socket = UNIXSocket.pair
+        InheritedFdRegistry.register(parent_socket)
+        parent_pid = Process.pid
+        pid = Process.fork do
+          parent_socket.close
+          serve(server_socket, parent_pid)
+        end
+        server_socket.close
+        Client.new(socket: parent_socket, pid:)
+      end
+
+      private
+
+      def serve(socket, parent_pid)
+        Integration::ChildBootstrap.after_fork!(parent_pid:)
+        redirect_output
+        wake_reader, wake_writer = IO.pipe
+        [wake_reader, wake_writer].each { |io| InheritedFdRegistry.register(io) }
+        launcher = Launcher.new(integration: @integration, mutants: @mutants, socket:)
+        boot = -> { boot_environment(wake_writer) }
+        Server.new(socket:, boot:, launcher:, wake_reader:).serve
+        Process.exit!(0)
+      end
+
+      def boot_environment(wake_writer)
+        @integration.preload_environment(@test_files)
+        Signal.trap("CHLD") { wake(wake_writer) }
+      end
+
+      def wake(writer)
+        writer.write_nonblock(".")
+      rescue IO::WaitWritable, IOError
+        nil
+      end
+
+      def redirect_output
+        File.open(@log_path, "w") do |log|
+          $stdout.reopen(log)
+          $stderr.reopen(log)
+        end
+        $stdin.reopen(File::NULL)
+        $stdout.sync = true
+        $stderr.sync = true
+      end
+    end
+  end
+end

data/lib/henitai/fork_server/status.rb

@@ -0,0 +1,52 @@
+# frozen_string_literal: true
+
+module Henitai
+  module ForkServer
+    # Stands in for Process::Status of a mutant process that the parent cannot
+    # wait on itself: the fork server reaps it and relays how it ended.
+    # Answers the subset MutantVerdict and ScenarioExecutionResult read.
+    class Status
+      attr_reader :exitstatus, :termsig
+
+      def self.from_event(event)
+        new(exitstatus: event["exitstatus"], termsig: event["termsig"])
+      end
+
+      def self.from_process_status(status)
+        new(exitstatus: status.exitstatus, termsig: status.termsig)
+      end
+
+      # Stands in for a process whose end the lost server never relayed. It
+      # says nothing about the mutant, so it must never become a verdict.
+      def self.lost
+        new(exitstatus: nil, termsig: 9, lost: true)
+      end
+
+      def initialize(exitstatus:, termsig:, lost: false)
+        @exitstatus = exitstatus
+        @termsig = termsig
+        @lost = lost
+      end
+
+      def lost?
+        @lost
+      end
+
+      def exited?
+        !signaled?
+      end
+
+      def signaled?
+        !termsig.nil?
+      end
+
+      def success?
+        !signaled? && exitstatus.to_i.zero?
+      end
+
+      def to_event
+        { "exitstatus" => exitstatus, "termsig" => termsig }
+      end
+    end
+  end
+end

data/lib/henitai/fork_server.rb

@@ -0,0 +1,43 @@
+# frozen_string_literal: true
+
+require_relative "fork_server/protocol"
+require_relative "fork_server/status"
+require_relative "fork_server/reaper"
+require_relative "fork_server/group_killer"
+require_relative "fork_server/group_terminator"
+require_relative "fork_server/server"
+require_relative "fork_server/launcher"
+require_relative "fork_server/client"
+require_relative "fork_server/runtime"
+require_relative "fork_server/server_process"
+require_relative "fork_server/mutant_runner"
+
+module Henitai
+  # Opt-in execution model (ADR-15): the test environment boots once in a
+  # long-lived server process and every mutant process is forked from it.
+  module ForkServer
+    BOOT_TIMEOUT = 300.0
+
+    # Starts a server for the run and returns a runner that sends mutants to
+    # it, or nil, after a warning, when the server cannot be booted. The
+    # caller then keeps the per-mutant boot.
+    #
+    # @return [MutantRunner, nil]
+    # rubocop:disable Metrics/ParameterLists
+    def self.start(integration:, mutants:, test_files:, log_path:, warner: nil, boot_timeout: BOOT_TIMEOUT,
+                   fallback_timeout: nil)
+      warner ||= ->(message) { Kernel.warn(message) }
+      client = ServerProcess.start(integration:, mutants:, test_files:, log_path:)
+      begin
+        client.wait_until_ready(boot_timeout)
+      rescue Client::BootFailed, Client::ServerLost => e
+        client.shutdown
+        warner.call("henitai: the fork server failed to boot (#{e.message}); see #{log_path}; " \
+                    "falling back to per-mutant boot")
+        return nil
+      end
+      MutantRunner.new(integration:, client:, warner:, fallback_timeout:)
+    end
+    # rubocop:enable Metrics/ParameterLists
+  end
+end

data/lib/henitai/generated_artifacts.rb

@@ -18,7 +18,10 @@ module Henitai
       mutation-history.sqlite3
       mutation-history.json
       henitai_per_test.json
+      henitai_suite_boot.json
       henitai_dependency_manifest.json
+      henitai_flaky_tests.json
+      henitai_suite_outcomes.json
       .henitai-run.lock
       mutation-logs
       mutation-coverage