henitai 0.5.3 → 0.6.0

data/lib/henitai/fork_server/runtime.rb

@@ -0,0 +1,109 @@
+# frozen_string_literal: true
+
+module Henitai
+  module ForkServer
+    # The process primitives {SlotScheduler} drives, for a parallel run whose
+    # mutant processes belong to the fork server (ADR-15).
+    #
+    # The parent cannot wait on a grandchild, so a served pid completes when
+    # the server relays its +exited+ event. A pid forked locally (the per-mutant
+    # fallback) is waited on as usual. Signals go straight to the OS: every
+    # mutant process leads its own process group.
+    class Runtime
+      WAIT_SECONDS = 5.0
+      ANY_PID = -1
+
+      # @param client [Client]
+      # @param base [#clock_gettime, #wait2, #kill, #wait, #pause, #trap]
+      def initialize(client:, base: ProcessWorkerRunner::Runtime.new)
+        @client = client
+        @base = base
+        @served = []
+        @local = []
+      end
+
+      def served(pid)
+        @served << pid
+      end
+
+      def local(pid)
+        @local << pid
+      end
+
+      # Sockets the event loop must wake on besides its signal pipe.
+      def wakeup_ios
+        @client.lost? ? [] : [@client.io]
+      end
+
+      # An event already read off the socket does not wake a select on it.
+      def pending_events?
+        @client.pending_events?
+      end
+
+      def wait2(pid, flags = nil)
+        return any_exit(flags) if pid == ANY_PID
+        return served_exit(pid) if @served.include?(pid)
+
+        local_exit(pid, flags)
+      end
+
+      def wait(pid)
+        return @base.wait(pid) unless @served.include?(pid)
+
+        status = await_exit(pid)
+        @served.delete(pid) if status
+        status
+      end
+
+      # A served process leads no group until its bootstrap ran +setpgid+;
+      # until then the signal goes to the process itself.
+      def kill(signal, pid)
+        @base.kill(signal, pid)
+      rescue Errno::ESRCH
+        raise unless pid.negative? && @served.include?(-pid)
+
+        @base.kill(signal, -pid)
+      end
+
+      def clock_gettime(clock_id) = @base.clock_gettime(clock_id)
+      def pause(seconds) = @base.pause(seconds)
+      def trap(signal, handler = nil, &) = @base.trap(signal, handler, &)
+
+      private
+
+      def any_exit(flags)
+        finished = @client.poll_any_exit
+        if finished
+          @served.delete(finished.first)
+          return finished
+        end
+
+        @local.each do |pid|
+          result = local_exit(pid, flags)
+          return result if result
+        end
+        nil
+      end
+
+      def served_exit(pid)
+        status = @client.poll_exit(pid)
+        return nil unless status
+
+        @served.delete(pid)
+        [pid, status]
+      end
+
+      def local_exit(pid, flags)
+        result = @base.wait2(pid, flags)
+        @local.delete(pid) if result
+        result
+      end
+
+      def await_exit(pid)
+        @client.wait_exit(pid, WAIT_SECONDS)
+      rescue Client::ServerLost
+        @client.poll_exit(pid)
+      end
+    end
+  end
+end

data/lib/henitai/fork_server/server.rb

@@ -0,0 +1,108 @@
+# frozen_string_literal: true
+
+module Henitai
+  module ForkServer
+    # The request loop of the fork server process. It boots once (the boot
+    # callable loads the integration environment), then forks one mutant
+    # process per `run` request through the launcher and relays how each one
+    # ended, because only this process can wait on them.
+    class Server
+      POLL_SECONDS = 0.5
+
+      # @param socket [IO] the server end of the connection to the parent
+      # @param boot [#call] loads the environment; raising reports `boot_failed`
+      # @param launcher [#call] request Hash -> pid of the forked mutant process
+      # @param reaper [#call] -> Array of [pid, Status] for ended children
+      # @param wake_reader [IO, nil] readable when a child ended (CHLD wakeup)
+      # @param killer [#call] pid -> kills that pid's process group
+      # rubocop:disable Metrics/ParameterLists
+      def initialize(socket:, boot:, launcher:, reaper: Reaper.method(:reap), wake_reader: nil,
+                     killer: GroupKiller.method(:kill))
+        # rubocop:enable Metrics/ParameterLists
+        @socket = socket
+        @reader = Protocol::Reader.new(socket)
+        @boot = boot
+        @launcher = launcher
+        @reaper = reaper
+        @wake_reader = wake_reader
+        @killer = killer
+        @live = []
+        @shutdown = false
+      end
+
+      def serve
+        return unless announce_boot
+
+        until @shutdown
+          wait_for_activity
+          relay_exits
+          handle_requests
+        end
+      ensure
+        @live.each { |pid| @killer.call(pid) }
+      end
+
+      private
+
+      def announce_boot
+        @boot.call
+        send_event("type" => "ready")
+        true
+      rescue StandardError, ScriptError, SystemExit => e
+        send_event("type" => "boot_failed", "reason" => "#{e.class}: #{e.message}")
+        false
+      end
+
+      def wait_for_activity
+        ready = IO.select([@socket, @wake_reader].compact, nil, nil, POLL_SECONDS)
+        drain_wake_reader if @wake_reader && ready&.first&.include?(@wake_reader)
+      end
+
+      def drain_wake_reader
+        loop { @wake_reader.read_nonblock(4096) }
+      rescue IO::WaitReadable, EOFError
+        nil
+      end
+
+      def relay_exits
+        @reaper.call.each do |pid, status|
+          next unless @live.delete(pid)
+
+          send_event({ "type" => "exited", "pid" => pid }.merge(status.to_event))
+        end
+      end
+
+      def handle_requests
+        loop do
+          message = @reader.next_message(0)
+          return @shutdown = true if message.nil?
+          return unless message.is_a?(Hash)
+
+          dispatch(message)
+        end
+      end
+
+      def dispatch(message)
+        case message["type"]
+        when "run" then launch(message)
+        when "shutdown" then @shutdown = true
+        end
+      end
+
+      def launch(request)
+        pid = @launcher.call(request)
+        @live << pid
+        send_event("type" => "spawned", "request_id" => request["request_id"], "pid" => pid)
+      rescue StandardError => e
+        send_event("type" => "spawn_failed", "request_id" => request["request_id"],
+                   "reason" => "#{e.class}: #{e.message}")
+      end
+
+      def send_event(event)
+        @socket.write(Protocol.encode(event))
+      rescue IOError, SystemCallError
+        @shutdown = true
+      end
+    end
+  end
+end

data/lib/henitai/fork_server/server_process.rb

@@ -0,0 +1,73 @@
+# frozen_string_literal: true
+
+require "socket"
+require "fileutils"
+
+module Henitai
+  module ForkServer
+    # Forks the fork server from the parent, after the mutants exist, and
+    # returns the parent's Client for it. In the server process: set up like
+    # any mutant child, send boot output to a log file, boot the integration
+    # environment once, then serve requests until told to stop.
+    class ServerProcess
+      def self.start(integration:, mutants:, test_files:, log_path:)
+        new(integration:, mutants:, test_files:, log_path:).start
+      end
+
+      def initialize(integration:, mutants:, test_files:, log_path:)
+        @integration = integration
+        @mutants = mutants.to_h { |mutant| [mutant.id, mutant] }
+        @test_files = test_files
+        @log_path = log_path
+      end
+
+      # @return [Client]
+      def start
+        FileUtils.mkdir_p(File.dirname(@log_path))
+        parent_socket, server_socket = UNIXSocket.pair
+        InheritedFdRegistry.register(parent_socket)
+        parent_pid = Process.pid
+        pid = Process.fork do
+          parent_socket.close
+          serve(server_socket, parent_pid)
+        end
+        server_socket.close
+        Client.new(socket: parent_socket, pid:)
+      end
+
+      private
+
+      def serve(socket, parent_pid)
+        Integration::ChildBootstrap.after_fork!(parent_pid:)
+        redirect_output
+        wake_reader, wake_writer = IO.pipe
+        [wake_reader, wake_writer].each { |io| InheritedFdRegistry.register(io) }
+        launcher = Launcher.new(integration: @integration, mutants: @mutants, socket:)
+        boot = -> { boot_environment(wake_writer) }
+        Server.new(socket:, boot:, launcher:, wake_reader:).serve
+        Process.exit!(0)
+      end
+
+      def boot_environment(wake_writer)
+        @integration.preload_environment(@test_files)
+        Signal.trap("CHLD") { wake(wake_writer) }
+      end
+
+      def wake(writer)
+        writer.write_nonblock(".")
+      rescue IO::WaitWritable, IOError
+        nil
+      end
+
+      def redirect_output
+        File.open(@log_path, "w") do |log|
+          $stdout.reopen(log)
+          $stderr.reopen(log)
+        end
+        $stdin.reopen(File::NULL)
+        $stdout.sync = true
+        $stderr.sync = true
+      end
+    end
+  end
+end

data/lib/henitai/fork_server/status.rb

@@ -0,0 +1,52 @@
+# frozen_string_literal: true
+
+module Henitai
+  module ForkServer
+    # Stands in for Process::Status of a mutant process that the parent cannot
+    # wait on itself: the fork server reaps it and relays how it ended.
+    # Answers the subset MutantVerdict and ScenarioExecutionResult read.
+    class Status
+      attr_reader :exitstatus, :termsig
+
+      def self.from_event(event)
+        new(exitstatus: event["exitstatus"], termsig: event["termsig"])
+      end
+
+      def self.from_process_status(status)
+        new(exitstatus: status.exitstatus, termsig: status.termsig)
+      end
+
+      # Stands in for a process whose end the lost server never relayed. It
+      # says nothing about the mutant, so it must never become a verdict.
+      def self.lost
+        new(exitstatus: nil, termsig: 9, lost: true)
+      end
+
+      def initialize(exitstatus:, termsig:, lost: false)
+        @exitstatus = exitstatus
+        @termsig = termsig
+        @lost = lost
+      end
+
+      def lost?
+        @lost
+      end
+
+      def exited?
+        !signaled?
+      end
+
+      def signaled?
+        !termsig.nil?
+      end
+
+      def success?
+        !signaled? && exitstatus.to_i.zero?
+      end
+
+      def to_event
+        { "exitstatus" => exitstatus, "termsig" => termsig }
+      end
+    end
+  end
+end

data/lib/henitai/fork_server.rb

@@ -0,0 +1,43 @@
+# frozen_string_literal: true
+
+require_relative "fork_server/protocol"
+require_relative "fork_server/status"
+require_relative "fork_server/reaper"
+require_relative "fork_server/group_killer"
+require_relative "fork_server/group_terminator"
+require_relative "fork_server/server"
+require_relative "fork_server/launcher"
+require_relative "fork_server/client"
+require_relative "fork_server/runtime"
+require_relative "fork_server/server_process"
+require_relative "fork_server/mutant_runner"
+
+module Henitai
+  # Opt-in execution model (ADR-15): the test environment boots once in a
+  # long-lived server process and every mutant process is forked from it.
+  module ForkServer
+    BOOT_TIMEOUT = 300.0
+
+    # Starts a server for the run and returns a runner that sends mutants to
+    # it, or nil, after a warning, when the server cannot be booted. The
+    # caller then keeps the per-mutant boot.
+    #
+    # @return [MutantRunner, nil]
+    # rubocop:disable Metrics/ParameterLists
+    def self.start(integration:, mutants:, test_files:, log_path:, warner: nil, boot_timeout: BOOT_TIMEOUT,
+                   fallback_timeout: nil)
+      warner ||= ->(message) { Kernel.warn(message) }
+      client = ServerProcess.start(integration:, mutants:, test_files:, log_path:)
+      begin
+        client.wait_until_ready(boot_timeout)
+      rescue Client::BootFailed, Client::ServerLost => e
+        client.shutdown
+        warner.call("henitai: the fork server failed to boot (#{e.message}); see #{log_path}; " \
+                    "falling back to per-mutant boot")
+        return nil
+      end
+      MutantRunner.new(integration:, client:, warner:, fallback_timeout:)
+    end
+    # rubocop:enable Metrics/ParameterLists
+  end
+end

data/lib/henitai/generated_artifacts.rb

@@ -18,7 +18,10 @@ module Henitai
       mutation-history.sqlite3
       mutation-history.json
       henitai_per_test.json
+      henitai_suite_boot.json
       henitai_dependency_manifest.json
+      henitai_flaky_tests.json
+      henitai_suite_outcomes.json
       .henitai-run.lock
       mutation-logs
       mutation-coverage

data/lib/henitai/git_diff_analyzer.rb

@@ -9,14 +9,23 @@ module Henitai
   # Shells out to git to discover changed files between two refs.
   #
   # By default the analyzer runs in the current working directory. Callers can
-  # pass dir: to point it at another repository root without changing cwd.
+  # pass dir: to point it at another directory without changing cwd.
+  #
+  # Every path is reported relative to that directory, which need not be the
+  # repository root: `git diff` would otherwise print root-relative paths
+  # while `git ls-files` prints directory-relative ones. Paths are read
+  # NUL-separated so git never quotes non-ASCII names.
   class GitDiffAnalyzer
-    def changed_files(from:, to:, dir: Dir.pwd)
-      stdout, stderr, status = git_diff(dir, "--name-only", from, to)
+    NAME_ONLY = ["--name-only", "--relative", "-z"].freeze
 
-      raise GitDiffError, stderr.strip unless status.success?
+    def changed_files(from:, to:, dir: Dir.pwd)
+      diff_paths(dir, from, to)
+    end
 
-      stdout.split("\n").reject(&:empty?)
+    # Changes on HEAD since its merge base with ref. Commits that landed on
+    # ref after the fork point are not part of this diff.
+    def changed_files_since(ref, dir: Dir.pwd)
+      diff_paths(dir, "#{ref}...HEAD")
     end
 
     def working_tree_changed_files(dir: Dir.pwd)
@@ -55,7 +64,7 @@ module Henitai
     end
 
     def changed_line_ranges(path, from:, to:, dir:)
-      stdout, stderr, status = git_diff(dir, "--unified=0", from, to, "--", path)
+      stdout, stderr, status = git_diff(dir, "--unified=0", "--relative", from, to, "--", path)
 
       raise GitDiffError, stderr.strip unless status.success?
 
@@ -94,23 +103,31 @@ module Henitai
       Open3.capture3(*command)
     end
 
-    def working_tree_tracked_files(dir)
-      stdout, stderr, status = git_diff(dir, "--name-only", "HEAD")
+    def diff_paths(dir, *revisions)
+      stdout, stderr, status = git_diff(dir, *NAME_ONLY, *revisions)
 
       raise GitDiffError, stderr.strip unless status.success?
 
-      stdout.split("\n").reject(&:empty?)
+      paths(stdout)
+    end
+
+    def working_tree_tracked_files(dir)
+      diff_paths(dir, "HEAD")
     end
 
     def untracked_files(dir)
       command = ["git"]
       command += ["-C", dir] if dir
-      command += ["ls-files", "--others", "--exclude-standard"]
+      command += ["ls-files", "--others", "--exclude-standard", "-z"]
       stdout, stderr, status = Open3.capture3(*command)
 
       raise GitDiffError, stderr.strip unless status.success?
 
-      stdout.split("\n").reject(&:empty?)
+      paths(stdout)
+    end
+
+    def paths(stdout)
+      stdout.dup.force_encoding(Encoding::UTF_8).split("\0").reject(&:empty?)
     end
   end
 end

data/lib/henitai/incremental_filter.rb

@@ -1,5 +1,6 @@
 # frozen_string_literal: true
 
+require_relative "signature_line"
 require_relative "verdict_fingerprint"
 
 module Henitai
@@ -8,23 +9,26 @@ module Henitai
   #
   # Reuse rules (conservative — any doubt resolves to re-execution):
   #
+  #   * Both verdicts: the stored {ReuseContext} fingerprint (test
+  #     dependencies, verdict-relevant configuration, henitai version) must
+  #     equal the current one. Rows recorded without a context are never
+  #     reused.
   #   * Killed: the stored verdict must carry both fingerprints, and both the
   #     subject's source and every recorded covering test file must be
   #     byte-identical to what was recorded. Killed is monotone, so no live
   #     coverage is needed.
   #   * Survived: additionally the LIVE covering set (the full-map
   #     intersection computed via {PerTestCoverage}) must equal the recorded
-  #     set in membership, and the run-level dependency fingerprint must be
-  #     unchanged — a new or edited test reaching the mutant always forces a
-  #     re-run (ADR-11). Without a live per-test map or dependency
-  #     fingerprint, survivors always re-execute.
+  #     set in membership — a new or edited test reaching the mutant always
+  #     forces a re-run (ADR-11). Without a live per-test map or a context,
+  #     survivors always re-execute.
   #
   # Timeouts, errors, unknown ids and legacy rows always re-execute.
   class IncrementalFilter
-    def initialize(history_store:, per_test_coverage: nil, dependency_fingerprint: nil)
+    def initialize(history_store:, per_test_coverage: nil, reuse_context: nil)
       @history_store = history_store
       @per_test_coverage = per_test_coverage
-      @dependency_fingerprint = dependency_fingerprint
+      @reuse_context = reuse_context
     end
 
     # @return [Array<Mutant>] the same collection; cache hits get the stored
@@ -48,7 +52,7 @@ module Henitai
 
     private
 
-    attr_reader :history_store, :per_test_coverage, :dependency_fingerprint
+    attr_reader :history_store, :per_test_coverage, :reuse_context
 
     # MutantIdentity deliberately omits source coordinates (line-drift
     # tolerance), so distinct mutants inside one subject can share a stable
@@ -75,7 +79,9 @@ module Henitai
 
     def killed_reusable?(mutant, verdict)
       subject_source_unchanged?(mutant, verdict) &&
-        VerdictFingerprint.tests_fingerprint_current?(verdict.fetch(:covered_tests_fingerprint))
+        VerdictFingerprint.tests_fingerprint_current?(
+          verdict.fetch(:covered_tests_fingerprint), context_sha: context_sha
+        )
     end
 
     # The filter checks per-test map availability itself instead of trusting
@@ -84,17 +90,21 @@ module Henitai
     # unknowable means re-execute.
     def survived_reusable?(mutant, verdict)
       return false unless per_test_coverage&.available?
+      return false if SignatureLine.on?(mutant)
       return false unless subject_source_unchanged?(mutant, verdict)
 
       VerdictFingerprint.survivor_fingerprint_current?(
         verdict.fetch(:covered_tests_fingerprint),
         live_paths: per_test_coverage.tests_covering(mutant),
-        dependency_sha: dependency_fingerprint
+        context_sha: context_sha
       )
     end
 
+    def context_sha = reuse_context&.fingerprint
+
     def subject_source_unchanged?(mutant, verdict)
-      VerdictFingerprint.subject_source_hash(mutant) == verdict.fetch(:subject_source_hash)
+      current = VerdictFingerprint.subject_source_hash(mutant)
+      !current.nil? && current == verdict.fetch(:subject_source_hash)
     end
   end
 end

data/lib/henitai/integration/base.rb

@@ -15,6 +15,9 @@ module Henitai
     class Base
       include ChildRuntimeControl
 
+      EXITED_GROUP_POLL_INTERVAL = 0.05
+      EXITED_GROUP_POLL_COUNT = 40
+
       # Child-run diagnostics. Public so the child-side modules mixed into
       # concrete adapters can reach it without `send`.
       def child_debug_log
@@ -57,6 +60,20 @@ module Henitai
         false
       end
 
+      # True when the integration can boot once in a fork server (ADR-15) and
+      # run mutants from the booted state.
+      def fork_server_supported?
+        false
+      end
+
+      # Loads the test environment and the given test files without running
+      # anything; only called inside the fork server process.
+      #
+      # @param test_files [Array<String>]
+      def preload_environment(test_files)
+        raise NotImplementedError
+      end
+
       def wait_with_timeout(pid, timeout)
         wakeup = Henitai.const_get(:ProcessWakeup).new.install
         return Process.last_status if wait_nonblocking(pid)
@@ -78,6 +95,29 @@ module Henitai
       end
 
       def cleanup_process_group(pid)
+        if leader_exited?(pid)
+          terminate_exited_group(pid)
+        else
+          terminate_running_group(pid)
+        end
+      rescue Errno::EPERM
+        cleanup_child_process(pid)
+      rescue Errno::ESRCH
+        nil
+      end
+
+      private
+
+      # A leader that already exited normally cannot be waited on again, so
+      # the group is polled for members instead of burning the grace period.
+      def terminate_exited_group(pid)
+        Process.kill(:SIGTERM, -pid)
+        return unless group_members_remain?(pid)
+
+        Process.kill(:SIGKILL, -pid)
+      end
+
+      def terminate_running_group(pid)
         grace_period = 2.0
         wakeup = Henitai.const_get(:ProcessWakeup).new.install
         Process.kill(:SIGTERM, -pid)
@@ -88,15 +128,34 @@ module Henitai
         return if wait_nonblocking(pid)
 
         Process.kill(:SIGKILL, -pid)
-      rescue Errno::EPERM
-        cleanup_child_process(pid)
-      rescue Errno::ESRCH
-        nil
       ensure
         wakeup&.close
       end
 
-      private
+      def leader_exited?(pid)
+        !Process.wait(pid, Process::WNOHANG).nil?
+      rescue Errno::ECHILD, Errno::ESRCH
+        true
+      end
+
+      def group_members_remain?(pid)
+        EXITED_GROUP_POLL_COUNT.times do
+          return false unless group_alive?(pid)
+
+          pause(EXITED_GROUP_POLL_INTERVAL)
+        end
+        group_alive?(pid)
+      end
+
+      # Signal 0 probes existence only: ESRCH means the group is empty.
+      def group_alive?(pid)
+        Process.kill(0, -pid)
+        true
+      rescue Errno::ESRCH
+        false
+      rescue Errno::EPERM
+        true
+      end
 
       def pause(seconds)
         sleep(seconds)
@@ -133,6 +192,10 @@ module Henitai
         { "PARALLEL_WORKERS" => "1" }
       end
 
+      def suite_process_env
+        subprocess_env.merge(SuiteBootClock::STARTED_AT_ENV => Time.now.to_f.to_s)
+      end
+
       def wait_nonblocking(pid)
         Process.wait(pid, Process::WNOHANG)
       rescue Errno::ECHILD, Errno::ESRCH
@@ -143,6 +206,10 @@ module Henitai
         @scenario_log_support ||= ScenarioLogSupport.new
       end
 
+      def child_report_store
+        @child_report_store ||= ChildReportStore.new
+      end
+
       def with_subprocess_env
         original_env = {} # : Hash[String, String?]
         subprocess_env.each do |key, value|