evilution 0.32.0 → 0.34.0

data/lib/evilution/isolation/fork.rb

@@ -5,7 +5,7 @@ require "tmpdir"
 require_relative "../memory"
 require_relative "../temp_dir_tracker"
 require_relative "../child_output"
-require_relative "../process_cleanup"
+require_relative "../process_supervisor"
 
 require_relative "../isolation"
 
@@ -15,21 +15,25 @@ class Evilution::Isolation::Fork
 
   def initialize(hooks: nil)
     @hooks = hooks
+    # EV-3aw3 / EV-5rrh step 2: the supervisor owns this path's lifecycle --
+    # spawn + process-group isolation, the TERM/grace/KILL ladder, and reap +
+    # sandbox removal. fork.rb keeps only the marshal-pipe read protocol.
+    @supervisor = Evilution::ProcessSupervisor.new
   end
 
   def call(mutation:, test_command:, timeout:)
-    pid = nil
+    handle = nil
     sandbox_dir = Dir.mktmpdir("evilution-run")
     start_time = Process.clock_gettime(Process::CLOCK_MONOTONIC)
     parent_rss = Evilution::Memory.rss_kb
     read_io, write_io = binary_pipe
-    pid = fork_child(read_io, write_io, sandbox_dir, mutation, test_command)
+    handle = spawn_child(read_io, write_io, sandbox_dir, mutation, test_command)
     write_io.close
-    result = wait_for_result(pid, read_io, timeout)
+    result = wait_for_result(handle, read_io, timeout)
     duration = Process.clock_gettime(Process::CLOCK_MONOTONIC) - start_time
     build_mutation_result(mutation, result, duration, parent_rss)
   ensure
-    cleanup_resources(read_io, write_io, pid, sandbox_dir)
+    cleanup_resources(read_io, write_io, handle, sandbox_dir)
   end
 
   private
@@ -46,13 +50,18 @@ class Evilution::Isolation::Fork
     [read_io, write_io]
   end
 
-  def fork_child(read_io, write_io, sandbox_dir, mutation, test_command)
-    ::Process.fork do
+  # Supervisor.spawn makes the child its own process-group leader (setpgid)
+  # before this block runs, so any grandchildren test_command forks inherit the
+  # group and the TERM/KILL ladder sweeps the whole subtree on timeout (EV-2sh8
+  # / GH #1330). The block keeps the marshal-pipe protocol: write a
+  # length-prefixed payload, then exit with the pass/fail code.
+  def spawn_child(read_io, write_io, sandbox_dir, mutation, test_command)
+    @supervisor.spawn(sandbox_dir: sandbox_dir) do
       ENV["TMPDIR"] = sandbox_dir
       # Path-relativizing mutations (e.g. File.join(dir, name) -> name) would
       # otherwise write into the parent's CWD (typically the repo root) and
       # leak past the run. chdir here keeps such writes inside sandbox_dir,
-      # which the ensure block of #call removes. The in_isolated_worker! flag
+      # which the supervisor removes on reap. The in_isolated_worker! flag
       # signals the rest of evilution (SpecResolver/SpecSelector/SpecAstCache/
       # MutationApplier/SourceEvaluator/Integration) to anchor project-relative
       # paths to Evilution::PROJECT_ROOT instead of the sandbox CWD.
@@ -70,12 +79,20 @@ class Evilution::Isolation::Fork
     end
   end
 
-  def cleanup_resources(read_io, write_io, pid, sandbox_dir)
+  # The parent owns read_io/write_io (write_io is closed right after spawn so
+  # read_io can see EOF), so they are closed here rather than handed to the
+  # supervisor. The supervisor reaps the child and removes the sandbox dir; on
+  # the early-failure path (binary_pipe raised before spawn) handle is nil, so
+  # the orphaned sandbox is removed directly.
+  def cleanup_resources(read_io, write_io, handle, sandbox_dir)
     read_io.close unless read_io.nil?
     write_io.close unless write_io.nil?
-    ensure_reaped(pid)
+    if handle
+      @supervisor.terminate(handle, grace: GRACE_PERIOD)
+    elsif sandbox_dir
+      FileUtils.rm_rf(sandbox_dir)
+    end
     restore_original_source
-    FileUtils.rm_rf(sandbox_dir) if sandbox_dir
   end
 
   def restore_original_source
@@ -109,21 +126,21 @@ class Evilution::Isolation::Fork
   # never sees EOF and hangs forever. The length prefix makes payload reads
   # bounded; the waitpid-WNOHANG check inside the poll loop lets us exit
   # promptly when the child died without writing anything.
-  def wait_for_result(pid, read_io, timeout)
+  def wait_for_result(handle, read_io, timeout)
     deadline = Process.clock_gettime(Process::CLOCK_MONOTONIC) + timeout
     loop do
       remaining = deadline - Process.clock_gettime(Process::CLOCK_MONOTONIC)
-      return timeout_result(pid) if remaining <= 0
+      return timeout_result(handle) if remaining <= 0
 
       if read_io.wait_readable([remaining, 0.5].min)
         payload = read_payload(read_io, deadline)
-        return reap_and_decode(pid, payload) if payload
+        return reap_and_decode(handle, payload) if payload
       end
 
-      next unless ::Process.waitpid(pid, ::Process::WNOHANG)
+      next unless @supervisor.reap_nonblock(handle)
 
       # Child exited. Drain any final payload that arrived between
-      # wait_readable timeout and waitpid (race) before declaring empty.
+      # wait_readable timeout and the reap (race) before declaring empty.
       final = read_payload(read_io, Process.clock_gettime(Process::CLOCK_MONOTONIC) + 0.1)
       return decode_payload(final) if final
 
@@ -137,13 +154,13 @@ class Evilution::Isolation::Fork
   # in execute_in_child waiting on a subject grandchild the mutation broke.
   # wait_for_result has already returned by this point, so the per-mutation
   # timeout cannot fire. Bound the wait and fall back to the TERM/KILL ladder.
-  def reap_and_decode(pid, payload)
+  def reap_and_decode(handle, payload)
     deadline = Process.clock_gettime(Process::CLOCK_MONOTONIC) + REAP_DEADLINE
     loop do
-      break if ::Process.waitpid(pid, ::Process::WNOHANG)
+      break if @supervisor.reap_nonblock(handle)
 
       if Process.clock_gettime(Process::CLOCK_MONOTONIC) >= deadline
-        terminate_child(pid)
+        @supervisor.terminate(handle, grace: GRACE_PERIOD)
         break
       end
       sleep 0.05
@@ -194,40 +211,11 @@ class Evilution::Isolation::Fork
     { timeout: false, passed: false, error: "empty result from child" }
   end
 
-  def timeout_result(pid)
-    terminate_child(pid)
+  def timeout_result(handle)
+    @supervisor.terminate(handle, grace: GRACE_PERIOD)
     { timeout: true }
   end
 
-  # Defensive reap: if normal control flow raised before wait_for_result
-  # reaped the child (e.g. Marshal.load on corrupt payload), the child becomes
-  # a zombie. Reuse terminate_child for the bounded TERM + GRACE_PERIOD + KILL
-  # ladder so this never hangs the ensure path; swallow SystemCallError so
-  # cleanup can't mask the primary failure.
-  def ensure_reaped(pid)
-    return unless pid
-
-    reaped = ::Process.waitpid(pid, ::Process::WNOHANG)
-    return if reaped
-
-    terminate_child(pid)
-  rescue SystemCallError
-    nil
-  end
-
-  def terminate_child(pid)
-    Evilution::ProcessCleanup.safe_kill("TERM", pid)
-    _, status = ::Process.waitpid2(pid, ::Process::WNOHANG)
-    return if status
-
-    sleep(GRACE_PERIOD)
-    _, status = ::Process.waitpid2(pid, ::Process::WNOHANG)
-    return if status
-
-    Evilution::ProcessCleanup.safe_kill("KILL", pid)
-    Evilution::ProcessCleanup.safe_wait(pid)
-  end
-
   def classify_status(result)
     return :timeout if result[:timeout]
     return :killed if result[:test_crashed]

data/lib/evilution/parallel/work_queue/dispatcher/deadline_tracker.rb

@@ -0,0 +1,63 @@
+# frozen_string_literal: true
+
+require_relative "../dispatcher"
+
+# Owns the per-worker item-timeout deadline clock for the Dispatcher: arming a
+# worker's deadline when it goes busy, re-arming it on each result, surfacing the
+# workers whose deadline has passed, and computing how long IO.select may block.
+# Each worker carries its own deadline so a single stuck worker is reaped in
+# isolation rather than aborting the whole pool (EV-gl1e). Pulling this cohesive
+# timeout concern out of the Dispatcher keeps the dispatcher focused on the
+# collect/recycle orchestration (EV-9mij).
+#
+# `workers` is the Dispatcher's live array (mutated in place as workers recycle),
+# so the tracker always reads the current pool. `clock` is injectable for tests.
+class Evilution::Parallel::WorkQueue::Dispatcher::DeadlineTracker
+  def initialize(item_timeout:, workers:, clock: -> { Process.clock_gettime(Process::CLOCK_MONOTONIC) })
+    @item_timeout = item_timeout
+    @workers = workers
+    @clock = clock
+  end
+
+  def enabled?
+    !@item_timeout.nil?
+  end
+
+  # Seconds IO.select may block: until the nearest worker deadline (never
+  # negative), or the raw timeout when no worker is currently on the clock.
+  def select_timeout
+    return @item_timeout unless enabled?
+
+    deadlines = @workers.filter_map(&:deadline)
+    return @item_timeout if deadlines.empty?
+
+    [deadlines.min - now, 0].max
+  end
+
+  # Workers whose deadline has passed while still holding in-flight work.
+  def overdue
+    return [] unless enabled?
+
+    moment = now
+    @workers.select { |worker| worker.deadline && worker.deadline <= moment && worker.pending.positive? }
+  end
+
+  # Arm a worker's clock when it first goes busy; idempotent for the in-flight
+  # item so a refresh does not extend an already-running deadline.
+  def start(worker)
+    return unless enabled?
+
+    worker.deadline ||= now + @item_timeout
+  end
+
+  # After a result: re-arm while work remains, otherwise stop the clock.
+  def refresh(worker)
+    worker.deadline = (now + @item_timeout if enabled? && worker.pending.positive?)
+  end
+
+  private
+
+  def now
+    @clock.call
+  end
+end

data/lib/evilution/parallel/work_queue/dispatcher.rb

@@ -12,11 +12,11 @@ class Evilution::Parallel::WorkQueue::Dispatcher
     @workers = workers
     @items = items
     @prefetch = prefetch
-    @item_timeout = item_timeout
     @worker_max_items = worker_max_items
     @recycle_factory = recycle_factory
     @state = Evilution::Parallel::WorkQueue.send(:const_get, :CollectionState).new(items.length)
     @retired = []
+    @deadlines = DeadlineTracker.new(item_timeout:, workers: @workers)
   end
 
   def run
@@ -38,24 +38,34 @@ class Evilution::Parallel::WorkQueue::Dispatcher
     end
   end
 
+  # Each worker carries its own deadline (set when it goes busy, refreshed on
+  # every result). The select blocks only until the nearest worker deadline,
+  # so a single stuck worker is reaped in isolation -- its in-flight item gets
+  # the WorkQueue::TIMED_OUT sentinel and the worker is recycled -- instead of
+  # the old pool-wide watchdog that SIGKILLed every worker and aborted the run.
   def collect
     io_to_worker = @workers.to_h { |w| [w.res_io, w] }
     result_ios = io_to_worker.keys
 
     while @state.in_flight.positive?
-      readable, = IO.select(result_ios, nil, nil, @item_timeout)
-      if readable.nil?
-        record_timeout
-        break
-      end
+      readable, = IO.select(result_ios, nil, nil, @deadlines.select_timeout)
+      reap_timed_out(io_to_worker, result_ios)
+      next if readable.nil?
 
-      readable.each { |io| process_readable(io, io_to_worker, result_ios) }
+      readable.each do |io|
+        process_readable(io, io_to_worker, result_ios) if result_ios.include?(io)
+      end
     end
   end
 
-  def record_timeout
-    terminate_stuck
-    @state.first_error ||= Evilution::Error.new("worker timed out after #{@item_timeout}s")
+  def reap_timed_out(io_to_worker, result_ios)
+    @deadlines.overdue.each { |worker| time_out_worker(worker, io_to_worker, result_ios) }
+  end
+
+  def time_out_worker(worker, io_to_worker, result_ios)
+    worker.kill
+    mark_unfinished(worker, Evilution::Parallel::WorkQueue::TIMED_OUT)
+    retire_or_replace(worker, io_to_worker, result_ios)
   end
 
   def process_readable(io, io_to_worker, result_ios)
@@ -65,7 +75,7 @@ class Evilution::Parallel::WorkQueue::Dispatcher
 
   def handle(worker, io_to_worker, result_ios)
     message = worker.read_result
-    return handle_dead(worker) if message.nil?
+    return handle_dead(worker, io_to_worker, result_ios) if message.nil?
 
     record(message, worker)
     return false if recycle_and_dispatch(worker, io_to_worker, result_ios)
@@ -82,13 +92,24 @@ class Evilution::Parallel::WorkQueue::Dispatcher
     @state.in_flight -= 1
     worker.pending -= 1
     worker.items_completed += 1
+    worker.in_flight_indices.delete(index)
+    @deadlines.refresh(worker)
+  end
+
+  # A worker that exited without replying loses only its in-flight item(s)
+  # (marked :died) and is recycled; the run continues rather than aborting.
+  def handle_dead(worker, io_to_worker, result_ios)
+    mark_unfinished(worker, Evilution::Parallel::WorkQueue::DIED)
+    retire_or_replace(worker, io_to_worker, result_ios)
+    false
   end
 
-  def handle_dead(worker)
-    @state.first_error ||= Evilution::Error.new("worker process exited unexpectedly")
+  def mark_unfinished(worker, sentinel)
+    worker.in_flight_indices.each { |index| @state.results[index] = sentinel }
     @state.in_flight -= worker.pending
     worker.pending = 0
-    false
+    worker.in_flight_indices.clear
+    worker.deadline = nil
   end
 
   def draining_for_recycle?(worker)
@@ -113,28 +134,52 @@ class Evilution::Parallel::WorkQueue::Dispatcher
   end
 
   def recycle(old_worker, io_to_worker, result_ios)
-    io_to_worker.delete(old_worker.res_io)
-    result_ios.delete(old_worker.res_io)
-    @retired << old_worker.retire
-
+    index = @workers.index(old_worker)
+    detach(old_worker, io_to_worker, result_ios)
     new_worker = @recycle_factory.call(old_worker)
-    @workers[@workers.index(old_worker)] = new_worker
-    io_to_worker[new_worker.res_io] = new_worker
-    result_ios << new_worker.res_io
+    @workers[index] = new_worker
+    attach(new_worker, io_to_worker, result_ios)
     new_worker
   end
 
+  # Shared failure-path recovery: retire the worker, and as long as work
+  # remains spin up a replacement to keep the pool full and hand it the next
+  # item. When the queue is already drained, just drop the worker.
+  def retire_or_replace(worker, io_to_worker, result_ios)
+    index = @workers.index(worker)
+    detach(worker, io_to_worker, result_ios)
+
+    if more_to_send? && @state.first_error.nil?
+      new_worker = @recycle_factory.call(worker)
+      @workers[index] = new_worker
+      attach(new_worker, io_to_worker, result_ios)
+      send_item(new_worker)
+    else
+      @workers.delete_at(index)
+    end
+  end
+
+  def detach(worker, io_to_worker, result_ios)
+    io_to_worker.delete(worker.res_io)
+    result_ios.delete(worker.res_io)
+    @retired << worker.retire
+  end
+
+  def attach(worker, io_to_worker, result_ios)
+    io_to_worker[worker.res_io] = worker
+    result_ios << worker.res_io
+  end
+
   def send_item(worker)
     worker.send_item(@state.next_index, @items[@state.next_index])
     @state.next_index += 1
     @state.in_flight += 1
+    @deadlines.start(worker)
   end
 
   def more_to_send?
     @state.next_index < @items.length
   end
-
-  def terminate_stuck
-    @workers.each(&:kill)
-  end
 end
+
+require_relative "dispatcher/deadline_tracker"

data/lib/evilution/parallel/work_queue/worker.rb

@@ -2,23 +2,34 @@
 
 require_relative "../work_queue"
 require_relative "../../child_output"
+require_relative "../../process_supervisor"
+require_relative "../../temp_dir_tracker"
 require_relative "channel"
 require_relative "channel/frame"
 
 class Evilution::Parallel::WorkQueue::Worker
   Timing = Data.define(:busy, :wall)
 
-  attr_reader :pid, :worker_index
-  attr_accessor :items_completed, :pending, :busy_time, :wall_time
-
-  def self.spawn(worker_index:, hooks:, &block)
+  attr_reader :pid, :worker_index, :in_flight_indices
+  attr_accessor :items_completed, :pending, :busy_time, :wall_time, :deadline
+
+  # EV-dg69 / EV-5rrh step 3: the supervisor owns the worker's process-group
+  # isolation, signal-safe registry, group-kill and reap. spawn passes
+  # isolate_in_child: false so the worker becomes its own group leader only
+  # parent-side, AFTER the supervisor has registered it -- preserving the
+  # EV-jwao register-before-isolate ordering (the trap can never see a leader
+  # missing from the registry). EV-cnx8 group-leadership (so #kill sweeps the
+  # whole subtree) is still established, now by the supervisor's parent-side
+  # setpgid.
+  def self.spawn(worker_index:, hooks:, supervisor: Evilution::ProcessSupervisor.new, &block)
     cmd_read, cmd_write = IO.pipe
     res_read, res_write = IO.pipe
     [cmd_read, cmd_write, res_read, res_write].each(&:binmode)
 
-    pid = Process.fork do
+    handle = supervisor.spawn(isolate_in_child: false) do
       cmd_write.close
       res_read.close
+      install_child_signal_handlers
       ENV["TEST_ENV_NUMBER"] = test_env_number_for(worker_index)
       Evilution::ChildOutput.redirect!
       Loop.run(cmd_read, res_write, hooks: hooks, &block)
@@ -26,7 +37,25 @@ class Evilution::Parallel::WorkQueue::Worker
 
     cmd_read.close
     res_write.close
-    new(pid: pid, cmd_write: cmd_write, res_read: res_read, worker_index: worker_index)
+    new(handle:, supervisor:, cmd_write:, res_read:, worker_index:)
+  end
+
+  # EV-7a91: a worker is the parent of the inner per-mutation Fork children it
+  # spawns, and those children are their own process-group leaders (EV-2sh8), so
+  # the Runner's group-kill of the worker never reaches them. On a terminal
+  # INT/TERM the worker must therefore tear down AND reap the inner children it
+  # owns before it dies, or they survive as zombies (their parent gone) until an
+  # ancestor exits. cleanup_all clears any per-mutation sandbox dirs the inner
+  # children registered in this worker's TempDirTracker.
+  def self.install_child_signal_handlers
+    %w[INT TERM].each do |sig|
+      Signal.trap(sig) do
+        Evilution::TempDirTracker.cleanup_all
+        Evilution::ProcessSupervisor.kill_and_reap_all
+        Signal.trap(sig, "DEFAULT")
+        Process.kill(sig, Process.pid)
+      end
+    end
   end
 
   # EV-kdns / GH #817: translate 0-based worker slot to parallel_tests'
@@ -37,8 +66,10 @@ class Evilution::Parallel::WorkQueue::Worker
     worker_index.zero? ? "" : (worker_index + 1).to_s
   end
 
-  def initialize(pid:, cmd_write:, res_read:, worker_index:)
-    @pid = pid
+  def initialize(handle:, supervisor:, cmd_write:, res_read:, worker_index:)
+    @handle = handle
+    @supervisor = supervisor
+    @pid = handle.pid
     @cmd_write = cmd_write
     @res_read = res_read
     @worker_index = worker_index
@@ -46,6 +77,8 @@ class Evilution::Parallel::WorkQueue::Worker
     @pending = 0
     @busy_time = 0.0
     @wall_time = 0.0
+    @in_flight_indices = []
+    @deadline = nil
   end
 
   def res_io
@@ -55,6 +88,7 @@ class Evilution::Parallel::WorkQueue::Worker
   def send_item(index, item)
     Evilution::Parallel::WorkQueue::Channel.write(@cmd_write, [index, item])
     @pending += 1
+    @in_flight_indices << index
   end
 
   def read_result
@@ -67,10 +101,11 @@ class Evilution::Parallel::WorkQueue::Worker
     nil
   end
 
+  # SIGKILL the worker's whole process group (negative pid), reaping any
+  # grandchildren it forked, with the bare pid as a fallback for the case where
+  # the group is gone (already reaped, or setpgid did not take).
   def kill
-    Process.kill("KILL", @pid)
-  rescue Errno::ESRCH
-    nil
+    @supervisor.signal_group("KILL", @handle)
   end
 
   def close_pipes
@@ -78,10 +113,11 @@ class Evilution::Parallel::WorkQueue::Worker
     @res_read.close unless @res_read.closed?
   end
 
+  # Reap the leader and drop it from the registry so the trap never signals a
+  # group whose pid the OS may have recycled. ECHILD-tolerant; unregister is a
+  # no-op if it was never registered.
   def reap
-    Process.wait(@pid)
-  rescue Errno::ECHILD
-    nil
+    @supervisor.reap(@handle)
   end
 
   def retire

data/lib/evilution/parallel/work_queue.rb

@@ -9,6 +9,14 @@ class Evilution::Parallel::WorkQueue
 
   TIMING_GRACE_PERIOD = 5
 
+  # Sentinel results for items whose worker never produced a value. The
+  # dispatcher writes these into the results array (instead of aborting the
+  # whole run) so a single stuck/dead worker only loses its own in-flight
+  # item(s). Mutation-aware callers translate the reason into a status.
+  Unfinished = Data.define(:reason)
+  TIMED_OUT = Unfinished.new(reason: :timeout)
+  DIED = Unfinished.new(reason: :died)
+
   def initialize(size:, hooks: nil, prefetch: 1, item_timeout: nil, worker_max_items: nil)
     Validators::PositiveInt.call!(:size, size)
     Validators::PositiveInt.call!(:prefetch, prefetch)