henitai 0.2.0 → 0.3.0

data/lib/henitai/slot_scheduler/draining.rb

@@ -0,0 +1,146 @@
+# frozen_string_literal: true
+
+module Henitai
+  class SlotScheduler
+    # Drain/timeout state machine for in-flight slots.
+    #
+    # A slot enters the draining state either when it exceeds its timeout
+    # ({#check_timeouts}) or when a shutdown is requested
+    # ({#interrupt_active_slots}). {#drain_draining_slots} then performs the
+    # two-phase SIGTERM/SIGKILL broadcast and the final blocking reap.
+    #
+    # Mixed into {SlotScheduler}; relies on its slot table, +integration+,
+    # +progress_reporter+, +wakeup+ and the {ProcessControl} primitives.
+    module Draining
+      # Per-slot timeout check. Must be called after reap_all_completed_children
+      # so that naturally-exited processes are already removed from slots.
+      def check_timeouts
+        now = monotonic_time
+        slots.each_value do |slot|
+          next if slot.draining
+          next unless now >= slot.started_at_monotonic + slot.timeout
+
+          # Final targeted reap: if the child already exited, classify it normally.
+          pid, status = wnohang_reap(slot.pid)
+          if pid
+            complete_slot(pid, status)
+          else
+            slot.forced_outcome = :timeout
+            slot.draining = true
+          end
+        end
+      end
+
+      def draining_slots?
+        slots.any? { |_, slot| slot.draining }
+      end
+
+      # Two-phase broadcast cleanup for all slots that are in draining state.
+      #
+      # Precision rule: before signalling, do one final WNOHANG pass to catch
+      # processes that exited naturally in the window between check_timeouts and
+      # now. If SIGTERM gets ESRCH, the process is already gone — we must not
+      # force-label those as :timeout.
+      def drain_draining_slots
+        draining = draining_slots
+        return if draining.empty?
+
+        prune_raced_draining_slots(draining)
+
+        return if draining.empty?
+
+        broadcast_term(draining)
+        wait_for_drain_window
+        signal_draining_slots(draining)
+        reap_and_remove_draining(draining)
+      end
+
+      def interrupt_active_slots
+        slots.each_value do |slot|
+          next if slot.draining
+
+          slot.forced_outcome = :interrupted
+          slot.draining = true
+        end
+      end
+
+      private
+
+      def draining_slots
+        slots.select { |_, slot| slot.draining }
+      end
+
+      def prune_raced_draining_slots(draining)
+        draining.reject! do |_, slot|
+          pid, status = wnohang_reap(slot.pid)
+          next false unless pid
+
+          complete_slot(pid, status)
+          true
+        end
+      end
+
+      def wait_for_drain_window
+        wakeup&.wait(PROCESS_DRAIN_WINDOW)
+        wakeup&.drain
+      end
+
+      def signal_draining_slots(draining)
+        draining.each_value { |slot| signal_process_group(slot.pid, :SIGKILL) }
+      end
+
+      def broadcast_term(draining)
+        now = monotonic_time
+        draining.each_value do |slot|
+          slot.term_sent_at_monotonic = now
+          signal_process_group(slot.pid, :SIGTERM)
+        end
+      end
+
+      # After SIGKILL window: blocking reap each slot, then build its result.
+      #
+      # Interrupted slots are cleaned up but produce no result — the scheduler
+      # is shutting down and does not emit verdicts for in-flight mutants.
+      #
+      # For timeout slots: a real exit status only wins if observed before any
+      # parent signal was sent. Once SIGTERM has been dispatched, the forced
+      # outcome is authoritative — a child handling SIGTERM and exiting 0 must
+      # not be misclassified as :survived.
+      def reap_and_remove_draining(draining)
+        draining.each_value { |slot| reap_and_finalize_slot(slot) }
+      end
+
+      # One last WNOHANG before blocking: catches processes that exited
+      # between SIGKILL and here.
+      def reap_and_finalize_slot(slot)
+        _, final_status = wnohang_reap(slot.pid)
+        reap_pid(slot.pid) unless final_status
+
+        pid_to_slot.delete(slot.pid)
+        slots.delete(slot.slot_id)
+        Integration::SchedulerDiagnostics.child_ended(slot.pid)
+
+        return if slot.forced_outcome == :interrupted
+
+        record_drain_result(slot, final_status)
+      end
+
+      def record_drain_result(slot, final_status)
+        result = build_drain_result(slot, final_status)
+        slot.mutant.status = result.status
+        results << result
+        progress_reporter&.progress(slot.mutant, scenario_result: result)
+      end
+
+      # Choose result: use real exit status only if observed before any parent
+      # signal was sent. After SIGTERM, the forced outcome is authoritative.
+      def build_drain_result(slot, final_status)
+        if final_status&.exited? && slot.term_sent_at_monotonic.nil?
+          integration.build_result(final_status, slot.log_paths)
+        else
+          integration.build_result(slot.forced_outcome || :timeout, slot.log_paths)
+        end
+      end
+    end
+  end
+end

data/lib/henitai/slot_scheduler/process_control.rb

@@ -0,0 +1,43 @@
+# frozen_string_literal: true
+
+module Henitai
+  class SlotScheduler
+    # Low-level bridge to the OS process and signal primitives.
+    #
+    # Every Process.wait*/kill call routes through +runtime+ so that the
+    # scheduler remains the single caller of the process table. Mixed into
+    # {SlotScheduler}; relies on its +runtime+ reader.
+    module ProcessControl
+      private
+
+      def monotonic_time
+        runtime.clock_gettime(Process::CLOCK_MONOTONIC)
+      end
+
+      def wnohang_reap(pid)
+        runtime.wait2(pid, Process::WNOHANG)
+      rescue Errno::ECHILD, Errno::ESRCH
+        nil
+      end
+
+      def signal_process_group(pid, signal)
+        runtime.kill(signal, -pid)
+      rescue Errno::ESRCH
+        nil
+      rescue Errno::EPERM
+        # Process group not yet established; fall back to signalling the pid.
+        begin
+          runtime.kill(signal, pid)
+        rescue Errno::ESRCH
+          nil
+        end
+      end
+
+      def reap_pid(pid)
+        runtime.wait(pid)
+      rescue Errno::ECHILD, Errno::ESRCH
+        nil
+      end
+    end
+  end
+end

data/lib/henitai/slot_scheduler.rb

@@ -0,0 +1,272 @@
+# frozen_string_literal: true
+
+require_relative "slot_scheduler/process_control"
+require_relative "slot_scheduler/draining"
+
+module Henitai
+  # Owns the process-slot table for a single parallel mutation run.
+  #
+  # {ProcessWorkerRunner} drives the event loop and OS signal handling and
+  # delegates every slot operation here: filling idle slots, reaping completed
+  # children, retrying flaky survivors, detecting timeouts and running the
+  # drain/broadcast state machine. Keeping the table behind one collaborator
+  # means Process.wait* has a single caller, so there are no races between
+  # threads reaping the same child.
+  #
+  # The drain/timeout state machine lives in {Draining}; the low-level process
+  # and signal primitives live in {ProcessControl}. +host+ is the owning
+  # {ProcessWorkerRunner}, which supplies +runtime+, +wakeup+, +worker_count+
+  # and the shutdown flag.
+  class SlotScheduler
+    include ProcessControl
+    include Draining
+
+    PROCESS_DRAIN_WINDOW = 0.2
+
+    # Environment variable exposing a stable worker-slot index (0..jobs-1) to
+    # each forked child so test suites can isolate shared resources per slot
+    # (e.g. "myapp_test_#{ENV['HENITAI_WORKER_SLOT']}"). A flaky-retry respawn
+    # keeps the original attempt's value.
+    WORKER_SLOT_ENV = "HENITAI_WORKER_SLOT"
+
+    # Tracks one in-flight mutant child process.
+    Slot = Struct.new(
+      :slot_id, :mutant, :pid, :started_at_monotonic, :timeout,
+      :log_paths, :retry_count, :draining, :term_sent_at_monotonic,
+      :forced_outcome, :worker_index
+    )
+
+    # @return [Integer] mutants that required at least one retry during the run.
+    # @return [Array<ScenarioExecutionResult>] verdicts accumulated so far.
+    attr_reader :flaky_retry_count, :results
+
+    def initialize(integration:, config:, progress_reporter:, options:, host:)
+      @integration = integration
+      @config = config
+      @progress_reporter = progress_reporter
+      @options = options
+      @host = host
+      @pending = []
+      @slots = {}
+      @pid_to_slot = {}
+      @results = []
+      @flaky_retry_count = 0
+      @next_slot_id = 0
+    end
+
+    # Queues the mutants to be scheduled into worker slots.
+    def enqueue(mutants)
+      @pending = mutants.dup
+    end
+
+    def done?
+      pending.empty? && slots.empty?
+    end
+
+    def fill_idle_slots
+      while slots.size < worker_count && !pending.empty?
+        mutant = pending.shift
+        spawn_into_slot(mutant)
+      end
+    end
+
+    def reap_all_completed_children
+      loop do
+        pid, status = runtime.wait2(-1, Process::WNOHANG)
+        break unless pid
+
+        complete_slot(pid, status)
+      end
+    rescue Errno::ECHILD
+      nil
+    end
+
+    def next_event_timeout
+      now = monotonic_time
+      slot_timeouts = slots.each_value.filter_map do |slot|
+        remaining_slot_timeout(slot, now)
+      end
+
+      slot_timeouts.min
+    end
+
+    private
+
+    attr_reader :pending, :slots, :pid_to_slot, :integration, :config,
+                :progress_reporter, :options, :host
+
+    def worker_count = host.worker_count
+    def runtime = host.runtime
+    def wakeup = host.wakeup
+    def shutdown? = host.shutdown_requested?
+
+    def spawn_into_slot(mutant)
+      test_files = resolve_test_files(mutant)
+      mutant.covered_by = test_files if mutant.respond_to?(:covered_by=)
+      mutant.tests_completed = test_files.size if mutant.respond_to?(:tests_completed=)
+      return record_no_coverage(mutant) if resolved_selection_empty?(test_files)
+
+      worker_index = next_free_worker_index
+      with_worker_slot(worker_index) do
+        handle = integration.spawn_mutant(mutant: mutant, test_files: test_files)
+        register_slot(handle, mutant, worker_index, slot_timeout(mutant, test_files))
+      end
+    rescue StandardError => e
+      record_spawn_failure(mutant, e)
+    end
+
+    def register_slot(handle, mutant, worker_index, timeout)
+      slot_id = next_slot_id!
+      slot = build_slot(slot_id, mutant, handle, worker_index, timeout)
+      slots[slot_id] = slot
+      pid_to_slot[handle.pid] = slot_id
+      Integration::SchedulerDiagnostics.child_started(handle.pid)
+    end
+
+    def build_slot(slot_id, mutant, handle, worker_index, timeout)
+      Slot.new(
+        slot_id, mutant, handle.pid,
+        monotonic_time,
+        timeout, handle.log_paths, 0, false, nil, nil,
+        worker_index
+      )
+    end
+
+    def slot_timeout(mutant, test_files)
+      resolver = options[:timeout_resolver]
+      resolver ? resolver.call(mutant, test_files) : config.timeout
+    end
+
+    # Smallest index in 0...worker_count not held by a live slot, so
+    # concurrently-running children always see distinct values and freed
+    # indices are reused. Slot ids themselves grow monotonically and are
+    # unsuitable as a resource token.
+    def next_free_worker_index
+      used = slots.each_value.map(&:worker_index)
+      (0...worker_count).find { |index| !used.include?(index) } || used.size
+    end
+
+    def complete_slot(pid, wait_result)
+      slot_id = pid_to_slot.delete(pid)
+      return unless slot_id
+
+      slot = slots[slot_id]
+      return unless slot
+
+      Integration::SchedulerDiagnostics.child_ended(pid)
+      result = integration.build_result(wait_result, slot.log_paths)
+      dispatch_slot_result(slot, result)
+    end
+
+    def dispatch_slot_result(slot, result)
+      if should_retry?(slot, result)
+        retry_slot(slot)
+      else
+        slots.delete(slot.slot_id)
+        slot.mutant.status = result.status
+        results << result
+        progress_reporter&.progress(slot.mutant, scenario_result: result)
+        result.release_output! if result.respond_to?(:release_output!)
+      end
+    end
+
+    def should_retry?(slot, result)
+      !shutdown? && result.survived? && slot.retry_count < config.max_flaky_retries.to_i
+    end
+
+    def retry_slot(slot)
+      test_files = resolve_test_files(slot.mutant)
+      with_worker_slot(slot.worker_index) do
+        handle = integration.spawn_mutant(mutant: slot.mutant, test_files: test_files)
+        finish_retry(slot, handle)
+      end
+    rescue StandardError => e
+      slots.delete(slot.slot_id)
+      record_spawn_failure(slot.mutant, e)
+    end
+
+    def finish_retry(slot, handle)
+      @flaky_retry_count += 1 if slot.retry_count.zero?
+      slot.retry_count += 1
+      reset_slot_for_retry(slot, handle)
+      pid_to_slot[handle.pid] = slot.slot_id
+      Integration::SchedulerDiagnostics.child_started(handle.pid)
+    end
+
+    def reset_slot_for_retry(slot, handle)
+      slot.pid = handle.pid
+      slot.log_paths = handle.log_paths
+      slot.started_at_monotonic = monotonic_time
+      slot.draining = false
+      slot.term_sent_at_monotonic = nil
+      slot.forced_outcome = nil
+    end
+
+    def record_spawn_failure(mutant, error)
+      result = ScenarioExecutionResult.new(
+        status: :compile_error,
+        stdout: "",
+        stderr: "spawn failed: #{error.message}",
+        log_path: "/dev/null",
+        exit_status: nil
+      )
+      mutant.status = result.status
+      results << result
+      progress_reporter&.progress(mutant, scenario_result: result)
+    end
+
+    def record_no_coverage(mutant)
+      mutant.status = :no_coverage
+      progress_reporter&.progress(mutant, scenario_result: nil)
+    end
+
+    def resolved_selection_empty?(test_files)
+      options.key?(:test_file_resolver) && test_files.empty?
+    end
+
+    def remaining_slot_timeout(slot, now)
+      # Invariant: drain_draining_slots runs (and removes draining slots) before
+      # the event wait, so next_event_timeout never observes a draining slot
+      # whose SIGTERM has not been sent. Guard term_sent_at_monotonic defensively
+      # against a future ordering change: an unsignalled draining slot is due now.
+      return 0.0 if slot.draining && slot.term_sent_at_monotonic.nil?
+
+      deadline =
+        if slot.draining
+          slot.term_sent_at_monotonic + PROCESS_DRAIN_WINDOW
+        else
+          slot.started_at_monotonic + slot.timeout
+        end
+      remaining = deadline - now
+      remaining.positive? ? remaining : 0.0
+    end
+
+    def resolve_test_files(mutant)
+      if @options.key?(:test_file_resolver)
+        @options[:test_file_resolver].call(mutant)
+      elsif @options.key?(:test_files)
+        @options[:test_files]
+      else
+        integration.select_tests(mutant.subject)
+      end
+    end
+
+    def next_slot_id!
+      id = @next_slot_id
+      @next_slot_id += 1
+      id
+    end
+
+    def with_worker_slot(worker_index)
+      previous = ENV.fetch(WORKER_SLOT_ENV, nil)
+      ENV[WORKER_SLOT_ENV] = worker_index.to_s
+      yield
+    ensure
+      if previous.nil?
+        ENV.delete(WORKER_SLOT_ENV)
+      else
+        ENV[WORKER_SLOT_ENV] = previous
+      end
+    end
+  end
+end

data/lib/henitai/static_filter.rb

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 
 require_relative "coverage_report_reader"
+require_relative "mutation_skip_directives"
 
 module Henitai
   # Applies static, pre-execution filtering to generated mutants.
@@ -8,8 +9,10 @@ module Henitai
     DEFAULT_COVERAGE_REPORT_PATH = CoverageReportReader::DEFAULT_COVERAGE_REPORT_PATH
     DEFAULT_PER_TEST_COVERAGE_REPORT_PATH = CoverageReportReader::DEFAULT_PER_TEST_COVERAGE_REPORT_PATH
 
-    def initialize(coverage_report_reader: CoverageReportReader.new)
+    def initialize(coverage_report_reader: CoverageReportReader.new,
+                   skip_directives: MutationSkipDirectives.new)
       @coverage_report_reader = coverage_report_reader
+      @skip_directives = skip_directives
     end
 
     # This method is the gate-level filter orchestrator.
@@ -18,7 +21,7 @@ module Henitai
       coverage_report_present = coverage_report_present?(config)
 
       Array(mutants).each do |mutant|
-        next if ignored_mutant?(mutant, config)
+        next if ignored_mutant?(mutant, config) || skip_directive_mutant?(mutant)
 
         mark_equivalent_mutant(mutant)
         mark_no_coverage_mutant(
@@ -61,7 +64,7 @@ module Henitai
 
     private
 
-    attr_reader :coverage_report_reader
+    attr_reader :coverage_report_reader, :skip_directives
 
     def ignored?(mutant, config)
       source = source_for(mutant)
@@ -79,6 +82,15 @@ module Henitai
       true
     end
 
+    def skip_directive_mutant?(mutant)
+      directive = skip_directives.directive_for(mutant)
+      return false unless directive
+
+      mutant.status = :ignored
+      mutant.ignore_reason = directive.reason if mutant.respond_to?(:ignore_reason=)
+      true
+    end
+
     def mark_equivalent_mutant(mutant)
       return unless mutant.pending?
 
@@ -96,9 +108,7 @@ module Henitai
     def covered?(mutant, coverage_lines)
       file = normalize_path(mutant.location[:file])
       covered = Array(coverage_lines[file])
-      (mutant.location[:start_line]..mutant.location[:end_line]).any? do |line|
-        covered.include?(line)
-      end
+      covered.any? { |line| (mutant.location[:start_line]..mutant.location[:end_line]).cover?(line) }
     end
 
     def source_for(mutant)