henitai 0.5.3 → 0.6.0

data/lib/henitai/execution_engine/fork_server_scope.rb

@@ -0,0 +1,57 @@
+# frozen_string_literal: true
+
+require_relative "../fork_server"
+
+module Henitai
+  class ExecutionEngine
+    # Opt-in fork-server execution (`mutation.fork_server`, ADR-15) for the
+    # engine's linear path. While the block runs, @fork_server_runner (when a
+    # server booted) runs the mutants; it is nil when the flag is off, the
+    # integration cannot boot once, or the server failed to boot, and the
+    # integration then boots per mutant.
+    module ForkServerScope
+      private
+
+      def with_fork_server(mutants, integration, config)
+        @fork_server_runner = start_fork_server(mutants, integration, config)
+        yield
+      ensure
+        @fork_server_runner&.stop
+        @fork_server_runner = nil
+      end
+
+      def start_fork_server(mutants, integration, config)
+        return unless fork_server_requested?(config)
+        return warn_fork_server_unsupported unless fork_server_supported?(integration)
+
+        test_files = mutants.flat_map { |mutant| prioritized_tests_for(mutant, integration, config) }.uniq
+        return if test_files.empty?
+
+        ForkServer.start(integration:, mutants:, test_files:, log_path: fork_server_log_path(config),
+                         fallback_timeout: ->(files) { resolved_timeout(files, config) })
+      end
+
+      def fork_server_requested?(config)
+        config.respond_to?(:fork_server) && config.fork_server ? true : false
+      end
+
+      def fork_server_supported?(integration)
+        integration.respond_to?(:fork_server_supported?) && integration.fork_server_supported?
+      end
+
+      def warn_fork_server_unsupported
+        warn("henitai: mutation.fork_server is not supported for this integration; using per-mutant boot")
+        nil
+      end
+
+      # The scheduler learns of served processes' exits through the server.
+      def parallel_runtime_options
+        @fork_server_runner&.active? ? { runtime: @fork_server_runner.runtime } : {}
+      end
+
+      def fork_server_log_path(config)
+        File.join(config.reports_dir, "mutation-logs", "fork-server.log")
+      end
+    end
+  end
+end

data/lib/henitai/execution_engine.rb

@@ -2,11 +2,13 @@
 
 require_relative "process_worker_runner"
 require_relative "execution_engine/env_scope"
+require_relative "execution_engine/fork_server_scope"
 
 module Henitai
   # Runs pending mutants through the selected integration.
   class ExecutionEngine
     include EnvScope
+    include ForkServerScope
 
     def run(mutants, integration, config, progress_reporter: nil)
       with_reports_dir(config) do
@@ -23,17 +25,36 @@ module Henitai
     private
 
     def execute(mutants, integration, config, progress_reporter)
-      @flaky_retry_count = 0
       pending_mutants = Array(mutants).select(&:pending?)
       mutex = Mutex.new
-      if parallel_execution?(config, pending_mutants)
-        run_parallel(pending_mutants, integration, config, progress_reporter)
-      else
-        run_linear(pending_mutants, integration, config, progress_reporter, mutex)
+      @flaky_tests = flaky_tests_for(config)
+      @retrier = FlakyMutantRetrier.new(flaky_tests: @flaky_tests, max_retries: max_flaky_retries(config))
+      run_pending(pending_mutants, integration, config, progress_reporter, mutex)
+      warn_flaky_mutants
+      verify_controls(pending_mutants, integration, config)
+      mutants
+    end
+
+    def run_pending(pending_mutants, integration, config, progress_reporter, mutex)
+      with_fork_server(pending_mutants, integration, config) do
+        if parallel_execution?(config, pending_mutants)
+          run_parallel(pending_mutants, integration, config, progress_reporter)
+        else
+          run_linear(pending_mutants, integration, config, progress_reporter, mutex)
+          @flaky_retry_count = @retrier.retried_count
+        end
       end
+    end
 
-      warn_flaky_mutants(pending_mutants.size)
-      mutants
+    def verify_controls(mutants, integration, config)
+      return unless config.respond_to?(:control_runs) && config.control_runs
+
+      timeout_for = ->(test_files) { resolved_timeout(test_files, config) }
+      stats = ControlRun.new(integration:, timeout_for:).verify(mutants)
+      return if stats[:failed].zero?
+
+      warn format("henitai: control runs failed for %<failed>d of %<verified>d subjects; " \
+                  "%<reclassified>d detected verdicts reclassified as CompileError", **stats)
     end
 
     def parallel_execution?(config, mutants)
@@ -57,10 +78,14 @@ module Henitai
     end
 
     def run_parallel(mutants, integration, config, progress_reporter)
-      runner = ProcessWorkerRunner.new(worker_count: worker_count(config))
+      runner = ProcessWorkerRunner.new(
+        worker_count: worker_count(config),
+        flaky_tests: @flaky_tests,
+        **parallel_runtime_options
+      )
       results = runner.run(
         mutants,
-        integration,
+        @fork_server_runner || integration,
         config,
         progress_reporter,
         test_file_resolver: ->(mutant) { prioritized_tests_for(mutant, integration, config) },
@@ -75,8 +100,12 @@ module Henitai
       record_test_files(mutant, test_files)
       return record_no_coverage(mutant, progress_reporter, mutex) if test_files.empty?
 
-      scenario_result = run_with_flaky_retry(mutant, integration, config, test_files, mutex)
+      scenario_result = @retrier.run(
+        mutant: mutant, integration: @fork_server_runner || integration, test_files: test_files,
+        timeout: resolved_timeout(test_files, config)
+      )
       mutant.status = scenario_status(scenario_result)
+      ScenarioExecutionResult.copy_reason(scenario_result, mutant)
 
       report_progress(mutant, scenario_result, progress_reporter, mutex)
     end
@@ -92,19 +121,15 @@ module Henitai
     end
 
     def report_progress(mutant, scenario_result, progress_reporter, mutex)
-      if mutex
-        mutex.synchronize { progress_reporter&.progress(mutant, scenario_result:) }
-      else
-        progress_reporter&.progress(mutant, scenario_result:)
-      end
+      mutex.synchronize { progress_reporter&.progress(mutant, scenario_result:) }
     end
 
     def prioritized_tests_for(mutant, integration, config)
-      tests = reject_excluded_tests(integration.select_tests(mutant.subject), config)
-      tests = per_test_coverage_selector.filter(
-        tests,
+      tests = per_test_coverage_selector.select(
         mutant,
-        reports_dir: config.reports_dir
+        reports_dir: config.reports_dir,
+        test_files: -> { reject_excluded_tests(integration.test_files, config) },
+        fallback: -> { reject_excluded_tests(integration.select_tests(mutant.subject), config) }
       )
       test_prioritizer(config).sort(tests, mutant, test_history(config))
     end
@@ -133,11 +158,21 @@ module Henitai
     def resolved_timeout(test_files, config)
       return config.timeout unless calibration_enabled?(config)
 
-      calibrated = timeout_calibrator(config).timeout_for(test_files)
-      return [calibrated, max_timeout(config)].min if calibrated
+      calibrator = timeout_calibrator(config)
+      calibrated = calibrator.timeout_for(test_files)
+      unless calibrated
+        warn_timeout_fallback_once
+        return Configuration::DEFAULT_TIMEOUT
+      end
 
-      warn_timeout_fallback_once
-      Configuration::DEFAULT_TIMEOUT
+      timeout_ceiling(config).cap(calibrated, calibrator.baseline_for(test_files))
+    end
+
+    def timeout_ceiling(config)
+      @timeout_ceiling ||= TimeoutCeiling.new(
+        ceiling: max_timeout(config),
+        warner: ->(message) { warn(message) }
+      )
     end
 
     # Ceiling on the auto-calibrated timeout so a runaway mutant is killed in
@@ -153,13 +188,21 @@ module Henitai
       config.respond_to?(:timeout_configured?) && !config.timeout_configured?
     end
 
+    # Boot stays in the timeout even with a fork server: served mutants still
+    # run the suite's before(:suite) hooks, which the boot measurement includes.
     def timeout_calibrator(config)
       @timeout_calibrator ||= TimeoutCalibrator.new(
         timing_source: timing_source(config),
-        multiplier: config.timeout_multiplier
+        multiplier: config.timeout_multiplier,
+        boot_seconds: -> { boot_seconds(config) }
       )
     end
 
+    def boot_seconds(config)
+      path = File.join(config.reports_dir, PerTestCoverageCollector::BOOT_FILE_NAME)
+      CoverageReportReader.new.boot_seconds(path)
+    end
+
     def warn_timeout_fallback_once
       return if @warned_timeout_fallback
 
@@ -178,34 +221,9 @@ module Henitai
       config.history || {}
     end
 
-    # Retry logic is kept in one place to preserve the status transition flow.
-    # The retry budget is configurable because repeated survivors can multiply
-    # runtime on real CI workloads.
-    # rubocop:disable Metrics/MethodLength
-    def run_with_flaky_retry(mutant, integration, config, test_files, mutex)
-      timeout = resolved_timeout(test_files, config)
-      scenario_result = integration.run_mutant(
-        mutant:,
-        test_files:,
-        timeout: timeout
-      )
-      return scenario_result unless scenario_status(scenario_result) == :survived
-
-      retries = 0
-      max_flaky_retries(config).times do
-        retries += 1
-        scenario_result = integration.run_mutant(
-          mutant:,
-          test_files:,
-          timeout: timeout
-        )
-        break unless scenario_status(scenario_result) == :survived
-      end
-
-      mutex.synchronize { @flaky_retry_count += 1 } if retries.positive?
-      scenario_result
+    def flaky_tests_for(config)
+      FlakyTestRegistry.for_config(config)
     end
-    # rubocop:enable Metrics/MethodLength
 
     def scenario_status(result)
       return result if result.is_a?(Symbol)
@@ -213,17 +231,14 @@ module Henitai
       result.status
     end
 
-    def warn_flaky_mutants(total_mutants)
-      return if total_mutants.zero?
-
-      flaky_ratio = @flaky_retry_count.to_f / total_mutants
-      return unless flaky_ratio > 0.05
+    def warn_flaky_mutants
+      return if @flaky_retry_count.to_i.zero?
 
       warn format(
-        "Flaky-test mitigation: %<flaky>d/%<total>d mutants required retries (%<ratio>.2f%%)",
-        flaky: @flaky_retry_count,
-        total: total_mutants,
-        ratio: flaky_ratio * 100.0
+        "Flaky-test mitigation: %<count>d killed mutant(s) were re-checked because their killing tests " \
+        "are flaky (%<files>s)",
+        count: @flaky_retry_count,
+        files: @flaky_tests.all.join(", ")
       )
     end
 

data/lib/henitai/fidelity_check.rb

@@ -0,0 +1,173 @@
+# frozen_string_literal: true
+
+require "prism"
+require_relative "parser_current"
+require_relative "source_parser"
+
+module Henitai
+  # Verifies, before a mutant runs, that its activation source differs from
+  # the original method by exactly the reported mutation and nothing else.
+  #
+  # The checks run in order:
+  #
+  # 1. Site: the mutated node must lie inside the method body that the
+  #    activation replaces.
+  # 2. Syntax: the activation source must be valid Ruby in context. Prism's
+  #    error list is used because the parser translation layer recovers from
+  #    errors silently.
+  # 3. Structure: the activated method body, re-parsed, must equal the
+  #    original body with the original node replaced by the mutated node.
+  #    Both sides are normalised first: parentheses around one expression
+  #    are dropped, an interpolation-free +dstr+ is the same value as one
+  #    +str+, nested statement sequences flatten, an empty kwargs node is no
+  #    argument, and +__FILE__+ compares as an absolute path.
+  #
+  # A violation is a harness defect, not a test verdict: the mutant must not
+  # run, and the reason must be reported.
+  class FidelityCheck
+    # @return [String, nil] the reason the mutant is unfaithful, or nil
+    def violation(mutant)
+      return unless checkable?(mutant)
+
+      activation = Mutant::Activator.activation_for(mutant)
+      return "activation source could not be built" unless activation
+
+      whole_def = activation.mode == :toplevel
+      expected = expected_tree(mutant, whole_def)
+      return "mutation site is outside the executed method body" unless expected
+
+      syntax_violation(activation.source) || structure_violation(mutant, expected, activation, whole_def)
+    rescue StandardError => e
+      "fidelity could not be verified (#{e.class}: #{e.message})"
+    end
+
+    private
+
+    def checkable?(mutant)
+      return false unless mutant.original_node
+
+      subject = mutant.subject
+      !subject.method_name.nil? && !subject.ast_node.nil?
+    end
+
+    def syntax_violation(source)
+      error = Prism.parse(source).errors.first
+      "activation source is not valid Ruby: #{error.message}" if error
+    end
+
+    def structure_violation(mutant, expected, activation, whole_def)
+      source = activation.source
+      tree = SourceParser.parse(source, path: activation.file || Mutant::Activator::EVAL_FILE,
+                                        line: activation.line || 1)
+      actual = normalize(activated_tree(tree, mutant.subject.method_name, whole_def))
+      expected = normalize(expected)
+      "activated code differs from the reported mutation" unless actual == expected
+    end
+
+    # A re-evaluated def (ADR-13) is compared as a whole, parameters
+    # included; the define_method fallback only by body.
+    def expected_tree(mutant, whole_def)
+      definition = mutant.subject.ast_node
+      scope = whole_def ? definition : method_body(definition)
+      replaced, found = substitute(scope, mutant.original_node, mutant.mutated_node)
+      replaced if found
+    end
+
+    def method_body(node)
+      case node.type
+      when :defs then node.children[3]
+      when :def, :block then node.children[2]
+      end
+    end
+
+    def substitute(node, target, replacement)
+      return [replacement, true] if node.equal?(target)
+      return [node, false] unless node.is_a?(Parser::AST::Node)
+
+      found = false
+      children = node.children.map do |child|
+        child, child_found = substitute(child, target, replacement)
+        found ||= child_found
+        child
+      end
+      [found ? node.updated(nil, children) : node, found]
+    end
+
+    # The method as activated: a re-evaluated +def+ (ADR-13) or the
+    # +define_method+ block of the fallback.
+    def activated_tree(tree, method_name, whole_def)
+      name = method_name.to_sym
+      definition = find_node(tree) do |node|
+        activated_def?(node, name) || (node.type == :block && define_method_call?(node.children[0], name))
+      end
+      whole_def ? definition : definition && method_body(definition)
+    end
+
+    def activated_def?(node, name)
+      (node.type == :def && node.children[0] == name) || (node.type == :defs && node.children[1] == name)
+    end
+
+    def define_method_call?(call, method_name)
+      call.type == :send && call.children[0].nil? && call.children[1] == :define_method &&
+        call.children[2]&.type == :sym && call.children[2].children[0] == method_name
+    end
+
+    def find_node(node, &)
+      return unless node.is_a?(Parser::AST::Node)
+      return node if yield(node)
+
+      node.children.each do |child|
+        found = find_node(child, &)
+        return found if found
+      end
+      nil
+    end
+
+    def normalize(node)
+      return node unless node.is_a?(Parser::AST::Node)
+      return normalize(node.children[0]) if parenthesised?(node)
+      return node.updated(:str, [literal_text(node)]) if literal_dstr?(node)
+      return node.updated(nil, [File.expand_path(node.children[0])]) if file_keyword?(node)
+
+      node.updated(nil, normalized_children(node))
+    end
+
+    # `foo(a, )` -- left behind when all keyword arguments are removed --
+    # calls `foo(a)`, so an empty kwargs node is no argument at all.
+    def normalized_children(node)
+      children = node.children.map { |child| normalize(child) }.reject { |child| empty_kwargs?(child) }
+      node.type == :begin ? children.flat_map { |child| statements(child) } : children
+    end
+
+    # Parentheses around one expression carry no meaning once the tree
+    # encodes precedence; Mutant::Replacement adds them around operators.
+    def parenthesised?(node)
+      node.type == :begin && node.children.size == 1
+    end
+
+    def empty_kwargs?(node)
+      node.is_a?(Parser::AST::Node) && node.type == :kwargs && node.children.empty?
+    end
+
+    # A statement sequence spliced into another one parses as one flat
+    # sequence.
+    def statements(node)
+      node.is_a?(Parser::AST::Node) && node.type == :begin ? node.children : [node]
+    end
+
+    def file_keyword?(node)
+      node.type == :str && node.location&.expression&.source == "__FILE__"
+    end
+
+    # An interpolation-free heredoc parses into a +dstr+ of line segments.
+    def literal_dstr?(node)
+      node.type == :dstr && node.children.all? { |child| child.type == :str || literal_dstr?(child) }
+    end
+
+    def literal_text(node)
+      return node.children[0] if node.type == :str
+
+      node.children.map { |child| literal_text(child) }.join
+    end
+  end
+end

data/lib/henitai/flaky_detection_baseline.rb

@@ -0,0 +1,61 @@
+# frozen_string_literal: true
+
+require "fileutils"
+
+module Henitai
+  # Runs the suite twice and persists the test files whose outcome differed.
+  #
+  # Only used when `mutation.flaky_detection` is enabled. Each run records
+  # per-file outcomes via the integration's coverage hooks; the files that
+  # disagree between the two runs are written to `henitai_flaky_tests.json`.
+  class FlakyDetectionBaseline
+    def initialize(reports_dir:)
+      @reports_dir = reports_dir
+    end
+
+    def run(integration:, test_files:)
+      FileUtils.mkdir_p(@reports_dir)
+      first, first_result = run_once(integration, test_files)
+      second, second_result = run_once(integration, test_files)
+      registry = FlakyTestRegistry.from_outcomes(first.outcomes, second.outcomes)
+      ensure_usable_suite(registry, [[first, first_result], [second, second_result]])
+      registry.write(File.join(@reports_dir, FlakyTestRegistry::FILE_NAME))
+      warn_flaky(registry)
+      registry
+    end
+
+    private
+
+    def run_once(integration, test_files)
+      outcome_path = TestFileOutcomes.report_path(@reports_dir)
+      FileUtils.rm_f(outcome_path)
+      result = integration.run_suite(test_files)
+      raise CoverageError, "could not record suite outcomes" unless File.exist?(outcome_path)
+
+      [TestFileOutcomes.load(outcome_path), result]
+    end
+
+    # A failing run is only acceptable as evidence of flakiness: the suite
+    # must pass at least once and every file that failed in a failing run must
+    # be a flaky file. A run that failed without a file to blame (a load error,
+    # a suite-level hook) is never explained.
+    def ensure_usable_suite(registry, runs)
+      failing = runs.reject { |_outcomes, result| result.survived? }
+      return if failing.empty?
+      return if failing.size < runs.size && failing.all? { |outcomes, _result| explained?(registry, outcomes) }
+
+      raise CoverageError, BootstrapErrorMessage.build(failing.first.last)
+    end
+
+    def explained?(registry, outcomes)
+      failed = outcomes.failed_files
+      !outcomes.unattributed_failures? && !failed.empty? && failed.all? { |file| registry.flaky?(file) }
+    end
+
+    def warn_flaky(registry)
+      return if registry.empty?
+
+      warn("Flaky tests detected (#{registry.all.size}): #{registry.all.join(', ')}")
+    end
+  end
+end

data/lib/henitai/flaky_mutant_retrier.rb

@@ -0,0 +1,43 @@
+# frozen_string_literal: true
+
+module Henitai
+  # Re-runs a mutant whose kill is attributable only to known-flaky test files.
+  #
+  # Survivors are never retried: a retry could only turn a survivor into a
+  # kill, which would inflate the score. A kill is retried only when every
+  # killing test file is in the flaky set, up to `max_retries` times.
+  class FlakyMutantRetrier
+    attr_reader :retried_count
+
+    def initialize(flaky_tests:, max_retries:)
+      @decision = FlakyRetryDecision.new(flaky_tests: flaky_tests, max_retries: max_retries)
+      @retried_count = 0
+      @mutex = Mutex.new
+    end
+
+    def run(mutant:, integration:, test_files:, timeout:)
+      result = integration.run_mutant(mutant: mutant, test_files: test_files, timeout: timeout)
+      attempts = 0
+      while @decision.retry?(**decision_input(result, attempts))
+        attempts += 1
+        result = integration.run_mutant(mutant: mutant, test_files: test_files, timeout: timeout)
+      end
+      @mutex.synchronize { @retried_count += 1 } if attempts.positive?
+      result
+    end
+
+    private
+
+    def decision_input(result, attempts)
+      {
+        status: status_of(result),
+        killing_tests: result.respond_to?(:killing_tests) ? result.killing_tests : [],
+        attempts: attempts
+      }
+    end
+
+    def status_of(result)
+      result.is_a?(Symbol) ? result : result.status
+    end
+  end
+end

data/lib/henitai/flaky_retry_decision.rb

@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+module Henitai
+  # Decides whether a mutant verdict earns another attempt.
+  #
+  # Survivors are never retried: flakiness can only turn a survivor into a
+  # kill, which would inflate the score. A kill is retried only when every test
+  # that killed the mutant is a known flaky test, because flakiness can fake a
+  # kill but not a survivor. Without a flaky set nothing is retried.
+  class FlakyRetryDecision
+    def initialize(flaky_tests:, max_retries:)
+      @flaky_tests = flaky_tests
+      @max_retries = max_retries.to_i
+    end
+
+    def retry?(status:, killing_tests:, attempts:)
+      status == :killed &&
+        attempts < @max_retries &&
+        killing_tests.any? &&
+        killing_tests.all? { |file| @flaky_tests.flaky?(file) }
+    end
+  end
+end

data/lib/henitai/flaky_test_registry.rb

@@ -0,0 +1,63 @@
+# frozen_string_literal: true
+
+require "json"
+
+module Henitai
+  # The set of test files whose outcome differed between two baseline runs.
+  #
+  # Produced by comparing per-file pass/fail maps from two runs of the same
+  # suite, and persisted as `henitai_flaky_tests.json` so that later runs can
+  # tell a kill caused by a flaky test from one caused by a stable test.
+  class FlakyTestRegistry
+    FILE_NAME = "henitai_flaky_tests.json"
+
+    # A file is flaky only when both runs ran it and disagreed. A file seen in
+    # one run only has no second observation, so it is never called flaky;
+    # FlakyDetectionBaseline rejects a failing run that such a file leaves
+    # unexplained.
+    def self.from_outcomes(first, second)
+      shared = first.keys & second.keys
+      flaky = shared.reject { |file| first[file] == second[file] }
+      new(flaky)
+    end
+
+    def self.load(path)
+      return new([]) unless File.file?(path)
+
+      new(Array(JSON.parse(File.read(path))["flaky_tests"]))
+    end
+
+    def self.for_config(config)
+      return new([]) unless config.respond_to?(:flaky_detection) && config.flaky_detection
+
+      for_reports_dir(config.respond_to?(:reports_dir) ? config.reports_dir : nil)
+    end
+
+    def self.for_reports_dir(dir)
+      return new([]) if dir.nil?
+
+      load(File.join(dir, FILE_NAME))
+    end
+
+    def initialize(files)
+      @files = files.map { |file| File.expand_path(file) }.uniq.sort.freeze
+      @lookup = @files.to_set.freeze
+    end
+
+    def all
+      @files
+    end
+
+    def flaky?(path)
+      @lookup.include?(File.expand_path(path))
+    end
+
+    def empty?
+      @files.empty?
+    end
+
+    def write(path)
+      File.write(path, JSON.pretty_generate("flaky_tests" => @files))
+    end
+  end
+end