henitai 0.5.1 → 0.6.0

data/lib/henitai/fidelity_check.rb

@@ -0,0 +1,173 @@
+# frozen_string_literal: true
+
+require "prism"
+require_relative "parser_current"
+require_relative "source_parser"
+
+module Henitai
+  # Verifies, before a mutant runs, that its activation source differs from
+  # the original method by exactly the reported mutation and nothing else.
+  #
+  # The checks run in order:
+  #
+  # 1. Site: the mutated node must lie inside the method body that the
+  #    activation replaces.
+  # 2. Syntax: the activation source must be valid Ruby in context. Prism's
+  #    error list is used because the parser translation layer recovers from
+  #    errors silently.
+  # 3. Structure: the activated method body, re-parsed, must equal the
+  #    original body with the original node replaced by the mutated node.
+  #    Both sides are normalised first: parentheses around one expression
+  #    are dropped, an interpolation-free +dstr+ is the same value as one
+  #    +str+, nested statement sequences flatten, an empty kwargs node is no
+  #    argument, and +__FILE__+ compares as an absolute path.
+  #
+  # A violation is a harness defect, not a test verdict: the mutant must not
+  # run, and the reason must be reported.
+  class FidelityCheck
+    # @return [String, nil] the reason the mutant is unfaithful, or nil
+    def violation(mutant)
+      return unless checkable?(mutant)
+
+      activation = Mutant::Activator.activation_for(mutant)
+      return "activation source could not be built" unless activation
+
+      whole_def = activation.mode == :toplevel
+      expected = expected_tree(mutant, whole_def)
+      return "mutation site is outside the executed method body" unless expected
+
+      syntax_violation(activation.source) || structure_violation(mutant, expected, activation, whole_def)
+    rescue StandardError => e
+      "fidelity could not be verified (#{e.class}: #{e.message})"
+    end
+
+    private
+
+    def checkable?(mutant)
+      return false unless mutant.original_node
+
+      subject = mutant.subject
+      !subject.method_name.nil? && !subject.ast_node.nil?
+    end
+
+    def syntax_violation(source)
+      error = Prism.parse(source).errors.first
+      "activation source is not valid Ruby: #{error.message}" if error
+    end
+
+    def structure_violation(mutant, expected, activation, whole_def)
+      source = activation.source
+      tree = SourceParser.parse(source, path: activation.file || Mutant::Activator::EVAL_FILE,
+                                        line: activation.line || 1)
+      actual = normalize(activated_tree(tree, mutant.subject.method_name, whole_def))
+      expected = normalize(expected)
+      "activated code differs from the reported mutation" unless actual == expected
+    end
+
+    # A re-evaluated def (ADR-13) is compared as a whole, parameters
+    # included; the define_method fallback only by body.
+    def expected_tree(mutant, whole_def)
+      definition = mutant.subject.ast_node
+      scope = whole_def ? definition : method_body(definition)
+      replaced, found = substitute(scope, mutant.original_node, mutant.mutated_node)
+      replaced if found
+    end
+
+    def method_body(node)
+      case node.type
+      when :defs then node.children[3]
+      when :def, :block then node.children[2]
+      end
+    end
+
+    def substitute(node, target, replacement)
+      return [replacement, true] if node.equal?(target)
+      return [node, false] unless node.is_a?(Parser::AST::Node)
+
+      found = false
+      children = node.children.map do |child|
+        child, child_found = substitute(child, target, replacement)
+        found ||= child_found
+        child
+      end
+      [found ? node.updated(nil, children) : node, found]
+    end
+
+    # The method as activated: a re-evaluated +def+ (ADR-13) or the
+    # +define_method+ block of the fallback.
+    def activated_tree(tree, method_name, whole_def)
+      name = method_name.to_sym
+      definition = find_node(tree) do |node|
+        activated_def?(node, name) || (node.type == :block && define_method_call?(node.children[0], name))
+      end
+      whole_def ? definition : definition && method_body(definition)
+    end
+
+    def activated_def?(node, name)
+      (node.type == :def && node.children[0] == name) || (node.type == :defs && node.children[1] == name)
+    end
+
+    def define_method_call?(call, method_name)
+      call.type == :send && call.children[0].nil? && call.children[1] == :define_method &&
+        call.children[2]&.type == :sym && call.children[2].children[0] == method_name
+    end
+
+    def find_node(node, &)
+      return unless node.is_a?(Parser::AST::Node)
+      return node if yield(node)
+
+      node.children.each do |child|
+        found = find_node(child, &)
+        return found if found
+      end
+      nil
+    end
+
+    def normalize(node)
+      return node unless node.is_a?(Parser::AST::Node)
+      return normalize(node.children[0]) if parenthesised?(node)
+      return node.updated(:str, [literal_text(node)]) if literal_dstr?(node)
+      return node.updated(nil, [File.expand_path(node.children[0])]) if file_keyword?(node)
+
+      node.updated(nil, normalized_children(node))
+    end
+
+    # `foo(a, )` -- left behind when all keyword arguments are removed --
+    # calls `foo(a)`, so an empty kwargs node is no argument at all.
+    def normalized_children(node)
+      children = node.children.map { |child| normalize(child) }.reject { |child| empty_kwargs?(child) }
+      node.type == :begin ? children.flat_map { |child| statements(child) } : children
+    end
+
+    # Parentheses around one expression carry no meaning once the tree
+    # encodes precedence; Mutant::Replacement adds them around operators.
+    def parenthesised?(node)
+      node.type == :begin && node.children.size == 1
+    end
+
+    def empty_kwargs?(node)
+      node.is_a?(Parser::AST::Node) && node.type == :kwargs && node.children.empty?
+    end
+
+    # A statement sequence spliced into another one parses as one flat
+    # sequence.
+    def statements(node)
+      node.is_a?(Parser::AST::Node) && node.type == :begin ? node.children : [node]
+    end
+
+    def file_keyword?(node)
+      node.type == :str && node.location&.expression&.source == "__FILE__"
+    end
+
+    # An interpolation-free heredoc parses into a +dstr+ of line segments.
+    def literal_dstr?(node)
+      node.type == :dstr && node.children.all? { |child| child.type == :str || literal_dstr?(child) }
+    end
+
+    def literal_text(node)
+      return node.children[0] if node.type == :str
+
+      node.children.map { |child| literal_text(child) }.join
+    end
+  end
+end

data/lib/henitai/flaky_detection_baseline.rb

@@ -0,0 +1,61 @@
+# frozen_string_literal: true
+
+require "fileutils"
+
+module Henitai
+  # Runs the suite twice and persists the test files whose outcome differed.
+  #
+  # Only used when `mutation.flaky_detection` is enabled. Each run records
+  # per-file outcomes via the integration's coverage hooks; the files that
+  # disagree between the two runs are written to `henitai_flaky_tests.json`.
+  class FlakyDetectionBaseline
+    def initialize(reports_dir:)
+      @reports_dir = reports_dir
+    end
+
+    def run(integration:, test_files:)
+      FileUtils.mkdir_p(@reports_dir)
+      first, first_result = run_once(integration, test_files)
+      second, second_result = run_once(integration, test_files)
+      registry = FlakyTestRegistry.from_outcomes(first.outcomes, second.outcomes)
+      ensure_usable_suite(registry, [[first, first_result], [second, second_result]])
+      registry.write(File.join(@reports_dir, FlakyTestRegistry::FILE_NAME))
+      warn_flaky(registry)
+      registry
+    end
+
+    private
+
+    def run_once(integration, test_files)
+      outcome_path = TestFileOutcomes.report_path(@reports_dir)
+      FileUtils.rm_f(outcome_path)
+      result = integration.run_suite(test_files)
+      raise CoverageError, "could not record suite outcomes" unless File.exist?(outcome_path)
+
+      [TestFileOutcomes.load(outcome_path), result]
+    end
+
+    # A failing run is only acceptable as evidence of flakiness: the suite
+    # must pass at least once and every file that failed in a failing run must
+    # be a flaky file. A run that failed without a file to blame (a load error,
+    # a suite-level hook) is never explained.
+    def ensure_usable_suite(registry, runs)
+      failing = runs.reject { |_outcomes, result| result.survived? }
+      return if failing.empty?
+      return if failing.size < runs.size && failing.all? { |outcomes, _result| explained?(registry, outcomes) }
+
+      raise CoverageError, BootstrapErrorMessage.build(failing.first.last)
+    end
+
+    def explained?(registry, outcomes)
+      failed = outcomes.failed_files
+      !outcomes.unattributed_failures? && !failed.empty? && failed.all? { |file| registry.flaky?(file) }
+    end
+
+    def warn_flaky(registry)
+      return if registry.empty?
+
+      warn("Flaky tests detected (#{registry.all.size}): #{registry.all.join(', ')}")
+    end
+  end
+end

data/lib/henitai/flaky_mutant_retrier.rb

@@ -0,0 +1,43 @@
+# frozen_string_literal: true
+
+module Henitai
+  # Re-runs a mutant whose kill is attributable only to known-flaky test files.
+  #
+  # Survivors are never retried: a retry could only turn a survivor into a
+  # kill, which would inflate the score. A kill is retried only when every
+  # killing test file is in the flaky set, up to `max_retries` times.
+  class FlakyMutantRetrier
+    attr_reader :retried_count
+
+    def initialize(flaky_tests:, max_retries:)
+      @decision = FlakyRetryDecision.new(flaky_tests: flaky_tests, max_retries: max_retries)
+      @retried_count = 0
+      @mutex = Mutex.new
+    end
+
+    def run(mutant:, integration:, test_files:, timeout:)
+      result = integration.run_mutant(mutant: mutant, test_files: test_files, timeout: timeout)
+      attempts = 0
+      while @decision.retry?(**decision_input(result, attempts))
+        attempts += 1
+        result = integration.run_mutant(mutant: mutant, test_files: test_files, timeout: timeout)
+      end
+      @mutex.synchronize { @retried_count += 1 } if attempts.positive?
+      result
+    end
+
+    private
+
+    def decision_input(result, attempts)
+      {
+        status: status_of(result),
+        killing_tests: result.respond_to?(:killing_tests) ? result.killing_tests : [],
+        attempts: attempts
+      }
+    end
+
+    def status_of(result)
+      result.is_a?(Symbol) ? result : result.status
+    end
+  end
+end

data/lib/henitai/flaky_retry_decision.rb

@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+module Henitai
+  # Decides whether a mutant verdict earns another attempt.
+  #
+  # Survivors are never retried: flakiness can only turn a survivor into a
+  # kill, which would inflate the score. A kill is retried only when every test
+  # that killed the mutant is a known flaky test, because flakiness can fake a
+  # kill but not a survivor. Without a flaky set nothing is retried.
+  class FlakyRetryDecision
+    def initialize(flaky_tests:, max_retries:)
+      @flaky_tests = flaky_tests
+      @max_retries = max_retries.to_i
+    end
+
+    def retry?(status:, killing_tests:, attempts:)
+      status == :killed &&
+        attempts < @max_retries &&
+        killing_tests.any? &&
+        killing_tests.all? { |file| @flaky_tests.flaky?(file) }
+    end
+  end
+end

data/lib/henitai/flaky_test_registry.rb

@@ -0,0 +1,63 @@
+# frozen_string_literal: true
+
+require "json"
+
+module Henitai
+  # The set of test files whose outcome differed between two baseline runs.
+  #
+  # Produced by comparing per-file pass/fail maps from two runs of the same
+  # suite, and persisted as `henitai_flaky_tests.json` so that later runs can
+  # tell a kill caused by a flaky test from one caused by a stable test.
+  class FlakyTestRegistry
+    FILE_NAME = "henitai_flaky_tests.json"
+
+    # A file is flaky only when both runs ran it and disagreed. A file seen in
+    # one run only has no second observation, so it is never called flaky;
+    # FlakyDetectionBaseline rejects a failing run that such a file leaves
+    # unexplained.
+    def self.from_outcomes(first, second)
+      shared = first.keys & second.keys
+      flaky = shared.reject { |file| first[file] == second[file] }
+      new(flaky)
+    end
+
+    def self.load(path)
+      return new([]) unless File.file?(path)
+
+      new(Array(JSON.parse(File.read(path))["flaky_tests"]))
+    end
+
+    def self.for_config(config)
+      return new([]) unless config.respond_to?(:flaky_detection) && config.flaky_detection
+
+      for_reports_dir(config.respond_to?(:reports_dir) ? config.reports_dir : nil)
+    end
+
+    def self.for_reports_dir(dir)
+      return new([]) if dir.nil?
+
+      load(File.join(dir, FILE_NAME))
+    end
+
+    def initialize(files)
+      @files = files.map { |file| File.expand_path(file) }.uniq.sort.freeze
+      @lookup = @files.to_set.freeze
+    end
+
+    def all
+      @files
+    end
+
+    def flaky?(path)
+      @lookup.include?(File.expand_path(path))
+    end
+
+    def empty?
+      @files.empty?
+    end
+
+    def write(path)
+      File.write(path, JSON.pretty_generate("flaky_tests" => @files))
+    end
+  end
+end

data/lib/henitai/fork_server/client.rb

@@ -0,0 +1,231 @@
+# frozen_string_literal: true
+
+module Henitai
+  module ForkServer
+    # The parent's end of the connection to the fork server. It sends `run`
+    # requests and collects the server's events; the parent cannot wait on the
+    # mutant processes (they are the server's children), so their ends arrive
+    # as `exited` events.
+    class Client
+      class ServerLost < StandardError; end
+
+      class BootFailed < StandardError; end
+
+      class SpawnFailed < StandardError; end
+
+      SHUTDOWN_GRACE = 5.0
+      SPAWN_TIMEOUT = 30.0
+      REAP_POLL_SECONDS = 0.05
+
+      # @param socket [IO] the parent end of the connection
+      # @param pid [Integer, nil] the server process, reaped on #shutdown
+      # @param shutdown_grace [Float] seconds the server may take to exit before it is killed
+      def initialize(socket:, pid:, shutdown_grace: SHUTDOWN_GRACE)
+        @socket = socket
+        @pid = pid
+        @shutdown_grace = shutdown_grace
+        @reader = Protocol::Reader.new(socket)
+        @next_request_id = 0
+        @boot = nil
+        @spawned = {}
+        @exits = {}
+        @outstanding = []
+      end
+
+      def lost?
+        @lost == true
+      end
+
+      # The connection the parent can select on to learn of relayed events.
+      def io
+        @socket
+      end
+
+      # @raise [BootFailed] when the server reports failure or stays silent
+      # @raise [ServerLost] when the server dies while booting
+      def wait_until_ready(timeout)
+        outcome = await(timeout) { @boot }
+        raise BootFailed, "the fork server did not become ready within #{timeout}s" unless outcome
+        raise BootFailed, outcome.last if outcome.first == :failed
+
+        nil
+      end
+
+      # @return [Integer] pid of the forked mutant process
+      # @raise [SpawnFailed] when the server could not fork this mutant
+      # @raise [ServerLost]
+      def spawn_mutant(mutant_id:, test_files:, log_paths:, env: {})
+        request_id = (@next_request_id += 1)
+        send_message("type" => "run", "request_id" => request_id, "mutant_id" => mutant_id,
+                     "test_files" => test_files, "log_paths" => log_paths, "env" => env)
+        outcome = await(SPAWN_TIMEOUT) { @spawned.delete(request_id) }
+        lose_server("the fork server did not answer a run request") unless outcome
+        raise SpawnFailed, outcome.last if outcome.first == :failed
+
+        @outstanding << outcome.last
+        outcome.last
+      end
+
+      # @return [Status, nil] how the process ended; nil when it is still
+      #   running after +timeout+ seconds
+      def wait_exit(pid, timeout)
+        await(timeout) { @exits.delete(pid) }
+      end
+
+      # Non-blocking variants for the parallel scheduler, which reaps
+      # completions from relayed events. A lost server completes every
+      # outstanding process with a {Status.lost} instead of raising, so no
+      # slot waits for an event that cannot arrive and no verdict is invented.
+      #
+      # @return [Status, nil]
+      def poll_exit(pid)
+        drain_available
+        @exits.delete(pid)
+      end
+
+      # @return [Array(Integer, Status), nil]
+      def poll_any_exit
+        drain_available
+        pid, status = @exits.first
+        return nil unless pid
+
+        @exits.delete(pid)
+        [pid, status]
+      end
+
+      # Whether an event is already in user space (read off the socket but not
+      # yet handed out), which the socket's readability no longer signals.
+      def pending_events?
+        !@exits.empty? || @reader.buffered_message?
+      end
+
+      # Stops tracking a pid whose result is no longer wanted, such as a child
+      # the caller gave up on after a timeout. Its exit, if one already
+      # arrived or arrives later, is dropped rather than left for a recycled pid.
+      def forget(pid)
+        drain_available
+        @outstanding.delete(pid)
+        @exits.delete(pid)
+      end
+
+      # Gives up on a server that is lost or stalled: it must not fork another
+      # mutant that a local run is about to repeat.
+      def abandon
+        @lost = true
+        kill_outstanding
+        close_socket
+        reap_server
+        @pid = nil
+      end
+
+      def shutdown
+        send_message("type" => "shutdown")
+      rescue ServerLost
+        nil
+      ensure
+        close_socket
+        reap_server
+      end
+
+      private
+
+      # The registry keeps fallback children from inheriting this connection.
+      def close_socket
+        InheritedFdRegistry.unregister(@socket)
+        @socket.close unless @socket.closed?
+      end
+
+      # Without its server a mutant process has no one to reap it, and a local
+      # rerun of the same mutant must not race it.
+      def kill_outstanding
+        @outstanding.each { |pid| GroupKiller.kill(pid) }
+      end
+
+      def await(timeout)
+        deadline = now + timeout
+        until (result = yield)
+          remaining = deadline - now
+          return nil unless remaining.positive?
+
+          pump(remaining)
+        end
+        result
+      end
+
+      def pump(timeout)
+        message = @reader.next_message(timeout)
+        return if message == Protocol::Reader::TIMEOUT
+        return lose_server if message.nil?
+
+        record(message)
+      end
+
+      def drain_available
+        loop do
+          message = @reader.next_message(0)
+          break if message == Protocol::Reader::TIMEOUT
+
+          message.nil? ? lose_server : record(message)
+        end
+      rescue ServerLost
+        nil
+      end
+
+      def lose_server(reason = "the fork server closed the connection")
+        @lost = true
+        kill_outstanding
+        @outstanding.each { |pid| @exits[pid] ||= Status.lost }
+        @outstanding.clear
+        raise ServerLost, reason
+      end
+
+      def record(message)
+        case message["type"]
+        when "ready" then @boot = [:ready]
+        when "boot_failed" then @boot = [:failed, message["reason"]]
+        when "spawned" then @spawned[message["request_id"]] = [:ok, message["pid"]]
+        when "spawn_failed" then @spawned[message["request_id"]] = [:failed, message["reason"]]
+        when "exited" then record_exit(message)
+        end
+      end
+
+      def record_exit(message)
+        pid = message["pid"]
+        return unless @outstanding.delete(pid)
+
+        @exits[pid] = Status.from_event(message)
+      end
+
+      def send_message(message)
+        @socket.write(Protocol.encode(message))
+      rescue IOError, SystemCallError => e
+        raise ServerLost, "the fork server connection is gone: #{e.message}"
+      end
+
+      def reap_server
+        return if @pid.nil? || exited_within?(@shutdown_grace)
+
+        Process.kill(:SIGKILL, @pid)
+        Process.wait(@pid)
+      rescue Errno::ECHILD, Errno::ESRCH
+        nil
+      end
+
+      def exited_within?(grace)
+        deadline = now + grace
+        loop do
+          return true if Process.wait(@pid, Process::WNOHANG)
+          return false if now > deadline
+
+          sleep(REAP_POLL_SECONDS)
+        end
+      rescue Errno::ECHILD
+        true
+      end
+
+      def now
+        Process.clock_gettime(Process::CLOCK_MONOTONIC)
+      end
+    end
+  end
+end

data/lib/henitai/fork_server/group_killer.rb

@@ -0,0 +1,48 @@
+# frozen_string_literal: true
+
+module Henitai
+  module ForkServer
+    # Signals a mutant process the fork server forked. Such a process leads
+    # its own process group only once its bootstrap ran +setpgid+, so until
+    # then the group does not exist and the process itself is signalled.
+    # Its parent is the server, so it is never ours to wait on.
+    module GroupKiller
+      # @param leader [Boolean] whether the pid itself may still be alive;
+      #   false once it was reaped, when only stragglers in its group remain
+      def self.kill(pid, signal = :SIGKILL, leader: true)
+        Process.kill(signal, -pid)
+      rescue Errno::ESRCH
+        kill_leader(pid, signal) if leader
+      rescue Errno::EPERM
+        nil
+      end
+
+      def self.kill_leader(pid, signal)
+        Process.kill(signal, pid)
+      rescue Errno::ESRCH, Errno::EPERM
+        nil
+      end
+      private_class_method :kill_leader
+
+      # Signal 0 probes existence only; a missing process or group is not alive.
+      def self.alive?(pid, leader: true)
+        Process.kill(0, -pid)
+        true
+      rescue Errno::ESRCH
+        leader && probe(pid)
+      rescue Errno::EPERM
+        true
+      end
+
+      def self.probe(pid)
+        Process.kill(0, pid)
+        true
+      rescue Errno::ESRCH
+        false
+      rescue Errno::EPERM
+        true
+      end
+      private_class_method :probe
+    end
+  end
+end

data/lib/henitai/fork_server/group_terminator.rb

@@ -0,0 +1,47 @@
+# frozen_string_literal: true
+
+module Henitai
+  module ForkServer
+    # Stops a mutant process of the fork server and everything it started.
+    #
+    # The process is the server's child, not ours: waiting on it raises
+    # ECHILD, so liveness is probed with signal 0 instead.
+    class GroupTerminator
+      GRACE_SECONDS = 2.0
+      KILL_SECONDS = 1.0
+      POLL_SECONDS = 0.02
+
+      def initialize(killer: GroupKiller, clock: Process.method(:clock_gettime), pauser: Kernel.method(:sleep))
+        @killer = killer
+        @clock = clock
+        @pauser = pauser
+      end
+
+      # @param leader [Boolean] false when the process already ended and only
+      #   members of its group are left
+      def terminate(pid, leader: true)
+        @killer.kill(pid, :SIGTERM, leader:)
+        return if gone_within?(pid, GRACE_SECONDS, leader)
+
+        @killer.kill(pid, :SIGKILL, leader:)
+        gone_within?(pid, KILL_SECONDS, leader)
+      end
+
+      private
+
+      def gone_within?(pid, seconds, leader)
+        deadline = now + seconds
+        while @killer.alive?(pid, leader:)
+          return false if now >= deadline
+
+          @pauser.call(POLL_SECONDS)
+        end
+        true
+      end
+
+      def now
+        @clock.call(Process::CLOCK_MONOTONIC)
+      end
+    end
+  end
+end