mutation_tester 1.5.1 → 1.6.0

data/lib/mutation_tester/mutation_runner.rb

@@ -86,7 +86,7 @@ module MutationTester
           end
 
           result = run_single_mutation(mutation, :in_memory)
-          progress_callback.call(mutation, index + 1) if progress_callback
+          progress_callback.call(mutation, index + 1, result) if progress_callback
           results << result
           break if stop_early?(result)
         end
@@ -113,20 +113,10 @@ module MutationTester
 
       warn "[MutationTester] In-memory execution selected (parallel, #{pool.size} preloaded workers, zero file writes per mutant)."
       announce_load_time_routing(mutations)
-      total = mutations.size
-      completed_count = 0
       collected = []
       project_root = discoverable_project_root
       reserve_fallback_shadow_root
-
-      report_progress = lambda do |_item, _index, result|
-        completed_count += 1
-        collected << result
-        if progress_callback && (completed_count.even? || completed_count == total)
-          progress_callback.call(nil, completed_count)
-        end
-        raise Parallel::Break if stop_early?(result)
-      end
+      report_progress = parallel_progress_reporter(collected, progress_callback)
 
       mapped = with_parallel_interrupt_silenced do
         Parallel.map(mutations, in_processes: pool.size, finish: report_progress) do |mutation|
@@ -141,21 +131,11 @@ module MutationTester
     end
 
     def run_in_shadow_parallel(mutations, &progress_callback)
-      total = mutations.size
-      completed_count = 0
       collected = []
       project_root = find_project_root
       shadow_run_root
-      prepare_worker_preloads(total)
-
-      report_progress = lambda do |_item, _index, result|
-        completed_count += 1
-        collected << result
-        if progress_callback && (completed_count.even? || completed_count == total)
-          progress_callback.call(nil, completed_count)
-        end
-        raise Parallel::Break if stop_early?(result)
-      end
+      prepare_worker_preloads(mutations.size)
+      report_progress = parallel_progress_reporter(collected, progress_callback)
 
       mapped = with_parallel_interrupt_silenced do
         Parallel.map(mutations, in_processes: @config.parallel_processes, finish: report_progress) do |mutation|
@@ -180,7 +160,7 @@ module MutationTester
       results = []
       mutations.each_with_index do |mutation, index|
         result = run_single_mutation(mutation, :shadow, project_root)
-        progress_callback.call(mutation, index + 1) if progress_callback
+        progress_callback.call(mutation, index + 1, result) if progress_callback
         results << result
         break if stop_early?(result)
       end
@@ -194,7 +174,7 @@ module MutationTester
       results = []
       mutations.each_with_index do |mutation, index|
         result = run_single_mutation(mutation, :in_place)
-        progress_callback.call(mutation, index + 1) if progress_callback
+        progress_callback.call(mutation, index + 1, result) if progress_callback
         results << result
         break if stop_early?(result)
       end
@@ -218,6 +198,7 @@ module MutationTester
         status: :survived,
         description: mutation[:description]
       }.tap do |result|
+        result[:killed_by] = [] if @config.kill_matrix
         if unparseable?(mutation[:code])
           mark_stillborn(result)
         elsif strategy == :in_memory && mutation[:in_memory_safe] == false
@@ -370,17 +351,20 @@ module MutationTester
       runner = active_in_memory_runner
       return in_memory_worker_fallback(mutation, result, project_root, 'the in-memory worker is unavailable') unless runner&.ready?
 
-      outcome = runner.execute_in_memory(
-        source: mutation[:code],
-        path: @source_file,
-        timeout: @config.effective_timeout,
-        chdir: Dir.pwd
-      )
+      outcome, tests = recording_failed_tests do |recorder_env|
+        runner.execute_in_memory(
+          source: mutation[:code],
+          path: @source_file,
+          timeout: @config.effective_timeout,
+          chdir: Dir.pwd,
+          env: recorder_env
+        )
+      end
 
       if outcome.status == 'error'
         in_memory_apply_fallback(mutation, result, project_root, outcome.message || 'in-memory application failed')
       else
-        apply_outcome(result, TestCommand::Result.new(outcome.status == 'pass', outcome.status == 'timeout'))
+        apply_outcome(result, TestCommand::Result.new(outcome.status == 'pass', outcome.status == 'timeout', nil, tests))
       end
     rescue MutationTester::Error => e
       return in_memory_worker_fallback(mutation, result, project_root, e.message) if project_root
@@ -402,6 +386,7 @@ module MutationTester
 
     def subset_filters(mutation)
       return [] unless @config.test_selection
+      return [] if @config.kill_matrix
       return [] unless detect_test_framework(@spec_file) == :rspec
 
       method_name = mutation[:method_name].to_s
@@ -417,6 +402,10 @@ module MutationTester
       FrameworkDetector.detect(spec_path)
     end
 
+    def recording_root
+      @recording_root ||= discoverable_project_root || File.dirname(@spec_file)
+    end
+
     private
 
     def with_parallel_interrupt_silenced
@@ -442,7 +431,7 @@ module MutationTester
         return 'the in-memory worker failed to preload the environment'
       end
 
-      preloaded, message = runner.preload(@spec_file, chdir: Dir.pwd, stop_on_first_failure: true)
+      preloaded, message = runner.preload(@spec_file, chdir: Dir.pwd, stop_on_first_failure: stop_on_first_failure?)
       unless preloaded
         runner.shutdown
         return "the spec file could not be preloaded (#{message})"
@@ -479,8 +468,8 @@ module MutationTester
 
     def fall_back_to_file_based(reason, mutations, completed: 0, &progress_callback)
       announce_file_based_fallback(reason)
-      offset_callback = progress_callback && lambda do |mutation, index|
-        progress_callback.call(mutation, completed + index)
+      offset_callback = progress_callback && lambda do |mutation, index, result|
+        progress_callback.call(mutation, completed + index, result)
       end
       run_file_based_series(mutations, &offset_callback)
     end
@@ -615,6 +604,16 @@ module MutationTester
       @config.fail_fast && result[:status] == :survived
     end
 
+    def parallel_progress_reporter(collected, progress_callback)
+      completed_count = 0
+      lambda do |_item, _index, result|
+        completed_count += 1
+        collected << result
+        progress_callback.call(nil, completed_count, result) if progress_callback
+        raise Parallel::Break if stop_early?(result)
+      end
+    end
+
     def backup_path
       self.class.backup_path_for(@source_file)
     end
@@ -642,6 +641,17 @@ module MutationTester
         result[:status] = :killed
       end
       result[:kill_phase] = phase if result[:killed] && phase
+      result[:killed_by] = TestRecorder.failed_ids(outcome.tests) if @config.kill_matrix && result[:status] == :killed
+    end
+
+    def stop_on_first_failure?
+      !@config.kill_matrix
+    end
+
+    def recording_failed_tests(&block)
+      return [block.call(nil), nil] unless @config.kill_matrix
+
+      TestRecorder.capture(scope: :failures, root: recording_root, &block)
     end
 
     def mark_stillborn(result)
@@ -652,6 +662,7 @@ module MutationTester
       result[:killed] = false
       result[:timeout] = false
       result[:status] = :error
+      result[:killed_by] = [] if result.key?(:killed_by)
       result[:description] = "Error: #{error.message}"
     end
 
@@ -680,7 +691,9 @@ module MutationTester
         runner: @config.runner,
         example_filters: example_filters,
         worker_env_var: @config.worker_env_var,
-        stop_on_first_failure: true
+        stop_on_first_failure: stop_on_first_failure?,
+        record: @config.kill_matrix ? :failures : nil,
+        record_root: @config.kill_matrix ? recording_root : nil
       )
     end
   end

data/lib/mutation_tester/progress_display.rb

@@ -1,43 +1,84 @@
 module MutationTester
   class ProgressDisplay
-    attr_reader :total, :current, :current_mutation
-
-    def initialize(total, config, output_stream: $stdout.clone)
+    MONOTONIC_CLOCK = -> { Process.clock_gettime(Process::CLOCK_MONOTONIC) }
+    ANSI_STYLE = /\e\[[\d;]*m/
+    SEPARATOR = ' | '
+    BAR_WIDTH = 20
+    LAYOUTS = [
+      { bar_width: BAR_WIDTH, compact: false },
+      { bar_width: BAR_WIDTH / 2, compact: false },
+      { bar_width: BAR_WIDTH / 2, compact: true },
+      { bar_width: 0, compact: true }
+    ].freeze
+    ESTIMATE_MIN_PROCESSED = 5
+    ESTIMATE_MIN_ELAPSED = 10
+
+    attr_reader :total, :current, :survived, :timed_out, :errored
+
+    def initialize(total, config, output_stream: $stdout.clone, clock: MONOTONIC_CLOCK)
       @output_stream = output_stream
       @total = total
       @current = 0
-      @current_mutation = nil
+      @survived = 0
+      @timed_out = 0
+      @errored = 0
       @config = config
-      @start_time = Time.now
+      @clock = clock
+      @start_time = @clock.call
+      @sample_start_time = nil
+      @sample_start_count = nil
+      @rendered_length = 0
       @spinner_frames = ['⠋', '⠙', '⠹', '⠸', '⠼', '⠴', '⠦', '⠧', '⠇', '⠏']
       @spinner_index = 0
       @lock = Mutex.new
       @spinner_thread = nil
       @stop_spinner = false
+      @halted = false
 
       start_spinner if @config.show_progress
     end
 
-    def update(mutation, index)
+    def update(index, result = nil)
       @lock.synchronize do
+        @sample_start_time ||= @clock.call
+        @sample_start_count ||= index
         @current = index
-        @current_mutation = mutation
+        tally(result) if result
       end
     end
 
     def finish
+      halt do
+        elapsed = @clock.call - @start_time
+        @output_stream.puts "  #{Rainbow("Completed in #{format_duration(elapsed)}").green}"
+      end
+    end
+
+    def stop
+      halt
+    end
+
+    private
+
+    def halt
       @lock.synchronize do
         @stop_spinner = true
-        if @config.show_progress
+        if @config.show_progress && !@halted
           clear_line
-          elapsed = Time.now - @start_time
-          @output_stream.puts "  #{Rainbow("Completed in #{format_duration(elapsed)}").green}"
+          yield if block_given?
         end
+        @halted = true
       end
       stop_spinner
     end
 
-    private
+    def tally(result)
+      case Reporters::BaseReporter.status_for(result)
+      when :survived then @survived += 1
+      when :timeout then @timed_out += 1
+      when :error then @errored += 1
+      end
+    end
 
     def render(already_locked = false)
       return unless @config.show_progress
@@ -51,22 +92,88 @@ module MutationTester
 
     def _render_internal
       clear_line
+      line = fitted_line
+      @rendered_length = visible_length(line)
+      @output_stream.print line
+      @output_stream.flush
+    end
+
+    def fitted_line
+      limit = line_limit
+      now = @clock.call
+      segments = []
+      LAYOUTS.each do |layout|
+        segments = line_segments(now, **layout)
+        break if fits?(segments, limit)
+      end
+      segments.pop until fits?(segments, limit)
+      segments.join(SEPARATOR)
+    end
+
+    def fits?(segments, limit)
+      limit.nil? || visible_length(segments.join(SEPARATOR)) <= limit
+    end
 
+    def line_segments(now, bar_width:, compact:)
+      processed = Rainbow("#{@current}/#{@total}").cyan
+
+      [
+        progress_segment(bar_width),
+        compact ? processed : "#{processed} processed",
+        time_segment(now, compact),
+        tally_segment
+      ]
+    end
+
+    def progress_segment(bar_width)
       progress = (@current.to_f / @total * 100).round(1)
-      progress_bar = create_progress_bar(progress)
-      spinner = @spinner_frames[@spinner_index]
+      spinner = Rainbow(@spinner_frames[@spinner_index]).magenta
+      percentage = Rainbow("#{progress}%").bright
+      return "#{spinner} #{percentage}" if bar_width.zero?
 
-      mutation_info = @current_mutation ? " | #{@current_mutation[:type]} | line #{@current_mutation[:line]}" : ''
+      "#{spinner} [#{create_progress_bar(progress, bar_width)}] #{percentage}"
+    end
 
-      line = "#{Rainbow(spinner).magenta} [#{progress_bar}] #{Rainbow(progress.to_s + "%").bright} | " \
-             "#{Rainbow("#{@current}/#{@total}").cyan} mutations processed#{mutation_info}"
+    def time_segment(now, compact)
+      elapsed = format_duration((now - @start_time).floor)
+      remaining = estimated_remaining(now)
+      return "elapsed #{elapsed}" unless remaining
 
-      @output_stream.print line
-      @output_stream.flush
+      left = format_duration(remaining.ceil)
+      compact ? "#{elapsed}, ~#{left} left" : "elapsed #{elapsed}, remaining ~#{left}"
+    end
+
+    def tally_segment
+      survived = Rainbow("#{@survived} survived").color(@survived.zero? ? :green : :red)
+      segment = "#{survived}, #{@timed_out} timed out"
+      @errored.zero? ? segment : "#{segment}, #{Rainbow("#{@errored} errored").red}"
+    end
+
+    def estimated_remaining(now)
+      return if @sample_start_time.nil? || @current >= @total
+
+      sampled = @current - @sample_start_count
+      sample_elapsed = now - @sample_start_time
+      return unless sampled.positive?
+      return if sampled < ESTIMATE_MIN_PROCESSED && sample_elapsed < ESTIMATE_MIN_ELAPSED
+
+      sample_elapsed.fdiv(sampled) * (@total - @current)
+    end
+
+    def visible_length(text)
+      text.gsub(ANSI_STYLE, '').length
     end
 
-    def create_progress_bar(percentage)
-      width = 20
+    def line_limit
+      return unless @output_stream.respond_to?(:winsize) && @output_stream.tty?
+
+      columns = @output_stream.winsize[1]
+      columns - 1 if columns.positive?
+    rescue SystemCallError
+      nil
+    end
+
+    def create_progress_bar(percentage, width = BAR_WIDTH)
       filled = (percentage / 100.0 * width).round
       empty = width - filled
 
@@ -75,21 +182,18 @@ module MutationTester
 
     def clear_line
       @output_stream.print "\r"
-      @output_stream.print ' ' * 100
+      @output_stream.print ' ' * @rendered_length
       @output_stream.print "\r"
     end
 
     def format_duration(seconds)
-      if seconds < 60
-        "#{seconds.round(1)}s"
-      elsif seconds < 3600
-        minutes = (seconds / 60).floor
-        secs = (seconds % 60).round
-        "#{minutes}m #{secs}s"
+      return "#{seconds.round(1)}s" if seconds.round(1) < 60
+
+      whole = seconds.round
+      if whole < 3600
+        "#{whole / 60}m #{whole % 60}s"
       else
-        hours = (seconds / 3600).floor
-        minutes = ((seconds % 3600) / 60).floor
-        "#{hours}h #{minutes}m"
+        "#{whole / 3600}h #{whole % 3600 / 60}m"
       end
     end
 

data/lib/mutation_tester/reporters/base_reporter.rb

@@ -17,8 +17,9 @@ module MutationTester
         result[:status] || (result[:killed] ? :killed : :survived)
       end
 
-      def initialize(results, source_file, spec_file, config, interrupted: false)
+      def initialize(results, source_file, spec_file, config, interrupted: false, tests: [])
         @results = results
+        @tests = Array(tests)
         @source_file = source_file
         @spec_file = spec_file
         @config = config

data/lib/mutation_tester/reporters/batch_json_reporter.rb

@@ -63,7 +63,8 @@ module MutationTester
             File.expand_path(entry.source_file),
             File.expand_path(entry.spec_file),
             @config,
-            interrupted: !!entry.interrupted
+            interrupted: !!entry.interrupted,
+            tests: entry.tests
           ).report_data
         end
       end

data/lib/mutation_tester/reporters/console_reporter.rb

@@ -89,6 +89,7 @@ module MutationTester
 
       def selection_stats_available?
         return false unless @config.test_selection && FrameworkDetector.detect(@spec_file) == :rspec
+        return false if @config.kill_matrix
 
         @config.runner != :in_memory || @results.any? { |result| result.key?(:kill_phase) }
       end

data/lib/mutation_tester/reporters/json_reporter.rb

@@ -21,6 +21,7 @@ module MutationTester
         {
           schema_version: SCHEMA_VERSION,
           interrupted: interrupted?,
+          **kill_matrix_fields,
           metadata: {
             version: MutationTester::VERSION,
             generated_at: Time.now.iso8601,
@@ -47,6 +48,12 @@ module MutationTester
 
       private
 
+      def kill_matrix_fields
+        return {} unless @config.kill_matrix
+
+        { kill_matrix: true, tests: @tests.map { |test| test.slice(:id, :name, :line, :status) } }
+      end
+
       def mutation_entry(result)
         status = status_of(result)
         entry = result.merge(status: status)

data/lib/mutation_tester/test_command.rb

@@ -1,8 +1,9 @@
 require 'tempfile'
+require_relative 'test_recorder'
 
 module MutationTester
   class TestCommand
-    Result = Struct.new(:passed, :timed_out, :output) do
+    Result = Struct.new(:passed, :timed_out, :output, :tests) do
       alias_method :passed?, :passed
       alias_method :timed_out?, :timed_out
     end
@@ -13,7 +14,7 @@ module MutationTester
     attr_reader :spec_file, :framework, :use_bundle_exec, :example_filters
 
     def initialize(spec_file, use_bundle_exec:, framework: nil, runner: :spawn, example_filters: [], worker_env_var: nil,
-                   stop_on_first_failure: false)
+                   stop_on_first_failure: false, record: nil, record_root: nil)
       @spec_file = spec_file
       @framework = framework || self.class.detect_framework(spec_file)
       @use_bundle_exec = use_bundle_exec
@@ -21,10 +22,12 @@ module MutationTester
       @example_filters = @framework == :rspec ? Array(example_filters) : []
       @worker_env_var = worker_env_var
       @stop_on_first_failure = stop_on_first_failure
+      @record = record
+      @record_root = record_root
     end
 
     def argv
-      parts = [runner, *interpreter_args, spec_file, *filter_args, *fail_fast_args]
+      parts = [runner, *interpreter_args, spec_file, *filter_args, *fail_fast_args, *recorder_args]
       @use_bundle_exec ? ['bundle', 'exec', *parts] : parts
     end
 
@@ -41,29 +44,13 @@ module MutationTester
     end
 
     def run(timeout: nil, chdir: nil, capture: false, mirror_of: nil)
-      if fork_execution?
-        fork_runner = ForkRunner.acquire(use_bundle_exec: @use_bundle_exec, framework: @framework)
-        if fork_runner
-          return fork_runner.execute(
-            spec_file,
-            timeout: timeout,
-            chdir: chdir || Dir.pwd,
-            capture: capture,
-            args: filter_args,
-            stop_on_first_failure: @stop_on_first_failure,
-            mirror_of: mirror_of,
-            env: worker_env_overrides
-          )
-        end
-      end
-
-      return run_captured(timeout: timeout, chdir: chdir) if capture
-
-      spawn_options = { pgroup: true, %i[out err] => File::NULL }
-      spawn_options[:chdir] = chdir if chdir
+      return execute(timeout: timeout, chdir: chdir, capture: capture, mirror_of: mirror_of) unless @record
 
-      pid = Process.spawn(*spawn_argv, spawn_options)
-      wait_with_deadline(pid, timeout)
+      result, tests = TestRecorder.capture(scope: @record, root: mirror_of ? chdir : @record_root) do |recorder_env|
+        execute(timeout: timeout, chdir: chdir, capture: capture, mirror_of: mirror_of, recorder_env: recorder_env)
+      end
+      result.tests = tests
+      result
     end
 
     def fork_execution?
@@ -93,9 +80,42 @@ module MutationTester
 
     private
 
-    def spawn_argv
+    def execute(timeout:, chdir:, capture:, mirror_of:, recorder_env: nil)
+      env = child_env(recorder_env)
+      if fork_execution?
+        fork_runner = ForkRunner.acquire(use_bundle_exec: @use_bundle_exec, framework: @framework)
+        if fork_runner
+          return fork_runner.execute(
+            spec_file,
+            timeout: timeout,
+            chdir: chdir || Dir.pwd,
+            capture: capture,
+            args: filter_args,
+            stop_on_first_failure: @stop_on_first_failure,
+            mirror_of: mirror_of,
+            env: env
+          )
+        end
+      end
+
+      return run_captured(timeout: timeout, chdir: chdir, env: env) if capture
+
+      spawn_options = { pgroup: true, %i[out err] => File::NULL }
+      spawn_options[:chdir] = chdir if chdir
+
+      pid = Process.spawn(*spawn_argv(env), spawn_options)
+      wait_with_deadline(pid, timeout)
+    end
+
+    def spawn_argv(env)
+      env ? [env, *argv] : argv
+    end
+
+    def child_env(recorder_env)
       overrides = worker_env_overrides
-      overrides ? [overrides, *argv] : argv
+      return overrides unless recorder_env
+
+      (overrides || {}).merge(recorder_env)
     end
 
     def worker_env_overrides
@@ -114,9 +134,18 @@ module MutationTester
     end
 
     def interpreter_args
-      return [] unless @stop_on_first_failure && @framework == :minitest
+      return [] unless @framework == :minitest
+
+      args = []
+      args += ['-r', MINITEST_FAIL_FAST_PATH] if @stop_on_first_failure
+      args += ['-r', TestRecorder::MINITEST_HOOK_PATH] if @record
+      args
+    end
+
+    def recorder_args
+      return [] unless @record && @framework == :rspec
 
-      ['-r', MINITEST_FAIL_FAST_PATH]
+      ['--require', TestRecorder::RSPEC_HOOK_PATH]
     end
 
     def fail_fast_args
@@ -125,12 +154,12 @@ module MutationTester
       ['--fail-fast']
     end
 
-    def run_captured(timeout:, chdir:)
+    def run_captured(timeout:, chdir:, env:)
       log = Tempfile.new(['mutation_tester_baseline', '.log'])
       spawn_options = { pgroup: true, %i[out err] => log.path }
       spawn_options[:chdir] = chdir if chdir
 
-      pid = Process.spawn(*spawn_argv, spawn_options)
+      pid = Process.spawn(*spawn_argv(env), spawn_options)
       result = wait_with_deadline(pid, timeout)
       result.output = File.read(log.path)
       result

data/lib/mutation_tester/test_recorder/minitest_hook.rb

@@ -0,0 +1,55 @@
+# frozen_string_literal: true
+
+require_relative '../test_recorder'
+require_relative '../minitest_load_hook'
+
+MutationTester::MinitestLoadHook.on_load do
+  module MutationTester
+    module TestRecorder
+      module MinitestHook
+        PLUGIN_NAME = :mutation_tester_test_recorder
+
+        class Reporter < Minitest::AbstractReporter
+          def record(result)
+            return unless TestRecorder.active?
+
+            TestRecorder.write([entry_for(result)])
+          end
+
+          private
+
+          def entry_for(result)
+            owner = result.respond_to?(:klass) && result.klass ? result.klass : result.class.name
+            {
+              id: "#{owner}##{result.name}",
+              name: result.name,
+              line: line_of(result),
+              status: status_of(result)
+            }
+          end
+
+          def line_of(result)
+            location = result.respond_to?(:source_location) ? result.source_location : nil
+            location.is_a?(Array) ? location[1] : nil
+          end
+
+          def status_of(result)
+            return 'skipped' if result.skipped?
+
+            result.passed? ? 'passed' : 'failed'
+          end
+        end
+      end
+    end
+  end
+
+  module Minitest
+    def self.plugin_mutation_tester_test_recorder_init(_options)
+      reporter << MutationTester::TestRecorder::MinitestHook::Reporter.new
+    end
+  end
+
+  unless Minitest.extensions.include?(MutationTester::TestRecorder::MinitestHook::PLUGIN_NAME)
+    Minitest.register_plugin(MutationTester::TestRecorder::MinitestHook::PLUGIN_NAME)
+  end
+end