ci-queue 0.82.0 → 0.83.0

data/lib/minitest/queue/worker_profile_reporter.rb

@@ -0,0 +1,77 @@
+# frozen_string_literal: true
+
+module Minitest
+  module Queue
+    class WorkerProfileReporter
+      def initialize(supervisor, out: $stdout)
+        @supervisor = supervisor
+        @out = out
+      end
+
+      def print_summary
+        return unless CI::Queue.debug?
+
+        expected = @supervisor.workers_count
+        profiles = {}
+        3.times do
+          profiles = @supervisor.build.worker_profiles
+          break if profiles.size >= expected
+          sleep 1
+        end
+        return if profiles.empty?
+
+        sorted = profiles.values.sort_by { |p| p['worker_id'].to_s }
+        mode = sorted.first&.dig('mode') || 'unknown'
+
+        @out.puts
+        @out.puts "Worker profile summary (#{sorted.size} workers, mode: #{mode}):"
+        @out.puts "  %-12s %-12s %8s %14s %14s %14s %14s %10s" % ['Worker', 'Role', 'Tests', '1st Test', 'Wall Clock', 'Load Tests', 'File Load', 'Memory']
+        @out.puts "  #{'-' * 100}"
+
+        sorted.each do |profile|
+          @out.puts format_profile_row(profile)
+        end
+
+        print_first_test_summary(sorted)
+      rescue StandardError
+        # Don't fail the build if profile printing fails
+      end
+
+      private
+
+      def format_profile_row(profile)
+        tests = profile['tests_run'] ? profile['tests_run'].to_s : 'n/a'
+        first_test = profile['time_to_first_test'] ? "#{profile['time_to_first_test']}s" : 'n/a'
+        wall = "#{profile['total_wall_clock']}s"
+        load_tests = profile['load_tests_duration'] ? "#{profile['load_tests_duration']}s" : 'n/a'
+        files = if profile['files_loaded'] && profile['total_files']
+          "#{profile['file_load_time']}s (#{profile['files_loaded']}/#{profile['total_files']})"
+        elsif profile['file_load_time']
+          "#{profile['file_load_time']}s"
+        else
+          'n/a'
+        end
+        mem = profile['memory_rss_kb'] ? "#{(profile['memory_rss_kb'] / 1024.0).round(0)} MB" : 'n/a'
+
+        "  %-12s %-12s %8s %14s %14s %14s %14s %10s" % [
+          profile['worker_id'], profile['role'], tests, first_test, wall, load_tests, files, mem
+        ]
+      end
+
+      def print_first_test_summary(sorted)
+        leaders = sorted.select { |p| p['role'] == 'leader' }
+        non_leaders = sorted.select { |p| p['role'] == 'non-leader' }
+        return unless leaders.any? && non_leaders.any?
+
+        leader_first = leaders.filter_map { |p| p['time_to_first_test'] }.min
+        nl_firsts = non_leaders.filter_map { |p| p['time_to_first_test'] }
+        return unless leader_first && nl_firsts.any?
+
+        avg_nl = (nl_firsts.sum / nl_firsts.size).round(2)
+        @out.puts
+        @out.puts "  Leader time to 1st test: #{leader_first}s"
+        @out.puts "  Avg non-leader time to 1st test: #{avg_nl}s"
+      end
+    end
+  end
+end

data/lib/minitest/queue.rb

@@ -2,6 +2,7 @@
 require 'shellwords'
 require 'minitest'
 require 'minitest/reporters'
+require 'concurrent/map'
 
 require 'minitest/queue/failure_formatter'
 require 'minitest/queue/error_report'
@@ -106,6 +107,10 @@ module Minitest
     attr_accessor :start_timestamp, :finish_timestamp
   end
 
+  module ResultMetadata
+    attr_accessor :queue_id, :queue_entry
+  end
+
   module Queue
     extend ::CI::Queue::OutputHelpers
     attr_writer :run_command_formatter, :project_root
@@ -156,19 +161,29 @@ module Minitest
 
       def run(reporter, *)
         rescue_run_errors do
-          queue.poll do |example|
-            result = queue.with_heartbeat(example.id) do
-              example.run
+          begin
+            queue.poll do |example|
+              result = queue.with_heartbeat(example.queue_entry) do
+                example.run
+              end
+
+              handle_test_result(reporter, example, result)
             end
 
-            handle_test_result(reporter, example, result)
+            report_load_stats(queue)
+          ensure
+            store_worker_profile(queue)
+            queue.stop_heartbeat!
           end
-
-          queue.stop_heartbeat!
         end
       end
 
       def handle_test_result(reporter, example, result)
+        if result.respond_to?(:queue_id=)
+          result.queue_id = example.id
+          result.queue_entry = example.queue_entry if result.respond_to?(:queue_entry=)
+        end
+
         failed = !(result.passed? || result.skipped?)
 
         if example.flaky?
@@ -194,6 +209,85 @@ module Minitest
 
       private
 
+      def report_load_stats(queue)
+        return unless CI::Queue.debug?
+        return unless queue.respond_to?(:file_loader)
+        return unless queue.respond_to?(:config) && queue.config.lazy_load
+
+        loader = queue.file_loader
+        return if loader.load_stats.empty?
+
+        total_time = loader.total_load_time
+        file_count = loader.load_stats.size
+        average = file_count.zero? ? 0 : (total_time / file_count)
+
+        puts
+        puts "File loading stats:"
+        puts "  Total time: #{total_time.round(2)}s"
+        puts "  Files loaded: #{file_count}"
+        puts "  Average: #{average.round(3)}s per file"
+
+        slowest = loader.slowest_files(5)
+        return if slowest.empty?
+
+        puts "  Slowest files:"
+        slowest.each do |file_path, duration|
+          puts "    #{duration.round(3)}s - #{Minitest::Queue.relative_path(file_path)}"
+        end
+      end
+
+      def store_worker_profile(queue)
+        debug = CI::Queue.debug?
+        return unless queue.respond_to?(:config)
+        config = queue.config
+
+        run_start = Minitest::Queue::Runner.run_start
+        return unless run_start
+
+        run_end = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+        profile = {
+          'worker_id' => config.worker_id,
+          'mode' => config.lazy_load ? 'lazy' : 'eager',
+          'role' => queue.master? ? 'leader' : 'non-leader',
+          'total_wall_clock' => (run_end - run_start).round(2),
+        }
+
+        first_test = queue.respond_to?(:first_reserve_at) ? queue.first_reserve_at : nil
+        profile['time_to_first_test'] = (first_test - run_start).round(2) if first_test
+
+        tests_run = queue.rescue_connection_errors { queue.worker_queue_length } if queue.respond_to?(:worker_queue_length)
+        profile['tests_run'] = tests_run.to_i if tests_run
+
+        load_tests_duration = Minitest::Queue::Runner.load_tests_duration
+        profile['load_tests_duration'] = load_tests_duration.round(2) if load_tests_duration
+
+        if queue.respond_to?(:file_loader) && queue.file_loader.load_stats.any?
+          loader = queue.file_loader
+          profile['files_loaded'] = loader.load_stats.size
+          profile['file_load_time'] = loader.total_load_time.round(2)
+        end
+
+        profile['total_files'] = Minitest::Queue::Runner.total_files if Minitest::Queue::Runner.total_files
+
+        rss_kb = begin
+          if File.exist?("/proc/#{Process.pid}/statm")
+            pages = Integer(File.read("/proc/#{Process.pid}/statm").split[1])
+            pages * 4
+          else
+            Integer(`ps -o rss= -p #{Process.pid}`.strip)
+          end
+        rescue
+          nil
+        end
+        profile['memory_rss_kb'] = rss_kb if rss_kb
+
+        queue.rescue_connection_errors do
+          queue.build.record_worker_profile(profile)
+        end
+      rescue => e
+        puts "WARNING: Failed to store worker profile: #{e.message}" if debug
+      end
+
       def rescue_run_errors(&block)
         block.call
       rescue Errno::EPIPE
@@ -232,6 +326,10 @@ module Minitest
         @id ||= "#{@runnable}##{@method_name}".freeze
       end
 
+      def queue_entry
+        id
+      end
+
       def <=>(other)
         id <=> other.id
       end
@@ -270,6 +368,171 @@ module Minitest
       end
     end
 
+    class LazySingleExample
+      attr_reader :class_name, :method_name, :file_path
+
+      def initialize(class_name, method_name, file_path, loader:, resolver:, load_error: nil, queue_entry: nil)
+        @class_name = class_name
+        @method_name = method_name
+        @file_path = file_path
+        @loader = loader
+        @resolver = resolver
+        @load_error = load_error
+        @queue_entry_override = queue_entry
+        @runnable = nil
+      end
+
+      def id
+        @id ||= "#{@class_name}##{@method_name}".freeze
+      end
+
+      def queue_entry
+        @queue_entry ||= @queue_entry_override || CI::Queue::QueueEntry.format(id, file_path)
+      end
+
+      def <=>(other)
+        id <=> other.id
+      end
+
+      RUNNABLE_METHODS_TRIGGERED = Concurrent::Map.new # :nodoc:
+
+      def runnable
+        @runnable ||= begin
+          klass = @resolver.resolve(@class_name, file_path: @file_path, loader: @loader)
+          unless RUNNABLE_METHODS_TRIGGERED[klass]
+            klass.runnable_methods
+            RUNNABLE_METHODS_TRIGGERED[klass] = true
+          end
+
+          # If the method doesn't exist, the class may have been autoloaded by
+          # Zeitwerk without executing test-specific code (includes, helpers).
+          # Force load the file so all class-definition-time code executes.
+          unless klass.method_defined?(@method_name) || klass.private_method_defined?(@method_name)
+            if @file_path && @loader
+              @loader.load_file(@file_path)
+              RUNNABLE_METHODS_TRIGGERED.delete(klass)
+              klass.runnable_methods
+              RUNNABLE_METHODS_TRIGGERED[klass] = true
+            end
+          end
+
+          klass
+        end
+      end
+
+      def with_timestamps
+        start_timestamp = current_timestamp
+        result = yield
+        result
+      ensure
+        if result
+          result.start_timestamp = start_timestamp
+          result.finish_timestamp = current_timestamp
+        end
+      end
+
+      def run
+        with_timestamps do
+          if @load_error
+            build_error_result(@load_error)
+          elsif skip_stale_tests? && !(runnable.method_defined?(@method_name) || runnable.private_method_defined?(@method_name))
+            build_stale_skip_result
+          else
+            Minitest.run_one_method(runnable, @method_name)
+          end
+        rescue StandardError, ScriptError => error
+          build_error_result(error)
+        end
+      end
+
+      def flaky?
+        Minitest.queue.flaky?(self)
+      end
+
+      def source_location
+        return nil if @load_error
+
+        runnable.instance_method(@method_name).source_location
+      rescue NameError, NoMethodError, CI::Queue::FileLoadError, CI::Queue::ClassNotFoundError
+        nil
+      end
+
+      def marshal_dump
+        {
+          'class_name' => @class_name,
+          'method_name' => @method_name,
+          'file_path' => @file_path,
+          'load_error' => serialize_error(@load_error),
+          'queue_entry' => @queue_entry_override,
+        }
+      end
+
+      def marshal_load(payload)
+        @class_name = payload['class_name']
+        @method_name = payload['method_name']
+        @file_path = payload['file_path']
+        @load_error = deserialize_error(payload['load_error'])
+        @queue_entry_override = payload['queue_entry']
+        @loader = CI::Queue::FileLoader.new
+        @resolver = CI::Queue::ClassResolver
+        @runnable = nil
+        @id = nil
+        @queue_entry = nil
+      end
+
+      private
+
+      def serialize_error(error)
+        return nil unless error
+
+        {
+          'class' => error.class.name,
+          'message' => error.message,
+          'backtrace' => error.backtrace,
+        }
+      end
+
+      def deserialize_error(payload)
+        return nil unless payload
+
+        message = "#{payload['class']}: #{payload['message']}"
+        error = StandardError.new(message)
+        error.set_backtrace(payload['backtrace']) if payload['backtrace']
+        CI::Queue::FileLoadError.new(@file_path, error)
+      end
+
+      def build_error_result(error)
+        result_class = defined?(Minitest::Result) ? Minitest::Result : Minitest::Test
+        result = result_class.new(@method_name)
+        result.klass = @class_name if result.respond_to?(:klass=)
+        result.source_location = [@file_path || 'unknown', -1] if result.respond_to?(:source_location=)
+        result.failures << Minitest::UnexpectedError.new(error)
+        result
+      end
+
+      def skip_stale_tests?
+        Minitest.queue&.respond_to?(:config) && Minitest.queue.config.skip_stale_tests
+      end
+
+      def build_stale_skip_result
+        $stderr.puts "[ci-queue] Skipping stale preresolved entry: #{@class_name}##{@method_name} " \
+          "(method no longer exists in #{@file_path || 'unknown file'})"
+
+        result_class = defined?(Minitest::Result) ? Minitest::Result : Minitest::Test
+        result = result_class.new(@method_name)
+        result.klass = @class_name if result.respond_to?(:klass=)
+        result.source_location = [@file_path || 'unknown', -1] if result.respond_to?(:source_location=)
+        result.failures << Minitest::Skip.new(
+          "[ci-queue] Stale preresolved entry: #{@class_name}##{@method_name} no longer exists"
+        )
+        result
+      end
+
+      def current_timestamp
+        CI::Queue.time_now.to_i
+      end
+    end
+
     attr_accessor :queue
 
     def queue_reporters=(reporters)
@@ -310,10 +573,12 @@ if defined? Minitest::Result
   Minitest::Result.prepend(Minitest::Requeueing)
   Minitest::Result.prepend(Minitest::Flakiness)
   Minitest::Result.prepend(Minitest::WithTimestamps)
+  Minitest::Result.prepend(Minitest::ResultMetadata)
 else
   Minitest::Test.prepend(Minitest::Requeueing)
   Minitest::Test.prepend(Minitest::Flakiness)
   Minitest::Test.prepend(Minitest::WithTimestamps)
+  Minitest::Test.prepend(Minitest::ResultMetadata)
 
   module MinitestBackwardCompatibility
     def source_location

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: ci-queue
 version: !ruby/object:Gem::Version
-  version: 0.82.0
+  version: 0.83.0
 platform: ruby
 authors:
 - Jean Boussier
@@ -190,12 +190,16 @@ files:
 - lib/ci/queue/bisect.rb
 - lib/ci/queue/build_record.rb
 - lib/ci/queue/circuit_breaker.rb
+- lib/ci/queue/class_resolver.rb
 - lib/ci/queue/common.rb
 - lib/ci/queue/configuration.rb
 - lib/ci/queue/file.rb
+- lib/ci/queue/file_loader.rb
 - lib/ci/queue/grind.rb
 - lib/ci/queue/output_helpers.rb
+- lib/ci/queue/queue_entry.rb
 - lib/ci/queue/redis.rb
+- lib/ci/queue/redis/_entry_helpers.lua
 - lib/ci/queue/redis/acknowledge.lua
 - lib/ci/queue/redis/base.rb
 - lib/ci/queue/redis/build_record.rb
@@ -222,14 +226,18 @@ files:
 - lib/minitest/queue/grind_recorder.rb
 - lib/minitest/queue/grind_reporter.rb
 - lib/minitest/queue/junit_reporter.rb
+- lib/minitest/queue/lazy_entry_resolver.rb
+- lib/minitest/queue/lazy_test_discovery.rb
 - lib/minitest/queue/local_requeue_reporter.rb
 - lib/minitest/queue/order_reporter.rb
+- lib/minitest/queue/queue_population_strategy.rb
 - lib/minitest/queue/runner.rb
 - lib/minitest/queue/statsd.rb
 - lib/minitest/queue/test_data.rb
 - lib/minitest/queue/test_data_reporter.rb
 - lib/minitest/queue/test_time_recorder.rb
 - lib/minitest/queue/test_time_reporter.rb
+- lib/minitest/queue/worker_profile_reporter.rb
 - lib/minitest/reporters/bisect_reporter.rb
 - lib/minitest/reporters/statsd_reporter.rb
 - lib/rspec/queue.rb