ci-queue 0.82.0 → 0.84.0

data/lib/minitest/queue.rb

@@ -2,6 +2,7 @@
 require 'shellwords'
 require 'minitest'
 require 'minitest/reporters'
+require 'concurrent/map'
 
 require 'minitest/queue/failure_formatter'
 require 'minitest/queue/error_report'
@@ -106,6 +107,10 @@ module Minitest
     attr_accessor :start_timestamp, :finish_timestamp
   end
 
+  module ResultMetadata
+    attr_accessor :queue_id, :queue_entry
+  end
+
   module Queue
     extend ::CI::Queue::OutputHelpers
     attr_writer :run_command_formatter, :project_root
@@ -156,19 +161,29 @@ module Minitest
 
       def run(reporter, *)
         rescue_run_errors do
-          queue.poll do |example|
-            result = queue.with_heartbeat(example.id) do
-              example.run
+          begin
+            queue.poll do |example|
+              result = queue.with_heartbeat(example.queue_entry) do
+                example.run
+              end
+
+              handle_test_result(reporter, example, result)
             end
 
-            handle_test_result(reporter, example, result)
+            report_load_stats(queue)
+          ensure
+            store_worker_profile(queue)
+            queue.stop_heartbeat!
           end
-
-          queue.stop_heartbeat!
         end
       end
 
       def handle_test_result(reporter, example, result)
+        if result.respond_to?(:queue_id=)
+          result.queue_id = example.id
+          result.queue_entry = example.queue_entry if result.respond_to?(:queue_entry=)
+        end
+
         failed = !(result.passed? || result.skipped?)
 
         if example.flaky?
@@ -180,20 +195,102 @@ module Minitest
           # When we do a bisect, we don't care about the result other than the test we're running the bisect on
           result.mark_as_flaked!
           failed = false
+        end
+
+        if failed && CI::Queue.requeueable?(result) && queue.requeue(example.queue_entry)
+          result.requeue!
+          if CI::Queue.debug?
+            $stderr.puts "[ci-queue][requeue] test_id=#{example.id} error_class=#{result.failures.first&.class} error=#{result.failures.first&.message&.lines&.first&.chomp}"
+          end
         elsif failed
           queue.report_failure!
         else
           queue.report_success!
         end
-
-        if failed && CI::Queue.requeueable?(result) && queue.requeue(example)
-          result.requeue!
-        end
         reporter.record(result)
       end
 
       private
 
+      def report_load_stats(queue)
+        return unless CI::Queue.debug?
+        return unless queue.respond_to?(:file_loader)
+        return unless queue.respond_to?(:config) && queue.config.lazy_load
+
+        loader = queue.file_loader
+        return if loader.load_stats.empty?
+
+        total_time = loader.total_load_time
+        file_count = loader.load_stats.size
+        average = file_count.zero? ? 0 : (total_time / file_count)
+
+        puts
+        puts "File loading stats:"
+        puts "  Total time: #{total_time.round(2)}s"
+        puts "  Files loaded: #{file_count}"
+        puts "  Average: #{average.round(3)}s per file"
+
+        slowest = loader.slowest_files(5)
+        return if slowest.empty?
+
+        puts "  Slowest files:"
+        slowest.each do |file_path, duration|
+          puts "    #{duration.round(3)}s - #{Minitest::Queue.relative_path(file_path)}"
+        end
+      end
+
+      def store_worker_profile(queue)
+        debug = CI::Queue.debug?
+        return unless queue.respond_to?(:config)
+        config = queue.config
+
+        run_start = Minitest::Queue::Runner.run_start
+        return unless run_start
+
+        run_end = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+        profile = {
+          'worker_id' => config.worker_id,
+          'mode' => config.lazy_load ? 'lazy' : 'eager',
+          'role' => queue.master? ? 'leader' : 'non-leader',
+          'total_wall_clock' => (run_end - run_start).round(2),
+        }
+
+        first_test = queue.respond_to?(:first_reserve_at) ? queue.first_reserve_at : nil
+        profile['time_to_first_test'] = (first_test - run_start).round(2) if first_test
+
+        tests_run = queue.rescue_connection_errors { queue.worker_queue_length } if queue.respond_to?(:worker_queue_length)
+        profile['tests_run'] = tests_run.to_i if tests_run
+
+        load_tests_duration = Minitest::Queue::Runner.load_tests_duration
+        profile['load_tests_duration'] = load_tests_duration.round(2) if load_tests_duration
+
+        if queue.respond_to?(:file_loader) && queue.file_loader.load_stats.any?
+          loader = queue.file_loader
+          profile['files_loaded'] = loader.load_stats.size
+          profile['file_load_time'] = loader.total_load_time.round(2)
+        end
+
+        profile['total_files'] = Minitest::Queue::Runner.total_files if Minitest::Queue::Runner.total_files
+
+        rss_kb = begin
+          if File.exist?("/proc/#{Process.pid}/statm")
+            pages = Integer(File.read("/proc/#{Process.pid}/statm").split[1])
+            pages * 4
+          else
+            Integer(`ps -o rss= -p #{Process.pid}`.strip)
+          end
+        rescue
+          nil
+        end
+        profile['memory_rss_kb'] = rss_kb if rss_kb
+
+        queue.rescue_connection_errors do
+          queue.build.record_worker_profile(profile)
+        end
+      rescue => e
+        puts "WARNING: Failed to store worker profile: #{e.message}" if debug
+      end
+
       def rescue_run_errors(&block)
         block.call
       rescue Errno::EPIPE
@@ -232,6 +329,10 @@ module Minitest
         @id ||= "#{@runnable}##{@method_name}".freeze
       end
 
+      def queue_entry
+        @queue_entry ||= CI::Queue::QueueEntry.format(id, nil)
+      end
+
       def <=>(other)
         id <=> other.id
       end
@@ -270,6 +371,171 @@ module Minitest
       end
     end
 
+    class LazySingleExample
+      attr_reader :class_name, :method_name, :file_path
+
+      def initialize(class_name, method_name, file_path, loader:, resolver:, load_error: nil, queue_entry: nil)
+        @class_name = class_name
+        @method_name = method_name
+        @file_path = file_path
+        @loader = loader
+        @resolver = resolver
+        @load_error = load_error
+        @queue_entry_override = queue_entry
+        @runnable = nil
+      end
+
+      def id
+        @id ||= "#{@class_name}##{@method_name}".freeze
+      end
+
+      def queue_entry
+        @queue_entry ||= @queue_entry_override || CI::Queue::QueueEntry.format(id, file_path)
+      end
+
+      def <=>(other)
+        id <=> other.id
+      end
+
+      RUNNABLE_METHODS_TRIGGERED = Concurrent::Map.new # :nodoc:
+
+      def runnable
+        @runnable ||= begin
+          klass = @resolver.resolve(@class_name, file_path: @file_path, loader: @loader)
+          unless RUNNABLE_METHODS_TRIGGERED[klass]
+            klass.runnable_methods
+            RUNNABLE_METHODS_TRIGGERED[klass] = true
+          end
+
+          # If the method doesn't exist, the class may have been autoloaded by
+          # Zeitwerk without executing test-specific code (includes, helpers).
+          # Force load the file so all class-definition-time code executes.
+          unless klass.method_defined?(@method_name) || klass.private_method_defined?(@method_name)
+            if @file_path && @loader
+              @loader.load_file(@file_path)
+              RUNNABLE_METHODS_TRIGGERED.delete(klass)
+              klass.runnable_methods
+              RUNNABLE_METHODS_TRIGGERED[klass] = true
+            end
+          end
+
+          klass
+        end
+      end
+
+      def with_timestamps
+        start_timestamp = current_timestamp
+        result = yield
+        result
+      ensure
+        if result
+          result.start_timestamp = start_timestamp
+          result.finish_timestamp = current_timestamp
+        end
+      end
+
+      def run
+        with_timestamps do
+          if @load_error
+            build_error_result(@load_error)
+          elsif skip_stale_tests? && !(runnable.method_defined?(@method_name) || runnable.private_method_defined?(@method_name))
+            build_stale_skip_result
+          else
+            Minitest.run_one_method(runnable, @method_name)
+          end
+        rescue StandardError, ScriptError => error
+          build_error_result(error)
+        end
+      end
+
+      def flaky?
+        Minitest.queue.flaky?(self)
+      end
+
+      def source_location
+        return nil if @load_error
+
+        runnable.instance_method(@method_name).source_location
+      rescue NameError, NoMethodError, CI::Queue::FileLoadError, CI::Queue::ClassNotFoundError
+        nil
+      end
+
+      def marshal_dump
+        {
+          'class_name' => @class_name,
+          'method_name' => @method_name,
+          'file_path' => @file_path,
+          'load_error' => serialize_error(@load_error),
+          'queue_entry' => @queue_entry_override,
+        }
+      end
+
+      def marshal_load(payload)
+        @class_name = payload['class_name']
+        @method_name = payload['method_name']
+        @file_path = payload['file_path']
+        @load_error = deserialize_error(payload['load_error'])
+        @queue_entry_override = payload['queue_entry']
+        @loader = CI::Queue::FileLoader.new
+        @resolver = CI::Queue::ClassResolver
+        @runnable = nil
+        @id = nil
+        @queue_entry = nil
+      end
+
+      private
+
+      def serialize_error(error)
+        return nil unless error
+
+        {
+          'class' => error.class.name,
+          'message' => error.message,
+          'backtrace' => error.backtrace,
+        }
+      end
+
+      def deserialize_error(payload)
+        return nil unless payload
+
+        message = "#{payload['class']}: #{payload['message']}"
+        error = StandardError.new(message)
+        error.set_backtrace(payload['backtrace']) if payload['backtrace']
+        CI::Queue::FileLoadError.new(@file_path, error)
+      end
+
+      def build_error_result(error)
+        result_class = defined?(Minitest::Result) ? Minitest::Result : Minitest::Test
+        result = result_class.new(@method_name)
+        result.klass = @class_name if result.respond_to?(:klass=)
+        result.source_location = [@file_path || 'unknown', -1] if result.respond_to?(:source_location=)
+        result.failures << Minitest::UnexpectedError.new(error)
+        result
+      end
+
+      def skip_stale_tests?
+        Minitest.queue&.respond_to?(:config) && Minitest.queue.config.skip_stale_tests
+      end
+
+      def build_stale_skip_result
+        $stderr.puts "[ci-queue] Skipping stale preresolved entry: #{@class_name}##{@method_name} " \
+          "(method no longer exists in #{@file_path || 'unknown file'})"
+
+        result_class = defined?(Minitest::Result) ? Minitest::Result : Minitest::Test
+        result = result_class.new(@method_name)
+        result.klass = @class_name if result.respond_to?(:klass=)
+        result.source_location = [@file_path || 'unknown', -1] if result.respond_to?(:source_location=)
+        result.failures << Minitest::Skip.new(
+          "[ci-queue] Stale preresolved entry: #{@class_name}##{@method_name} no longer exists"
+        )
+        result
+      end
+
+      def current_timestamp
+        CI::Queue.time_now.to_i
+      end
+    end
+
     attr_accessor :queue
 
     def queue_reporters=(reporters)
@@ -310,10 +576,12 @@ if defined? Minitest::Result
   Minitest::Result.prepend(Minitest::Requeueing)
   Minitest::Result.prepend(Minitest::Flakiness)
   Minitest::Result.prepend(Minitest::WithTimestamps)
+  Minitest::Result.prepend(Minitest::ResultMetadata)
 else
   Minitest::Test.prepend(Minitest::Requeueing)
   Minitest::Test.prepend(Minitest::Flakiness)
   Minitest::Test.prepend(Minitest::WithTimestamps)
+  Minitest::Test.prepend(Minitest::ResultMetadata)
 
   module MinitestBackwardCompatibility
     def source_location

data/lib/rspec/queue/build_status_recorder.rb

@@ -18,12 +18,14 @@ module RSpec
 
       def example_passed(notification)
         example = notification.example
-        build.record_success(example.id)
+        entry = CI::Queue::QueueEntry.format(example.id, nil)
+        build.record_success(entry)
       end
 
       def example_failed(notification)
         example = notification.example
-        build.record_error(example.id, dump(notification))
+        entry = CI::Queue::QueueEntry.format(example.id, nil)
+        build.record_error(entry, dump(notification))
       end
 
       private

data/lib/rspec/queue.rb

@@ -253,6 +253,10 @@ module RSpec
         example.id
       end
 
+      def queue_entry
+        @queue_entry ||= CI::Queue::QueueEntry.format(id, nil)
+      end
+
       def <=>(other)
         id <=> other.id
       end
@@ -411,7 +415,7 @@ module RSpec
       end
 
       def requeue
-        @queue.requeue(@example)
+        @queue.requeue(@example.queue_entry)
       end
 
       def cancel_run!
@@ -422,7 +426,7 @@ module RSpec
       end
 
       def acknowledge
-        @queue.acknowledge(@example)
+        @queue.acknowledge(@example.queue_entry)
       end
     end
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: ci-queue
 version: !ruby/object:Gem::Version
-  version: 0.82.0
+  version: 0.84.0
 platform: ruby
 authors:
 - Jean Boussier
@@ -149,6 +149,34 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: benchmark
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rexml
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: rubocop
   requirement: !ruby/object:Gem::Requirement
@@ -190,11 +218,14 @@ files:
 - lib/ci/queue/bisect.rb
 - lib/ci/queue/build_record.rb
 - lib/ci/queue/circuit_breaker.rb
+- lib/ci/queue/class_resolver.rb
 - lib/ci/queue/common.rb
 - lib/ci/queue/configuration.rb
 - lib/ci/queue/file.rb
+- lib/ci/queue/file_loader.rb
 - lib/ci/queue/grind.rb
 - lib/ci/queue/output_helpers.rb
+- lib/ci/queue/queue_entry.rb
 - lib/ci/queue/redis.rb
 - lib/ci/queue/redis/acknowledge.lua
 - lib/ci/queue/redis/base.rb
@@ -222,14 +253,18 @@ files:
 - lib/minitest/queue/grind_recorder.rb
 - lib/minitest/queue/grind_reporter.rb
 - lib/minitest/queue/junit_reporter.rb
+- lib/minitest/queue/lazy_entry_resolver.rb
+- lib/minitest/queue/lazy_test_discovery.rb
 - lib/minitest/queue/local_requeue_reporter.rb
 - lib/minitest/queue/order_reporter.rb
+- lib/minitest/queue/queue_population_strategy.rb
 - lib/minitest/queue/runner.rb
 - lib/minitest/queue/statsd.rb
 - lib/minitest/queue/test_data.rb
 - lib/minitest/queue/test_data_reporter.rb
 - lib/minitest/queue/test_time_recorder.rb
 - lib/minitest/queue/test_time_reporter.rb
+- lib/minitest/queue/worker_profile_reporter.rb
 - lib/minitest/reporters/bisect_reporter.rb
 - lib/minitest/reporters/statsd_reporter.rb
 - lib/rspec/queue.rb
@@ -249,14 +284,14 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: '2.7'
+      version: '3.1'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 4.0.6
+rubygems_version: 4.0.8
 specification_version: 4
 summary: Distribute tests over many workers using a queue
 test_files: []