ci-queue 0.81.0 → 0.83.0

data/lib/minitest/queue/lazy_test_discovery.rb

@@ -0,0 +1,169 @@
+# frozen_string_literal: true
+
+require 'set'
+require 'zlib'
+
+module Minitest
+  module Queue
+    class LazyTestDiscovery
+      def initialize(loader:, resolver:)
+        @loader = loader
+        @resolver = resolver
+      end
+
+      def enumerator(files)
+        Enumerator.new do |yielder|
+          each_test(files) do |test|
+            yielder << test
+          end
+        end
+      end
+
+      def each_test(files)
+        discovery_start = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+        total_files = 0
+        new_runnable_files = 0
+        reopened_files = 0
+        reopened_candidates = 0
+        reopened_scan_time = 0.0
+
+        seen = Set.new
+        runnables = Minitest::Test.runnables
+        known_count = runnables.size
+        by_full_name = {}
+        by_short_name = Hash.new { |h, k| h[k] = [] }
+        index_runnables(runnables, by_full_name, by_short_name)
+
+        files.each do |file|
+          total_files += 1
+          file_path = File.expand_path(file)
+          begin
+            @loader.load_file(file_path)
+          rescue CI::Queue::FileLoadError => error
+            method_name = "load_file_#{Zlib.crc32(file_path).to_s(16)}"
+            class_name = "CIQueue::FileLoadError"
+            test_id = "#{class_name}##{method_name}"
+            entry = CI::Queue::QueueEntry.format(
+              test_id,
+              CI::Queue::QueueEntry.encode_load_error(file_path, error),
+            )
+            yield Minitest::Queue::LazySingleExample.new(
+              class_name,
+              method_name,
+              file_path,
+              loader: @loader,
+              resolver: @resolver,
+              load_error: error,
+              queue_entry: entry,
+            )
+            next
+          end
+
+          runnables = Minitest::Test.runnables
+          candidates = []
+          if runnables.size > known_count
+            new_runnables = runnables[known_count..]
+            known_count = runnables.size
+            index_runnables(new_runnables, by_full_name, by_short_name)
+            candidates.concat(new_runnables)
+            new_runnable_files += 1
+          else
+            # Re-opened classes do not increase runnables size. In that case, map
+            # declared class names in the file to known runnables directly.
+            reopened_start = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+            reopened = reopened_runnables_for_file(file_path, by_full_name, by_short_name)
+            reopened_scan_time += Process.clock_gettime(Process::CLOCK_MONOTONIC) - reopened_start
+            unless reopened.empty?
+              reopened_files += 1
+              reopened_candidates += reopened.size
+            end
+            candidates.concat(reopened)
+          end
+
+          enqueue_discovered_tests(candidates.uniq, file_path, seen) do |test|
+            yield test
+          end
+        end
+      ensure
+        debug_discovery_profile(
+          discovery_start: discovery_start,
+          total_files: total_files,
+          new_runnable_files: new_runnable_files,
+          reopened_files: reopened_files,
+          reopened_candidates: reopened_candidates,
+          reopened_scan_time: reopened_scan_time,
+        )
+      end
+
+      private
+
+      def reopened_runnables_for_file(file_path, by_full_name, by_short_name)
+        declared = declared_class_names(file_path)
+        return [] if declared.empty?
+
+        declared.each_with_object([]) do |name, runnables|
+          runnable = by_full_name[name]
+          if runnable
+            runnables << runnable
+            next
+          end
+
+          short_name = name.split('::').last
+          runnables.concat(by_short_name[short_name])
+        end
+      end
+
+      def index_runnables(runnables, by_full_name, by_short_name)
+        runnables.each do |runnable|
+          name = runnable.name
+          next unless name
+
+          by_full_name[name] ||= runnable
+          short_name = name.split('::').last
+          by_short_name[short_name] << runnable
+        end
+      end
+
+      def debug_discovery_profile(discovery_start:, total_files:, new_runnable_files:, reopened_files:, reopened_candidates:, reopened_scan_time:)
+        return unless CI::Queue.debug?
+
+        total_time = Process.clock_gettime(Process::CLOCK_MONOTONIC) - discovery_start
+        puts "[ci-queue][lazy-discovery] files=#{total_files} new_runnable_files=#{new_runnable_files} " \
+          "reopened_files=#{reopened_files} reopened_candidates=#{reopened_candidates} " \
+          "reopened_scan_time=#{reopened_scan_time.round(2)}s total_time=#{total_time.round(2)}s"
+      end
+
+      def enqueue_discovered_tests(runnables, file_path, seen)
+        runnables.each do |runnable|
+          runnable.runnable_methods.each do |method_name|
+            test_id = "#{runnable.name}##{method_name}"
+            next if seen.include?(test_id)
+
+            seen.add(test_id)
+            yield Minitest::Queue::LazySingleExample.new(
+              runnable.name,
+              method_name,
+              file_path,
+              loader: @loader,
+              resolver: @resolver,
+            )
+          rescue NameError, NoMethodError
+            next
+          end
+        end
+      end
+
+      def declared_class_names(file_path)
+        names = Set.new
+        ::File.foreach(file_path) do |line|
+          match = line.match(/^\s*class\s+([A-Z]\w*(?:::[A-Z]\w*)*)\b/)
+          names.add(match[1]) if match
+        end
+        names
+      rescue SystemCallError
+        Set.new
+      end
+
+    end
+  end
+end

data/lib/minitest/queue/local_requeue_reporter.rb

@@ -46,6 +46,17 @@ module Minitest
       def result_line
         "#{super}, #{requeues} requeues"
       end
+
+      def location(exception)
+        backtrace = exception.backtrace
+        return super if backtrace && !backtrace.empty?
+
+        nested_exception = exception.respond_to?(:error) ? exception.error : nil
+        nested_backtrace = nested_exception&.backtrace
+        return super(nested_exception) if nested_backtrace && !nested_backtrace.empty?
+
+        'unknown'
+      end
     end
   end
 end

data/lib/minitest/queue/order_reporter.rb

@@ -5,6 +5,9 @@ class Minitest::Queue::OrderReporter < Minitest::Reporters::BaseReporter
   def initialize(options = {})
     @path = options.delete(:path)
     @file = nil
+    @flush_every = Integer(ENV.fetch('CI_QUEUE_ORDER_FLUSH_EVERY', '50'))
+    @flush_every = 1 if @flush_every < 1
+    @pending = 0
     super
   end
 
@@ -16,10 +19,15 @@ class Minitest::Queue::OrderReporter < Minitest::Reporters::BaseReporter
   def before_test(test)
     super
     file.puts("#{test.class.name}##{test.name}")
-    file.flush
+    @pending += 1
+    if @pending >= @flush_every
+      file.flush
+      @pending = 0
+    end
   end
 
   def report
+    file.flush
     file.close
   end
 
@@ -29,4 +37,3 @@ class Minitest::Queue::OrderReporter < Minitest::Reporters::BaseReporter
     @file ||= File.open(@path, 'a+')
   end
 end
-

data/lib/minitest/queue/queue_population_strategy.rb

@@ -0,0 +1,176 @@
+# frozen_string_literal: true
+
+require 'set'
+require 'minitest/queue/lazy_entry_resolver'
+require 'minitest/queue/lazy_test_discovery'
+
+module Minitest
+  module Queue
+    class QueuePopulationStrategy
+      attr_reader :load_tests_duration, :total_files
+
+      def initialize(queue:, queue_config:, argv:, test_files_file:, ordering_seed:, preresolved_test_list: nil)
+        @queue = queue
+        @queue_config = queue_config
+        @argv = argv
+        @test_files_file = test_files_file
+        @ordering_seed = ordering_seed
+        @preresolved_test_list = preresolved_test_list
+      end
+
+      def load_and_populate!
+        load_tests
+        populate_queue
+      end
+
+      private
+
+      attr_reader :queue, :queue_config, :argv, :test_files_file, :ordering_seed, :preresolved_test_list
+
+      def populate_queue
+        if preresolved_test_list && queue.respond_to?(:stream_populate)
+          configure_lazy_queue
+          queue.stream_populate(preresolved_entry_enumerator, random: ordering_seed, batch_size: queue_config.lazy_load_stream_batch_size)
+        elsif queue_config.lazy_load && queue.respond_to?(:stream_populate)
+          configure_lazy_queue
+          queue.stream_populate(lazy_test_enumerator, random: ordering_seed, batch_size: queue_config.lazy_load_stream_batch_size)
+        else
+          queue.populate(Minitest.loaded_tests, random: ordering_seed)
+        end
+      end
+
+      def load_tests
+        start = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+        if preresolved_test_list || queue_config.lazy_load
+          # In preresolved or lazy-load mode, test files are loaded on-demand by the entry resolver.
+          # Load test helpers (e.g., test/test_helper.rb via CI_QUEUE_LAZY_LOAD_TEST_HELPERS)
+          # to boot the app for all workers.
+          queue_config.lazy_load_test_helper_paths.each do |helper_path|
+            require File.expand_path(helper_path)
+          end
+        else
+          test_file_list.sort.each do |file_path|
+            require File.expand_path(file_path)
+          end
+        end
+      ensure
+        @load_tests_duration = Process.clock_gettime(Process::CLOCK_MONOTONIC) - start
+        @total_files = begin
+          preresolved_test_list ? nil : test_file_list.size
+        rescue StandardError
+          nil
+        end
+      end
+
+      def configure_lazy_queue
+        return unless queue.respond_to?(:entry_resolver=)
+
+        queue.entry_resolver = lazy_entry_resolver
+      end
+
+      def lazy_entry_resolver
+        loader = queue.respond_to?(:file_loader) ? queue.file_loader : CI::Queue::FileLoader.new
+        resolver = CI::Queue::ClassResolver
+        Minitest::Queue::LazyEntryResolver.new(loader: loader, resolver: resolver)
+      end
+
+      # Reads a pre-resolved test list file (one entry per line: "TestId|file/path.rb")
+      # and yields entries in the internal tab-delimited QueueEntry format.
+      #
+      # When --test-files is also provided, those files act as a reconcile set:
+      # preresolved entries whose file path matches are skipped (they may be stale),
+      # and the files are lazily discovered to enqueue their current tests.
+      # This runs only on the leader (stream_populate is leader-only), so discovery
+      # happens exactly once per build.
+      def preresolved_entry_enumerator
+        override_files = reconcile_file_set
+
+        Enumerator.new do |yielder|
+          preresolved_kept = 0
+          preresolved_skipped = 0
+
+          File.foreach(preresolved_test_list) do |line|
+            line = line.chomp
+            next if line.strip.empty?
+
+            # Split on the LAST pipe — test method names can contain '|'
+            # (e.g., regex patterns, boolean conditions) but file paths never do.
+            test_id, _, file_path = line.rpartition('|')
+            if test_id.empty?
+              # No pipe found — treat the whole line as a test ID with no file path
+              test_id = file_path
+              file_path = nil
+            end
+
+            # Skip entries for files that will be re-discovered via --test-files
+            if file_path && override_files.include?(file_path)
+              preresolved_skipped += 1
+              next
+            end
+
+            preresolved_kept += 1
+            yielder << CI::Queue::QueueEntry.format(test_id, file_path)
+          end
+
+          if CI::Queue.debug?
+            puts "[ci-queue][preresolved] kept=#{preresolved_kept} skipped=#{preresolved_skipped} " \
+              "reconcile_files=#{override_files.size}"
+          end
+
+          # Lazily discover current tests for each reconciled file
+          unless override_files.empty?
+            discovery_start = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+            discovered = 0
+            lazy_test_discovery.each_test(override_files.to_a) do |example|
+              discovered += 1
+              yielder << example.queue_entry
+            end
+            duration = Process.clock_gettime(Process::CLOCK_MONOTONIC) - discovery_start
+            if CI::Queue.debug?
+              puts "[ci-queue][reconcile-discovery] discovered=#{discovered} files=#{override_files.size} " \
+                "duration=#{duration.round(2)}s"
+            end
+          end
+        end
+      end
+
+      def lazy_test_discovery
+        @lazy_test_discovery ||= begin
+          loader = queue.respond_to?(:file_loader) ? queue.file_loader : CI::Queue::FileLoader.new
+          Minitest::Queue::LazyTestDiscovery.new(loader: loader, resolver: CI::Queue::ClassResolver)
+        end
+      end
+
+      # In preresolved mode, --test-files provides the reconcile set: test files
+      # whose preresolved entries should be discarded and re-discovered.
+      # Returns an empty Set if no test files file is provided.
+      def reconcile_file_set
+        return Set.new unless test_files_file
+        return Set.new unless File.exist?(test_files_file)
+
+        Set.new(File.readlines(test_files_file, chomp: true).reject(&:empty?))
+      end
+
+      def lazy_test_enumerator
+        loader = queue.respond_to?(:file_loader) ? queue.file_loader : CI::Queue::FileLoader.new
+        resolver = CI::Queue::ClassResolver
+        files = test_file_list.sort
+
+        Minitest::Queue::LazyTestDiscovery.new(loader: loader, resolver: resolver).enumerator(files)
+      end
+
+      # Returns the list of test files to process. Prefers --test-files FILE
+      # (reads paths from a file, one per line) over positional argv arguments.
+      # --test-files avoids ARG_MAX limits for large test suites (36K+ files).
+      def test_file_list
+        @test_file_list ||= begin
+          if test_files_file
+            File.readlines(test_files_file, chomp: true).reject { |f| f.strip.empty? }
+          else
+            argv
+          end
+        end
+      end
+    end
+  end
+end

data/lib/minitest/queue/runner.rb

@@ -1,11 +1,14 @@
 # frozen_string_literal: true
 require 'optparse'
 require 'json'
+require 'fileutils'
 require 'minitest/queue'
 require 'ci/queue'
 require 'digest/md5'
 require 'minitest/reporters/bisect_reporter'
 require 'minitest/reporters/statsd_reporter'
+require 'minitest/queue/queue_population_strategy'
+require 'minitest/queue/worker_profile_reporter'
 
 module Minitest
   module Queue
@@ -15,6 +18,10 @@ module Minitest
       Error = Class.new(StandardError)
       MissingParameter = Class.new(Error)
 
+      class << self
+        attr_accessor :run_start, :load_tests_duration, :total_files
+      end
+
       def self.invoke(argv)
         new(argv).run!
       end
@@ -48,6 +55,8 @@ module Minitest
       end
 
       def run_command
+        Runner.run_start = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+
         require_worker_id!
         # if it's an automatic job retry we should process the main queue
         if manual_retry?
@@ -97,19 +106,27 @@ module Minitest
             if remaining <= running
               puts green("Queue almost empty, exiting early...")
             else
-              load_tests
-              populate_queue
+              prepare_queue_for_execution
             end
           else
-            load_tests
-            populate_queue
+            prepare_queue_for_execution
           end
         end
 
-        at_exit {
+        if queue_config.lazy_load
+          # In lazy-load mode, run minitest explicitly instead of relying on
+          # minitest/autorun's at_exit hook, which may not be registered since
+          # test files haven't been loaded yet. exit! prevents double-execution
+          # if minitest/autorun was loaded by the leader during streaming.
+          passed = Minitest.run []
           verify_reporters!(reporters)
-        }
-        # Let minitest's at_exit hook trigger
+          exit!(passed ? 0 : 1)
+        else
+          at_exit {
+            verify_reporters!(reporters)
+          }
+          # Let minitest's at_exit hook trigger
+        end
       end
 
       def verify_reporters!(reporters)
@@ -170,11 +187,9 @@ module Minitest
         trap('TERM') { Minitest.queue.shutdown! }
         trap('INT') { Minitest.queue.shutdown! }
 
-        load_tests
-
         @queue = CI::Queue::Grind.new(grind_list, queue_config)
         Minitest.queue = queue
-        populate_queue
+        prepare_queue_for_execution
 
         # Let minitest's at_exit hook trigger
       end
@@ -183,10 +198,9 @@ module Minitest
         invalid_usage! "Missing the FAILING_TEST argument." unless queue_config.failing_test
 
         set_load_path
-        load_tests
         @queue = CI::Queue::Bisect.new(queue_url, queue_config)
         Minitest.queue = queue
-        populate_queue
+        prepare_queue_for_execution
 
         step("Testing the failing test in isolation")
         unless queue.failing_test_present?
@@ -242,16 +256,16 @@ module Minitest
           puts
 
           File.write('log/test_order.log', failing_order.to_a.map(&:id).join("\n"))
-          
+
           bisect_test_details = failing_order.to_a.map do |test|
             source_location = test.source_location
             file_path = source_location&.first || 'unknown'
             line_number = source_location&.last || -1
             "#{test.id} #{file_path}:#{line_number}"
           end
-          
+
           File.write('log/bisect_test_details.log', bisect_test_details.join("\n"))
-          
+
           exit! 0
         end
       end
@@ -284,6 +298,7 @@ module Minitest
         reporter.write_failure_file(queue_config.failure_file) if queue_config.failure_file
         reporter.write_flaky_tests_file(queue_config.export_flaky_tests_file) if queue_config.export_flaky_tests_file
         exit_code = reporter.report
+        print_worker_profiles(supervisor)
         exit! exit_code
       end
 
@@ -321,7 +336,7 @@ module Minitest
 
       attr_reader :queue_config, :options, :command, :argv
       attr_writer :queue_url
-      attr_accessor :queue, :grind_list, :grind_count, :load_paths, :verbose
+      attr_accessor :queue, :grind_list, :grind_count, :load_paths, :verbose, :test_files_file, :preresolved_test_list
 
       def require_worker_id!
         if queue.distributed?
@@ -336,8 +351,22 @@ module Minitest
         warnings = build.pop_warnings.map do |type, attributes|
           attributes.merge(type: type)
         end.compact
-        File.open(queue_config.warnings_file, 'w') do |f|
-          JSON.dump(warnings, f)
+
+        return if warnings.empty?
+
+        begin
+          # Ensure directory exists
+          dir = File.dirname(queue_config.warnings_file)
+          FileUtils.mkdir_p(dir) unless File.directory?(dir)
+
+          # Write each warning as a separate JSON line (JSONL format)
+          File.open(queue_config.warnings_file, 'a') do |f|
+            warnings.each do |warning|
+              f.puts(JSON.dump(warning))
+            end
+          end
+        rescue => error
+          STDERR.puts "Failed to write warnings: #{error.message}"
         end
       end
 
@@ -357,10 +386,6 @@ module Minitest
         queue.build.reset_worker_error
       end
 
-      def populate_queue
-        Minitest.queue.populate(Minitest.loaded_tests, random: ordering_seed)
-      end
-
       def set_load_path
         if paths = load_paths
           paths.split(':').reverse.each do |path|
@@ -369,10 +394,22 @@ module Minitest
         end
       end
 
-      def load_tests
-        argv.sort.each do |f|
-          require File.expand_path(f)
-        end
+      def prepare_queue_for_execution
+        strategy = queue_population_strategy
+        strategy.load_and_populate!
+        Runner.load_tests_duration = strategy.load_tests_duration
+        Runner.total_files = strategy.total_files
+      end
+
+      def queue_population_strategy
+        Minitest::Queue::QueuePopulationStrategy.new(
+          queue: queue,
+          queue_config: queue_config,
+          argv: argv,
+          test_files_file: test_files_file,
+          ordering_seed: ordering_seed,
+          preresolved_test_list: preresolved_test_list,
+        )
       end
 
       def parse(argv)
@@ -381,6 +418,10 @@ module Minitest
         return command, argv
       end
 
+      def print_worker_profiles(supervisor)
+        Minitest::Queue::WorkerProfileReporter.new(supervisor).print_summary
+      end
+
       def parser
         @parser ||= OptionParser.new do |opts|
           opts.banner = "Usage: minitest-queue [options] COMMAND [ARGS]"
@@ -487,7 +528,56 @@ module Minitest
           end
 
           help = <<~EOS
-            Sepcify a seed used to shuffle the test suite.
+            Load test files on demand instead of eagerly loading all tests.
+          EOS
+          opts.separator ""
+          opts.on('--lazy-load', help) do
+            queue_config.lazy_load = true
+          end
+
+          help = <<~EOS
+            Batch size for streaming tests to Redis in lazy mode.
+            Defaults to 5000.
+          EOS
+          opts.separator ""
+          opts.on('--lazy-load-stream-batch-size SIZE', Integer, help) do |size|
+            queue_config.lazy_load_stream_batch_size = size
+          end
+
+          help = <<~EOS
+            Read test file paths from FILE (one per line) instead of positional arguments.
+            Useful for large test suites to avoid ARG_MAX limits.
+          EOS
+          opts.separator ""
+          opts.on('--test-files FILE', help) do |file|
+            self.test_files_file = file
+          end
+
+          help = <<~EOS
+            Read a pre-resolved test list from FILE instead of discovering tests from files.
+            Each line must be in the format: TestClass#method_name|path/to/test_file.rb
+            The leader streams entries directly to Redis without loading or discovering tests.
+            Implies lazy-load mode for all workers.
+            When combined with --test-files, entries whose file path appears in that list
+            are skipped and the files are lazily re-discovered (reconciliation).
+          EOS
+          opts.separator ""
+          opts.on('--preresolved-tests FILE', help) do |file|
+            self.preresolved_test_list = file
+            queue_config.lazy_load = true
+          end
+
+          help = <<~EOS
+            Timeout in seconds without new streamed batches before failing.
+            Defaults to the max of --queue-init-timeout and 300 seconds.
+          EOS
+          opts.separator ""
+          opts.on('--lazy-load-stream-timeout SECONDS', Integer, help) do |seconds|
+            queue_config.lazy_load_streaming_timeout = seconds
+          end
+
+          help = <<~EOS
+            Specify a seed used to shuffle the test suite.
             On Buildkite, CircleCI, Heroku CI, and Travis, the commit revision will be used by default.
           EOS
           opts.separator ""

data/lib/minitest/queue/test_data.rb

@@ -137,11 +137,13 @@ module Minitest
       def error_location(exception)
         @error_location ||= begin
           last_before_assertion = ''
-          exception.backtrace.reverse_each do |s|
+          backtrace_for(exception).reverse_each do |s|
             break if s =~ /in .(assert|refute|flunk|pass|fail|raise|must|wont)/
 
             last_before_assertion = s
           end
+          return ['unknown', 0] if last_before_assertion.empty?
+
           path = last_before_assertion.sub(/:in .*$/, '')
           # the path includes the linenumber at the end,
           # which is seperated by a :
@@ -151,6 +153,17 @@ module Minitest
           [result.first, result.last.to_i]
         end
       end
+
+      def backtrace_for(exception)
+        backtrace = exception.backtrace
+        return backtrace if backtrace && !backtrace.empty?
+
+        nested_exception = exception.respond_to?(:error) ? exception.error : nil
+        nested_backtrace = nested_exception&.backtrace
+        return nested_backtrace if nested_backtrace && !nested_backtrace.empty?
+
+        []
+      end
     end
   end
 end