ci-queue 0.82.0 → 0.84.0

data/lib/ci/queue/redis/worker.rb

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 require 'ci/queue/static'
 require 'concurrent/set'
+require 'concurrent/map'
 
 module CI
   module Queue
@@ -13,11 +14,13 @@ module CI
       self.max_sleep_time = 2
 
       class Worker < Base
-        attr_reader :total
+        attr_accessor :entry_resolver
+        attr_reader :first_reserve_at
 
         def initialize(redis, config)
           @reserved_tests = Concurrent::Set.new
           @shutdown_required = false
+          @first_reserve_at = nil
           super(redis, config)
         end
 
@@ -27,15 +30,65 @@ module CI
 
         def populate(tests, random: Random.new)
           @index = tests.map { |t| [t.id, t] }.to_h
-          tests = Queue.shuffle(tests, random)
-          push(tests.map(&:id))
+          entries = Queue.shuffle(tests, random).map { |test| queue_entry_for(test) }
+          push(entries)
           self
         end
 
+        def stream_populate(tests, random: Random.new, batch_size: 10_000)
+          batch_size = batch_size.to_i
+          batch_size = 1 if batch_size < 1
+
+          value = key('setup', worker_id)
+          _, status = redis.pipelined do |pipeline|
+            pipeline.set(key('master-status'), value, nx: true)
+            pipeline.get(key('master-status'))
+          end
+
+          if @master = (value == status)
+            @total = 0
+            puts "Worker elected as leader, streaming tests to the queue."
+
+            duration = measure do
+              start_streaming!
+              buffer = []
+
+              tests.each do |test|
+                buffer << test
+
+                if buffer.size >= batch_size
+                  push_batch(buffer, random)
+                  buffer.clear
+                end
+              end
+
+              push_batch(buffer, random) unless buffer.empty?
+              finalize_streaming
+            end
+
+            puts "Streamed #{@total} tests in #{duration.round(2)}s."
+            $stdout.flush
+          end
+
+          register
+          redis.expire(key('workers'), config.redis_ttl)
+          self
+        rescue *CONNECTION_ERRORS
+          raise if @master
+        end
+
         def populated?
           !!defined?(@index)
         end
 
+        def total
+          return @total if defined?(@total) && @total
+
+          redis.get(key('total')).to_i
+        rescue *CONNECTION_ERRORS
+          @total || 0
+        end
+
         def shutdown!
           @shutdown_required = true
         end
@@ -51,13 +104,18 @@ module CI
         DEFAULT_SLEEP_SECONDS = 0.5
 
         def poll
-          wait_for_master
+          wait_for_master(timeout: config.queue_init_timeout, allow_streaming: true)
           attempt = 0
           until shutdown_required? || config.circuit_breakers.any?(&:open?) || exhausted? || max_test_failed?
-            if test = reserve
+            if entry = reserve
               attempt = 0
-              yield index.fetch(test)
+              yield resolve_entry(entry)
             else
+              if still_streaming?
+                raise LostMaster, "Streaming stalled for more than #{config.lazy_load_streaming_timeout}s" if streaming_stale?
+                sleep 0.1
+                next
+              end
               # Adding exponential backoff to avoid hammering Redis
               # we just stay online here in case a test gets retried or times out so we can afford to wait
               sleep_time = [DEFAULT_SLEEP_SECONDS * (2 ** attempt), Redis.max_sleep_time].min
@@ -89,7 +147,8 @@ module CI
         def retry_queue
           failures = build.failed_tests.to_set
           log = redis.lrange(key('worker', worker_id, 'queue'), 0, -1)
-          log.select! { |id| failures.include?(id) }
+          log = log.map { |entry| CI::Queue::QueueEntry.test_id(entry) }
+          log.select! { |test_id| failures.include?(test_id) }
           log.uniq!
           log.reverse!
           Retry.new(log, config, redis: redis)
@@ -103,23 +162,38 @@ module CI
           @build ||= CI::Queue::Redis::BuildRecord.new(self, redis, config)
         end
 
+        def file_loader
+          @file_loader ||= CI::Queue::FileLoader.new
+        end
+
+        def worker_queue_length
+          redis.llen(key('worker', worker_id, 'queue'))
+        rescue *CONNECTION_ERRORS
+          nil
+        end
+
         def report_worker_error(error)
           build.report_worker_error(error)
         end
 
-        def acknowledge(test_key, error: nil, pipeline: redis)
-          raise_on_mismatching_test(test_key)
+        def acknowledge(entry, error: nil, pipeline: redis)
+          test_id = CI::Queue::QueueEntry.test_id(entry)
+          assert_reserved!(test_id)
+          entry = reserved_entries.fetch(test_id, entry)
+          unreserve_entry(test_id)
           eval_script(
             :acknowledge,
-            keys: [key('running'), key('processed'), key('owners'), key('error-reports')],
-            argv: [test_key, error.to_s, config.redis_ttl],
+            keys: [key('running'), key('processed'), key('owners'), key('error-reports'), key('requeued-by')],
+            argv: [entry, error.to_s, config.redis_ttl],
             pipeline: pipeline,
           ) == 1
         end
 
-        def requeue(test, offset: Redis.requeue_offset)
-          test_key = test.id
-          raise_on_mismatching_test(test_key)
+        def requeue(entry, offset: Redis.requeue_offset)
+          test_id = CI::Queue::QueueEntry.test_id(entry)
+          assert_reserved!(test_id)
+          entry = reserved_entries.fetch(test_id, entry)
+          unreserve_entry(test_id)
           global_max_requeues = config.global_max_requeues(total)
 
           requeued = config.max_requeues > 0 && global_max_requeues > 0 && eval_script(
@@ -132,11 +206,16 @@ module CI
               key('worker', worker_id, 'queue'),
               key('owners'),
               key('error-reports'),
+              key('requeued-by'),
             ],
-            argv: [config.max_requeues, global_max_requeues, test_key, offset],
+            argv: [config.max_requeues, global_max_requeues, entry, offset, config.redis_ttl],
           ) == 1
 
-          reserved_tests << test_key unless requeued
+          unless requeued
+            reserved_tests << test_id
+            reserved_entries[test_id] = entry
+            reserved_entry_ids[entry] = test_id
+          end
           requeued
         end
 
@@ -157,19 +236,118 @@ module CI
           @reserved_tests ||= Concurrent::Set.new
         end
 
+        def reserved_entries
+          @reserved_entries ||= Concurrent::Map.new
+        end
+
+        def reserved_entry_ids
+          @reserved_entry_ids ||= Concurrent::Map.new
+        end
+
         def worker_id
           config.worker_id
         end
 
-        def raise_on_mismatching_test(test)
-          unless reserved_tests.delete?(test)
-            raise ReservationError, "Acknowledged #{test.inspect} but only #{reserved_tests.map(&:inspect).join(", ")} reserved"
+        def assert_reserved!(test_id)
+          unless reserved_tests.include?(test_id)
+            raise ReservationError, "Acknowledged #{test_id.inspect} but only #{reserved_tests.map(&:inspect).join(", ")} reserved"
+          end
+        end
+
+        def reserve_entry(entry)
+          test_id = CI::Queue::QueueEntry.test_id(entry)
+          reserved_tests << test_id
+          reserved_entries[test_id] = entry
+          reserved_entry_ids[entry] = test_id
+        end
+
+        def unreserve_entry(test_id)
+          entry = reserved_entries.delete(test_id)
+          reserved_tests.delete(test_id)
+          reserved_entry_ids.delete(entry) if entry
+        end
+
+        def queue_entry_for(test)
+          return test.queue_entry if test.respond_to?(:queue_entry)
+          return test.id if test.respond_to?(:id)
+
+          test
+        end
+
+        def resolve_entry(entry)
+          test_id = reserved_entry_ids[entry] || CI::Queue::QueueEntry.test_id(entry)
+          if populated?
+            return index[test_id] if index.key?(test_id)
+          end
+
+          return entry_resolver.call(entry) if entry_resolver
+
+          entry
+        end
+
+        def still_streaming?
+          master_status == 'streaming'
+        end
+
+        def streaming_stale?
+          timeout = config.lazy_load_streaming_timeout.to_i
+          updated_at = redis.get(key('streaming-updated-at'))
+          return true unless updated_at
+
+          (CI::Queue.time_now.to_f - updated_at.to_f) > timeout
+        rescue *CONNECTION_ERRORS
+          false
+        end
+
+        def start_streaming!
+          timeout = config.lazy_load_streaming_timeout.to_i
+          with_redis_timeout(5) do
+            redis.multi do |transaction|
+              transaction.set(key('total'), 0)
+              transaction.set(key('master-status'), 'streaming')
+              transaction.set(key('streaming-updated-at'), CI::Queue.time_now.to_f)
+              transaction.expire(key('streaming-updated-at'), timeout)
+              transaction.expire(key('queue'), config.redis_ttl)
+              transaction.expire(key('total'), config.redis_ttl)
+              transaction.expire(key('master-status'), config.redis_ttl)
+            end
+          end
+        end
+
+        def push_batch(tests, random)
+          # Use plain shuffle instead of Queue.shuffle — the custom shuffler expects
+          # test objects with .id, but streaming entries are pre-formatted strings.
+          entries = tests.shuffle(random: random).map { |test| queue_entry_for(test) }
+          return if entries.empty?
+
+          @total += entries.size
+          timeout = config.lazy_load_streaming_timeout.to_i
+          redis.multi do |transaction|
+            transaction.lpush(key('queue'), entries)
+            transaction.incrby(key('total'), entries.size)
+            transaction.set(key('master-status'), 'streaming')
+            transaction.set(key('streaming-updated-at'), CI::Queue.time_now.to_f)
+            transaction.expire(key('streaming-updated-at'), timeout)
+            transaction.expire(key('queue'), config.redis_ttl)
+            transaction.expire(key('total'), config.redis_ttl)
+            transaction.expire(key('master-status'), config.redis_ttl)
+          end
+        end
+
+        def finalize_streaming
+          redis.multi do |transaction|
+            transaction.set(key('master-status'), 'ready')
+            transaction.expire(key('master-status'), config.redis_ttl)
+            transaction.del(key('streaming-updated-at'))
           end
         end
 
         def reserve
-          (try_to_reserve_lost_test || try_to_reserve_test).tap do |test|
-            reserved_tests << test if test
+          (try_to_reserve_lost_test || try_to_reserve_test).tap do |entry|
+            if entry
+              @first_reserve_at ||= Process.clock_gettime(Process::CLOCK_MONOTONIC)
+              reserve_entry(entry)
+            end
           end
         end
 
@@ -182,8 +360,10 @@ module CI
               key('processed'),
               key('worker', worker_id, 'queue'),
               key('owners'),
+              key('requeued-by'),
+              key('workers'),
             ],
-            argv: [CI::Queue.time_now.to_f],
+            argv: [CI::Queue.time_now.to_f, Redis.requeue_offset],
           )
         end
 
@@ -194,7 +374,7 @@ module CI
             :reserve_lost,
             keys: [
               key('running'),
-              key('completed'),
+              key('processed'),
               key('worker', worker_id, 'queue'),
               key('owners'),
             ],
@@ -202,14 +382,17 @@ module CI
           )
 
           if lost_test
-            build.record_warning(Warnings::RESERVED_LOST_TEST, test: lost_test, timeout: config.timeout)
+            build.record_warning(Warnings::RESERVED_LOST_TEST, test: CI::Queue::QueueEntry.test_id(lost_test), timeout: config.timeout)
+            if CI::Queue.debug?
+              $stderr.puts "[ci-queue][reserve_lost] worker=#{worker_id} test_id=#{CI::Queue::QueueEntry.test_id(lost_test)}"
+            end
           end
 
           lost_test
         end
 
-        def push(tests)
-          @total = tests.size
+        def push(entries)
+          @total = entries.size
 
           # We set a unique value (worker_id) and read it back to make "SET if Not eXists" idempotent in case of a retry.
           value = key('setup', worker_id)
@@ -227,7 +410,7 @@ module CI
               with_redis_timeout(5) do
                 redis.without_reconnect do
                   redis.multi do |transaction|
-                    transaction.lpush(key('queue'), tests) unless tests.empty?
+                    transaction.lpush(key('queue'), entries) unless entries.empty?
                     transaction.set(key('total'), @total)
                     transaction.set(key('master-status'), 'ready')
 

data/lib/ci/queue/static.rb

@@ -125,12 +125,12 @@ module CI
         test_failed >= config.max_test_failed
       end
 
-      def requeue(test)
-        test_key = test.id
-        return false unless should_requeue?(test_key)
+      def requeue(entry)
+        test_id = CI::Queue::QueueEntry.test_id(entry)
+        return false unless should_requeue?(test_id)
 
-        requeues[test_key] += 1
-        @queue.unshift(test_key)
+        requeues[test_id] += 1
+        @queue.unshift(test_id)
         true
       end
 

data/lib/ci/queue/version.rb

@@ -2,7 +2,7 @@
 
 module CI
   module Queue
-    VERSION = '0.82.0'
+    VERSION = '0.84.0'
     DEV_SCRIPTS_ROOT = ::File.expand_path('../../../../../redis', __FILE__)
     RELEASE_SCRIPTS_ROOT = ::File.expand_path('../redis', __FILE__)
   end

data/lib/ci/queue.rb

@@ -14,6 +14,9 @@ require 'ci/queue/static'
 require 'ci/queue/file'
 require 'ci/queue/grind'
 require 'ci/queue/bisect'
+require 'ci/queue/queue_entry'
+require 'ci/queue/class_resolver'
+require 'ci/queue/file_loader'
 
 module CI
   module Queue
@@ -22,6 +25,18 @@ module CI
     attr_accessor :shuffler, :requeueable
 
     Error = Class.new(StandardError)
+    ClassNotFoundError = Class.new(Error)
+
+    class FileLoadError < Error
+      attr_reader :file_path, :original_error
+
+      def initialize(file_path, original_error)
+        @file_path = file_path
+        @original_error = original_error
+        super("Failed to load #{file_path}: #{original_error.class}: #{original_error.message}")
+        set_backtrace(original_error.backtrace)
+      end
+    end
 
     module Warnings
       RESERVED_LOST_TEST = :RESERVED_LOST_TEST
@@ -48,6 +63,17 @@ module CI
       end
     end
 
+    def debug?
+      return @debug if defined?(@debug)
+
+      value = ENV['CI_QUEUE_DEBUG']
+      @debug = !!(value && !value.strip.empty? && !%w[0 false].include?(value.strip.downcase))
+    end
+
+    def reset_debug!
+      remove_instance_variable(:@debug) if defined?(@debug)
+    end
+
     def from_uri(url, config)
       uri = URI(url)
       implementation = case uri.scheme
@@ -65,3 +91,4 @@ module CI
     end
   end
 end
+

data/lib/minitest/queue/build_status_recorder.rb

@@ -40,15 +40,15 @@ module Minitest
         self.total_time = Minitest.clock_time - start_time
         
         # Determine what type of result this is and record it
-        test_id = "#{test.klass}##{test.name}"
+        entry = test.queue_entry
         delta = delta_for(test)
 
         acknowledged = if (test.failure || test.error?) && !test.skipped?
-          build.record_error(test_id, dump(test), stat_delta: delta)
+          build.record_error(entry, dump(test), stat_delta: delta)
         elsif test.requeued?
-          build.record_requeue(test_id)
+          build.record_requeue(entry)
         else
-          build.record_success(test_id, skip_flaky_record: test.skipped?)
+          build.record_success(entry, skip_flaky_record: test.skipped?)
         end
 
         if acknowledged

data/lib/minitest/queue/junit_reporter.rb

@@ -136,9 +136,9 @@ module Minitest
         result[:time] = 0
         tests.each do |test|
           result[:"#{result(test)}_count"] += 1
-          result[:assertion_count] += test.assertions
+          result[:assertion_count] += test.assertions || 0
           result[:test_count] += 1
-          result[:time] += test.time
+          result[:time] += test.time || 0
         end
         result
       end

data/lib/minitest/queue/lazy_entry_resolver.rb

@@ -0,0 +1,55 @@
+# frozen_string_literal: true
+
+module Minitest
+  module Queue
+    class LazyEntryResolver
+      def initialize(loader:, resolver:)
+        @loader = loader
+        @resolver = resolver
+      end
+
+      def call(entry)
+        parsed = CI::Queue::QueueEntry.parse(entry)
+        class_name, method_name = parsed.fetch(:test_id).split('#', 2)
+        if CI::Queue::QueueEntry.load_error_payload?(parsed[:file_path])
+          payload = CI::Queue::QueueEntry.decode_load_error(parsed[:file_path])
+          if payload
+            error = StandardError.new("#{payload['error_class']}: #{payload['error_message']}")
+            error.set_backtrace(payload['backtrace']) if payload['backtrace']
+            load_error = CI::Queue::FileLoadError.new(payload['file_path'], error)
+            return Minitest::Queue::LazySingleExample.new(
+              class_name,
+              method_name,
+              payload['file_path'],
+              loader: @loader,
+              resolver: @resolver,
+              load_error: load_error,
+              queue_entry: entry,
+            )
+          else
+            error = StandardError.new("Corrupt load error payload for #{class_name}##{method_name}")
+            load_error = CI::Queue::FileLoadError.new("unknown", error)
+            return Minitest::Queue::LazySingleExample.new(
+              class_name,
+              method_name,
+              nil,
+              loader: @loader,
+              resolver: @resolver,
+              load_error: load_error,
+              queue_entry: entry,
+            )
+          end
+        end
+
+        Minitest::Queue::LazySingleExample.new(
+          class_name,
+          method_name,
+          parsed[:file_path],
+          loader: @loader,
+          resolver: @resolver,
+          queue_entry: entry,
+        )
+      end
+    end
+  end
+end

data/lib/minitest/queue/lazy_test_discovery.rb

@@ -0,0 +1,169 @@
+# frozen_string_literal: true
+
+require 'set'
+require 'zlib'
+
+module Minitest
+  module Queue
+    class LazyTestDiscovery
+      def initialize(loader:, resolver:)
+        @loader = loader
+        @resolver = resolver
+      end
+
+      def enumerator(files)
+        Enumerator.new do |yielder|
+          each_test(files) do |test|
+            yielder << test
+          end
+        end
+      end
+
+      def each_test(files)
+        discovery_start = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+        total_files = 0
+        new_runnable_files = 0
+        reopened_files = 0
+        reopened_candidates = 0
+        reopened_scan_time = 0.0
+
+        seen = Set.new
+        runnables = Minitest::Test.runnables
+        known_count = runnables.size
+        by_full_name = {}
+        by_short_name = Hash.new { |h, k| h[k] = [] }
+        index_runnables(runnables, by_full_name, by_short_name)
+
+        files.each do |file|
+          total_files += 1
+          file_path = File.expand_path(file)
+          begin
+            @loader.load_file(file_path)
+          rescue CI::Queue::FileLoadError => error
+            method_name = "load_file_#{Zlib.crc32(file_path).to_s(16)}"
+            class_name = "CIQueue::FileLoadError"
+            test_id = "#{class_name}##{method_name}"
+            entry = CI::Queue::QueueEntry.format(
+              test_id,
+              CI::Queue::QueueEntry.encode_load_error(file_path, error),
+            )
+            yield Minitest::Queue::LazySingleExample.new(
+              class_name,
+              method_name,
+              file_path,
+              loader: @loader,
+              resolver: @resolver,
+              load_error: error,
+              queue_entry: entry,
+            )
+            next
+          end
+
+          runnables = Minitest::Test.runnables
+          candidates = []
+          if runnables.size > known_count
+            new_runnables = runnables[known_count..]
+            known_count = runnables.size
+            index_runnables(new_runnables, by_full_name, by_short_name)
+            candidates.concat(new_runnables)
+            new_runnable_files += 1
+          else
+            # Re-opened classes do not increase runnables size. In that case, map
+            # declared class names in the file to known runnables directly.
+            reopened_start = Process.clock_gettime(Process::CLOCK_MONOTONIC)
+            reopened = reopened_runnables_for_file(file_path, by_full_name, by_short_name)
+            reopened_scan_time += Process.clock_gettime(Process::CLOCK_MONOTONIC) - reopened_start
+            unless reopened.empty?
+              reopened_files += 1
+              reopened_candidates += reopened.size
+            end
+            candidates.concat(reopened)
+          end
+
+          enqueue_discovered_tests(candidates.uniq, file_path, seen) do |test|
+            yield test
+          end
+        end
+      ensure
+        debug_discovery_profile(
+          discovery_start: discovery_start,
+          total_files: total_files,
+          new_runnable_files: new_runnable_files,
+          reopened_files: reopened_files,
+          reopened_candidates: reopened_candidates,
+          reopened_scan_time: reopened_scan_time,
+        )
+      end
+
+      private
+
+      def reopened_runnables_for_file(file_path, by_full_name, by_short_name)
+        declared = declared_class_names(file_path)
+        return [] if declared.empty?
+
+        declared.each_with_object([]) do |name, runnables|
+          runnable = by_full_name[name]
+          if runnable
+            runnables << runnable
+            next
+          end
+
+          short_name = name.split('::').last
+          runnables.concat(by_short_name[short_name])
+        end
+      end
+
+      def index_runnables(runnables, by_full_name, by_short_name)
+        runnables.each do |runnable|
+          name = runnable.name
+          next unless name
+
+          by_full_name[name] ||= runnable
+          short_name = name.split('::').last
+          by_short_name[short_name] << runnable
+        end
+      end
+
+      def debug_discovery_profile(discovery_start:, total_files:, new_runnable_files:, reopened_files:, reopened_candidates:, reopened_scan_time:)
+        return unless CI::Queue.debug?
+
+        total_time = Process.clock_gettime(Process::CLOCK_MONOTONIC) - discovery_start
+        puts "[ci-queue][lazy-discovery] files=#{total_files} new_runnable_files=#{new_runnable_files} " \
+          "reopened_files=#{reopened_files} reopened_candidates=#{reopened_candidates} " \
+          "reopened_scan_time=#{reopened_scan_time.round(2)}s total_time=#{total_time.round(2)}s"
+      end
+
+      def enqueue_discovered_tests(runnables, file_path, seen)
+        runnables.each do |runnable|
+          runnable.runnable_methods.each do |method_name|
+            test_id = "#{runnable.name}##{method_name}"
+            next if seen.include?(test_id)
+
+            seen.add(test_id)
+            yield Minitest::Queue::LazySingleExample.new(
+              runnable.name,
+              method_name,
+              file_path,
+              loader: @loader,
+              resolver: @resolver,
+            )
+          rescue NameError, NoMethodError
+            next
+          end
+        end
+      end
+
+      def declared_class_names(file_path)
+        names = Set.new
+        ::File.foreach(file_path) do |line|
+          match = line.match(/^\s*class\s+([A-Z]\w*(?:::[A-Z]\w*)*)\b/)
+          names.add(match[1]) if match
+        end
+        names
+      rescue SystemCallError
+        Set.new
+      end
+
+    end
+  end
+end