ci-queue 0.81.0 → 0.83.0

data/lib/ci/queue/redis/worker.rb

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 require 'ci/queue/static'
 require 'concurrent/set'
+require 'concurrent/map'
 
 module CI
   module Queue
@@ -13,11 +14,13 @@ module CI
       self.max_sleep_time = 2
 
       class Worker < Base
-        attr_reader :total
+        attr_accessor :entry_resolver
+        attr_reader :first_reserve_at
 
         def initialize(redis, config)
           @reserved_tests = Concurrent::Set.new
           @shutdown_required = false
+          @first_reserve_at = nil
           super(redis, config)
         end
 
@@ -27,15 +30,65 @@ module CI
 
         def populate(tests, random: Random.new)
           @index = tests.map { |t| [t.id, t] }.to_h
-          tests = Queue.shuffle(tests, random)
-          push(tests.map(&:id))
+          entries = Queue.shuffle(tests, random).map { |test| queue_entry_for(test) }
+          push(entries)
           self
         end
 
+        def stream_populate(tests, random: Random.new, batch_size: 10_000)
+          batch_size = batch_size.to_i
+          batch_size = 1 if batch_size < 1
+
+          value = key('setup', worker_id)
+          _, status = redis.pipelined do |pipeline|
+            pipeline.set(key('master-status'), value, nx: true)
+            pipeline.get(key('master-status'))
+          end
+
+          if @master = (value == status)
+            @total = 0
+            puts "Worker elected as leader, streaming tests to the queue."
+
+            duration = measure do
+              start_streaming!
+              buffer = []
+
+              tests.each do |test|
+                buffer << test
+
+                if buffer.size >= batch_size
+                  push_batch(buffer, random)
+                  buffer.clear
+                end
+              end
+
+              push_batch(buffer, random) unless buffer.empty?
+              finalize_streaming
+            end
+
+            puts "Streamed #{@total} tests in #{duration.round(2)}s."
+            $stdout.flush
+          end
+
+          register
+          redis.expire(key('workers'), config.redis_ttl)
+          self
+        rescue *CONNECTION_ERRORS
+          raise if @master
+        end
+
         def populated?
           !!defined?(@index)
         end
 
+        def total
+          return @total if defined?(@total) && @total
+
+          redis.get(key('total')).to_i
+        rescue *CONNECTION_ERRORS
+          @total || 0
+        end
+
         def shutdown!
           @shutdown_required = true
         end
@@ -51,13 +104,18 @@ module CI
         DEFAULT_SLEEP_SECONDS = 0.5
 
         def poll
-          wait_for_master
+          wait_for_master(timeout: config.queue_init_timeout, allow_streaming: true)
           attempt = 0
           until shutdown_required? || config.circuit_breakers.any?(&:open?) || exhausted? || max_test_failed?
-            if test = reserve
+            if entry = reserve
               attempt = 0
-              yield index.fetch(test)
+              yield resolve_entry(entry)
             else
+              if still_streaming?
+                raise LostMaster, "Streaming stalled for more than #{config.lazy_load_streaming_timeout}s" if streaming_stale?
+                sleep 0.1
+                next
+              end
               # Adding exponential backoff to avoid hammering Redis
               # we just stay online here in case a test gets retried or times out so we can afford to wait
               sleep_time = [DEFAULT_SLEEP_SECONDS * (2 ** attempt), Redis.max_sleep_time].min
@@ -89,6 +147,7 @@ module CI
         def retry_queue
           failures = build.failed_tests.to_set
           log = redis.lrange(key('worker', worker_id, 'queue'), 0, -1)
+          log = log.map { |entry| queue_entry_test_id(entry) }
           log.select! { |id| failures.include?(id) }
           log.uniq!
           log.reverse!
@@ -103,23 +162,38 @@ module CI
           @build ||= CI::Queue::Redis::BuildRecord.new(self, redis, config)
         end
 
+        def file_loader
+          @file_loader ||= CI::Queue::FileLoader.new
+        end
+
+        def worker_queue_length
+          redis.llen(key('worker', worker_id, 'queue'))
+        rescue *CONNECTION_ERRORS
+          nil
+        end
+
         def report_worker_error(error)
           build.report_worker_error(error)
         end
 
         def acknowledge(test_key, error: nil, pipeline: redis)
-          raise_on_mismatching_test(test_key)
+          test_id = normalize_test_id(test_key)
+          assert_reserved!(test_id)
+          entry = reserved_entries.fetch(test_id, queue_entry_for(test_key))
+          unreserve_entry(test_id)
           eval_script(
             :acknowledge,
-            keys: [key('running'), key('processed'), key('owners'), key('error-reports')],
-            argv: [test_key, error.to_s, config.redis_ttl],
+            keys: [key('running'), key('processed'), key('owners'), key('error-reports'), key('requeued-by')],
+            argv: [entry, test_id, error.to_s, config.redis_ttl],
             pipeline: pipeline,
           ) == 1
         end
 
         def requeue(test, offset: Redis.requeue_offset)
-          test_key = test.id
-          raise_on_mismatching_test(test_key)
+          test_id = normalize_test_id(test)
+          assert_reserved!(test_id)
+          entry = reserved_entries.fetch(test_id, queue_entry_for(test))
+          unreserve_entry(test_id)
           global_max_requeues = config.global_max_requeues(total)
 
           requeued = config.max_requeues > 0 && global_max_requeues > 0 && eval_script(
@@ -132,11 +206,16 @@ module CI
               key('worker', worker_id, 'queue'),
               key('owners'),
               key('error-reports'),
+              key('requeued-by'),
             ],
-            argv: [config.max_requeues, global_max_requeues, test_key, offset],
+            argv: [config.max_requeues, global_max_requeues, entry, test_id, offset, config.redis_ttl],
           ) == 1
 
-          reserved_tests << test_key unless requeued
+          unless requeued
+            reserved_tests << test_id
+            reserved_entries[test_id] = entry
+            reserved_entry_ids[entry] = test_id
+          end
           requeued
         end
 
@@ -157,19 +236,131 @@ module CI
           @reserved_tests ||= Concurrent::Set.new
         end
 
+        def reserved_entries
+          @reserved_entries ||= Concurrent::Map.new
+        end
+
+        def reserved_entry_ids
+          @reserved_entry_ids ||= Concurrent::Map.new
+        end
+
         def worker_id
           config.worker_id
         end
 
-        def raise_on_mismatching_test(test)
-          unless reserved_tests.delete?(test)
-            raise ReservationError, "Acknowledged #{test.inspect} but only #{reserved_tests.map(&:inspect).join(", ")} reserved"
+        def assert_reserved!(test_id)
+          unless reserved_tests.include?(test_id)
+            raise ReservationError, "Acknowledged #{test_id.inspect} but only #{reserved_tests.map(&:inspect).join(", ")} reserved"
+          end
+        end
+
+        def reserve_entry(entry)
+          test_id = queue_entry_test_id(entry)
+          reserved_tests << test_id
+          reserved_entries[test_id] = entry
+          reserved_entry_ids[entry] = test_id
+        end
+
+        def unreserve_entry(test_id)
+          entry = reserved_entries.delete(test_id)
+          reserved_tests.delete(test_id)
+          reserved_entry_ids.delete(entry) if entry
+        end
+
+        def normalize_test_id(test_key)
+          key = test_key.respond_to?(:id) ? test_key.id : test_key
+          if key.is_a?(String)
+            cached = reserved_entry_ids[key]
+            return cached if cached
+          end
+          queue_entry_test_id(key)
+        end
+
+        def queue_entry_test_id(entry)
+          CI::Queue::QueueEntry.test_id(entry)
+        end
+
+        def queue_entry_for(test)
+          return test.queue_entry if test.respond_to?(:queue_entry)
+          return test.id if test.respond_to?(:id)
+
+          test
+        end
+
+        def resolve_entry(entry)
+          test_id = reserved_entry_ids[entry] || queue_entry_test_id(entry)
+          if populated?
+            return index[test_id] if index.key?(test_id)
+          end
+
+          return entry_resolver.call(entry) if entry_resolver
+
+          entry
+        end
+
+        def still_streaming?
+          master_status == 'streaming'
+        end
+
+        def streaming_stale?
+          timeout = config.lazy_load_streaming_timeout.to_i
+          updated_at = redis.get(key('streaming-updated-at'))
+          return true unless updated_at
+
+          (CI::Queue.time_now.to_f - updated_at.to_f) > timeout
+        rescue *CONNECTION_ERRORS
+          false
+        end
+
+        def start_streaming!
+          timeout = config.lazy_load_streaming_timeout.to_i
+          with_redis_timeout(5) do
+            redis.multi do |transaction|
+              transaction.set(key('total'), 0)
+              transaction.set(key('master-status'), 'streaming')
+              transaction.set(key('streaming-updated-at'), CI::Queue.time_now.to_f)
+              transaction.expire(key('streaming-updated-at'), timeout)
+              transaction.expire(key('queue'), config.redis_ttl)
+              transaction.expire(key('total'), config.redis_ttl)
+              transaction.expire(key('master-status'), config.redis_ttl)
+            end
+          end
+        end
+
+        def push_batch(tests, random)
+          # Use plain shuffle instead of Queue.shuffle — the custom shuffler expects
+          # test objects with .id, but streaming entries are pre-formatted strings.
+          entries = tests.shuffle(random: random).map { |test| queue_entry_for(test) }
+          return if entries.empty?
+
+          @total += entries.size
+          timeout = config.lazy_load_streaming_timeout.to_i
+          redis.multi do |transaction|
+            transaction.lpush(key('queue'), entries)
+            transaction.incrby(key('total'), entries.size)
+            transaction.set(key('master-status'), 'streaming')
+            transaction.set(key('streaming-updated-at'), CI::Queue.time_now.to_f)
+            transaction.expire(key('streaming-updated-at'), timeout)
+            transaction.expire(key('queue'), config.redis_ttl)
+            transaction.expire(key('total'), config.redis_ttl)
+            transaction.expire(key('master-status'), config.redis_ttl)
+          end
+        end
+
+        def finalize_streaming
+          redis.multi do |transaction|
+            transaction.set(key('master-status'), 'ready')
+            transaction.expire(key('master-status'), config.redis_ttl)
+            transaction.del(key('streaming-updated-at'))
           end
         end
 
         def reserve
-          (try_to_reserve_lost_test || try_to_reserve_test).tap do |test|
-            reserved_tests << test if test
+          (try_to_reserve_lost_test || try_to_reserve_test).tap do |entry|
+            if entry
+              @first_reserve_at ||= Process.clock_gettime(Process::CLOCK_MONOTONIC)
+              reserve_entry(entry)
+            end
           end
         end
 
@@ -182,8 +373,10 @@ module CI
               key('processed'),
               key('worker', worker_id, 'queue'),
               key('owners'),
+              key('requeued-by'),
+              key('workers'),
             ],
-            argv: [CI::Queue.time_now.to_f],
+            argv: [CI::Queue.time_now.to_f, Redis.requeue_offset],
           )
         end
 
@@ -198,7 +391,7 @@ module CI
               key('worker', worker_id, 'queue'),
               key('owners'),
             ],
-            argv: [CI::Queue.time_now.to_f, timeout],
+            argv: [CI::Queue.time_now.to_f, timeout, CI::Queue::QueueEntry::DELIMITER],
           )
 
           if lost_test
@@ -208,8 +401,8 @@ module CI
           lost_test
         end
 
-        def push(tests)
-          @total = tests.size
+        def push(entries)
+          @total = entries.size
 
           # We set a unique value (worker_id) and read it back to make "SET if Not eXists" idempotent in case of a retry.
           value = key('setup', worker_id)
@@ -227,7 +420,7 @@ module CI
               with_redis_timeout(5) do
                 redis.without_reconnect do
                   redis.multi do |transaction|
-                    transaction.lpush(key('queue'), tests) unless tests.empty?
+                    transaction.lpush(key('queue'), entries) unless entries.empty?
                     transaction.set(key('total'), @total)
                     transaction.set(key('master-status'), 'ready')
 

data/lib/ci/queue/redis.rb

@@ -11,7 +11,6 @@ require 'ci/queue/redis/retry'
 require 'ci/queue/redis/supervisor'
 require 'ci/queue/redis/grind_supervisor'
 require 'ci/queue/redis/test_time_record'
-require 'ci/queue/redis/key_shortener'
 
 module CI
   module Queue

data/lib/ci/queue/version.rb

@@ -2,7 +2,7 @@
 
 module CI
   module Queue
-    VERSION = '0.81.0'
+    VERSION = '0.83.0'
     DEV_SCRIPTS_ROOT = ::File.expand_path('../../../../../redis', __FILE__)
     RELEASE_SCRIPTS_ROOT = ::File.expand_path('../redis', __FILE__)
   end

data/lib/ci/queue.rb

@@ -14,6 +14,9 @@ require 'ci/queue/static'
 require 'ci/queue/file'
 require 'ci/queue/grind'
 require 'ci/queue/bisect'
+require 'ci/queue/queue_entry'
+require 'ci/queue/class_resolver'
+require 'ci/queue/file_loader'
 
 module CI
   module Queue
@@ -22,6 +25,18 @@ module CI
     attr_accessor :shuffler, :requeueable
 
     Error = Class.new(StandardError)
+    ClassNotFoundError = Class.new(Error)
+
+    class FileLoadError < Error
+      attr_reader :file_path, :original_error
+
+      def initialize(file_path, original_error)
+        @file_path = file_path
+        @original_error = original_error
+        super("Failed to load #{file_path}: #{original_error.class}: #{original_error.message}")
+        set_backtrace(original_error.backtrace)
+      end
+    end
 
     module Warnings
       RESERVED_LOST_TEST = :RESERVED_LOST_TEST
@@ -48,6 +63,17 @@ module CI
       end
     end
 
+    def debug?
+      return @debug if defined?(@debug)
+
+      value = ENV['CI_QUEUE_DEBUG']
+      @debug = !!(value && !value.strip.empty? && !%w[0 false].include?(value.strip.downcase))
+    end
+
+    def reset_debug!
+      remove_instance_variable(:@debug) if defined?(@debug)
+    end
+
     def from_uri(url, config)
       uri = URI(url)
       implementation = case uri.scheme
@@ -65,3 +91,4 @@ module CI
     end
   end
 end
+

data/lib/minitest/queue/build_status_recorder.rb

@@ -38,28 +38,46 @@ module Minitest
         super
 
         self.total_time = Minitest.clock_time - start_time
-        if test.requeued?
-          self.requeues += 1
-        elsif test.skipped?
-          self.skips += 1
-        elsif test.error?
-          self.errors += 1
-        elsif test.failure
-          self.failures += 1
-        end
+        
+        # Determine what type of result this is and record it
+        test_id = "#{test.klass}##{test.name}"
+        delta = delta_for(test)
 
-        stats = COUNTERS.zip(COUNTERS.map { |c| send(c) }).to_h
-        if (test.failure || test.error?) && !test.skipped?
-          build.record_error("#{test.klass}##{test.name}", dump(test), stats: stats)
+        acknowledged = if (test.failure || test.error?) && !test.skipped?
+          build.record_error(test_id, dump(test), stat_delta: delta)
         elsif test.requeued?
-          build.record_requeue("#{test.klass}##{test.name}", stats: stats)
+          build.record_requeue(test_id)
         else
-          build.record_success("#{test.klass}##{test.name}", stats: stats, skip_flaky_record: test.skipped?)
+          build.record_success(test_id, skip_flaky_record: test.skipped?)
+        end
+
+        if acknowledged
+          if (test.failure || test.error?) && !test.skipped?
+            test.error? ? self.errors += 1 : self.failures += 1
+          elsif test.requeued?
+            self.requeues += 1
+          elsif test.skipped?
+            self.skips += 1
+          end
+          # Apply delta to Redis (record_success returns true when ack'd or when we replaced a failure)
+          build.record_stats_delta(delta)
         end
       end
 
       private
 
+      def delta_for(test)
+        h = { 'assertions' => (test.assertions || 0).to_i, 'errors' => 0, 'failures' => 0, 'skips' => 0, 'requeues' => 0, 'total_time' => test.time.to_f }
+        if (test.failure || test.error?) && !test.skipped?
+          test.error? ? h['errors'] = 1 : h['failures'] = 1
+        elsif test.requeued?
+          h['requeues'] = 1
+        elsif test.skipped?
+          h['skips'] = 1
+        end
+        h
+      end
+
       def dump(test)
         ErrorReport.new(self.class.failure_formatter.new(test).to_h).dump
       end

data/lib/minitest/queue/grind_recorder.rb

@@ -32,12 +32,12 @@ module Minitest
       private
 
       def record_test(test)
-        stats = self.class.counters
         if (test.failure || test.error?) && !test.skipped?
-          build.record_error(dump(test), stats: stats)
+          build.record_error(dump(test))
         else
-          build.record_success(stats: stats)
+          build.record_success
         end
+        build.record_stats(self.class.counters)
       end
 
       def increment_counter(test)

data/lib/minitest/queue/junit_reporter.rb

@@ -136,9 +136,9 @@ module Minitest
         result[:time] = 0
         tests.each do |test|
           result[:"#{result(test)}_count"] += 1
-          result[:assertion_count] += test.assertions
+          result[:assertion_count] += test.assertions || 0
           result[:test_count] += 1
-          result[:time] += test.time
+          result[:time] += test.time || 0
         end
         result
       end

data/lib/minitest/queue/lazy_entry_resolver.rb

@@ -0,0 +1,55 @@
+# frozen_string_literal: true
+
+module Minitest
+  module Queue
+    class LazyEntryResolver
+      def initialize(loader:, resolver:)
+        @loader = loader
+        @resolver = resolver
+      end
+
+      def call(entry)
+        parsed = CI::Queue::QueueEntry.parse(entry)
+        class_name, method_name = parsed.fetch(:test_id).split('#', 2)
+        if CI::Queue::QueueEntry.load_error_payload?(parsed[:file_path])
+          payload = CI::Queue::QueueEntry.decode_load_error(parsed[:file_path])
+          if payload
+            error = StandardError.new("#{payload['error_class']}: #{payload['error_message']}")
+            error.set_backtrace(payload['backtrace']) if payload['backtrace']
+            load_error = CI::Queue::FileLoadError.new(payload['file_path'], error)
+            return Minitest::Queue::LazySingleExample.new(
+              class_name,
+              method_name,
+              payload['file_path'],
+              loader: @loader,
+              resolver: @resolver,
+              load_error: load_error,
+              queue_entry: entry,
+            )
+          else
+            error = StandardError.new("Corrupt load error payload for #{class_name}##{method_name}")
+            load_error = CI::Queue::FileLoadError.new("unknown", error)
+            return Minitest::Queue::LazySingleExample.new(
+              class_name,
+              method_name,
+              nil,
+              loader: @loader,
+              resolver: @resolver,
+              load_error: load_error,
+              queue_entry: entry,
+            )
+          end
+        end
+
+        Minitest::Queue::LazySingleExample.new(
+          class_name,
+          method_name,
+          parsed[:file_path],
+          loader: @loader,
+          resolver: @resolver,
+          queue_entry: entry,
+        )
+      end
+    end
+  end
+end