ci-queue 0.82.0 → 0.83.0

data/lib/ci/queue/redis/requeue.lua

@@ -6,17 +6,20 @@ local zset_key = KEYS[4]
 local worker_queue_key = KEYS[5]
 local owners_key = KEYS[6]
 local error_reports_key = KEYS[7]
+local requeued_by_key = KEYS[8]
 
 local max_requeues = tonumber(ARGV[1])
 local global_max_requeues = tonumber(ARGV[2])
-local test = ARGV[3]
-local offset = ARGV[4]
+local entry = ARGV[3]
+local test_id = ARGV[4]
+local offset = ARGV[5]
+local ttl = tonumber(ARGV[6])
 
-if redis.call('hget', owners_key, test) == worker_queue_key then
-   redis.call('hdel', owners_key, test)
+if redis.call('hget', owners_key, entry) == worker_queue_key then
+   redis.call('hdel', owners_key, entry)
 end
 
-if redis.call('sismember', processed_key, test) == 1 then
+if redis.call('sismember', processed_key, test_id) == 1 then
   return false
 end
 
@@ -25,23 +28,28 @@ if global_requeues and global_requeues >= tonumber(global_max_requeues) then
   return false
 end
 
-local requeues = tonumber(redis.call('hget', requeues_count_key, test))
+local requeues = tonumber(redis.call('hget', requeues_count_key, test_id))
 if requeues and requeues >= max_requeues then
   return false
 end
 
 redis.call('hincrby', requeues_count_key, '___total___', 1)
-redis.call('hincrby', requeues_count_key, test, 1)
+redis.call('hincrby', requeues_count_key, test_id, 1)
 
-redis.call('hdel', error_reports_key, test)
+redis.call('hdel', error_reports_key, test_id)
 
 local pivot = redis.call('lrange', queue_key, -1 - offset, 0 - offset)[1]
 if pivot then
-  redis.call('linsert', queue_key, 'BEFORE', pivot, test)
+  redis.call('linsert', queue_key, 'BEFORE', pivot, entry)
 else
-  redis.call('lpush', queue_key, test)
+  redis.call('lpush', queue_key, entry)
 end
 
-redis.call('zrem', zset_key, test)
+redis.call('hset', requeued_by_key, entry, worker_queue_key)
+if ttl and ttl > 0 then
+  redis.call('expire', requeued_by_key, ttl)
+end
+
+redis.call('zrem', zset_key, entry)
 
 return true

data/lib/ci/queue/redis/reserve.lua

@@ -4,15 +4,54 @@ local zset_key = KEYS[2]
 local processed_key = KEYS[3]
 local worker_queue_key = KEYS[4]
 local owners_key = KEYS[5]
+local requeued_by_key = KEYS[6]
+local workers_key = KEYS[7]
 
 local current_time = ARGV[1]
+local defer_offset = tonumber(ARGV[2]) or 0
+local max_skip_attempts = 4
 
-local test = redis.call('rpop', queue_key)
-if test then
-  redis.call('zadd', zset_key, current_time, test)
-  redis.call('lpush', worker_queue_key, test)
-  redis.call('hset', owners_key, test, worker_queue_key)
-  return test
-else
-  return nil
+local function insert_with_offset(test)
+  local pivot = redis.call('lrange', queue_key, -1 - defer_offset, 0 - defer_offset)[1]
+  if pivot then
+    redis.call('linsert', queue_key, 'BEFORE', pivot, test)
+  else
+    redis.call('lpush', queue_key, test)
+  end
 end
+
+for attempt = 1, max_skip_attempts do
+  local test = redis.call('rpop', queue_key)
+  if not test then
+    return nil
+  end
+
+  local requeued_by = redis.call('hget', requeued_by_key, test)
+  if requeued_by == worker_queue_key then
+    -- If this build only has one worker, allow immediate self-pickup.
+    if redis.call('scard', workers_key) <= 1 then
+      redis.call('hdel', requeued_by_key, test)
+      redis.call('zadd', zset_key, current_time, test)
+      redis.call('lpush', worker_queue_key, test)
+      redis.call('hset', owners_key, test, worker_queue_key)
+      return test
+    end
+
+    insert_with_offset(test)
+
+    -- If this worker only finds its own requeued tests, defer once by returning nil,
+    -- then allow pickup on a subsequent reserve attempt.
+    if attempt == max_skip_attempts then
+      redis.call('hdel', requeued_by_key, test)
+      return nil
+    end
+  else
+    redis.call('hdel', requeued_by_key, test)
+    redis.call('zadd', zset_key, current_time, test)
+    redis.call('lpush', worker_queue_key, test)
+    redis.call('hset', owners_key, test, worker_queue_key)
+    return test
+  end
+end
+
+return nil

data/lib/ci/queue/redis/reserve_lost.lua

@@ -1,4 +1,6 @@
 -- AUTOGENERATED FILE DO NOT EDIT DIRECTLY
+-- @include _entry_helpers
+
 local zset_key = KEYS[1]
 local processed_key = KEYS[2]
 local worker_queue_key = KEYS[3]
@@ -6,10 +8,12 @@ local owners_key = KEYS[4]
 
 local current_time = ARGV[1]
 local timeout = ARGV[2]
+local entry_delimiter = ARGV[3]
 
 local lost_tests = redis.call('zrangebyscore', zset_key, 0, current_time - timeout)
 for _, test in ipairs(lost_tests) do
-  if redis.call('sismember', processed_key, test) == 0 then
+  local test_id = test_id_from_entry(test, entry_delimiter)
+  if redis.call('sismember', processed_key, test_id) == 0 then
     redis.call('zadd', zset_key, current_time, test)
     redis.call('lpush', worker_queue_key, test)
     redis.call('hset', owners_key, test, worker_queue_key) -- Take ownership

data/lib/ci/queue/redis/supervisor.rb

@@ -9,7 +9,7 @@ module CI
         end
 
         def total
-          wait_for_master(timeout: config.queue_init_timeout)
+          wait_for_master(timeout: config.queue_init_timeout, allow_streaming: true)
           redis.get(key('total')).to_i
         end
 
@@ -19,7 +19,7 @@ module CI
 
         def wait_for_workers
           duration = measure do
-            wait_for_master(timeout: config.queue_init_timeout)
+            wait_for_master(timeout: config.queue_init_timeout, allow_streaming: true)
           end
 
           yield if block_given?
@@ -30,7 +30,7 @@ module CI
             @time_left -= 1
             sleep 1
 
-            if active_workers?
+            if active_workers? || streaming?
               @time_left_with_no_workers = config.inactive_workers_timeout
             else
               @time_left_with_no_workers -= 1

data/lib/ci/queue/redis/worker.rb

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 require 'ci/queue/static'
 require 'concurrent/set'
+require 'concurrent/map'
 
 module CI
   module Queue
@@ -13,11 +14,13 @@ module CI
       self.max_sleep_time = 2
 
       class Worker < Base
-        attr_reader :total
+        attr_accessor :entry_resolver
+        attr_reader :first_reserve_at
 
         def initialize(redis, config)
           @reserved_tests = Concurrent::Set.new
           @shutdown_required = false
+          @first_reserve_at = nil
           super(redis, config)
         end
 
@@ -27,15 +30,65 @@ module CI
 
         def populate(tests, random: Random.new)
           @index = tests.map { |t| [t.id, t] }.to_h
-          tests = Queue.shuffle(tests, random)
-          push(tests.map(&:id))
+          entries = Queue.shuffle(tests, random).map { |test| queue_entry_for(test) }
+          push(entries)
           self
         end
 
+        def stream_populate(tests, random: Random.new, batch_size: 10_000)
+          batch_size = batch_size.to_i
+          batch_size = 1 if batch_size < 1
+
+          value = key('setup', worker_id)
+          _, status = redis.pipelined do |pipeline|
+            pipeline.set(key('master-status'), value, nx: true)
+            pipeline.get(key('master-status'))
+          end
+
+          if @master = (value == status)
+            @total = 0
+            puts "Worker elected as leader, streaming tests to the queue."
+
+            duration = measure do
+              start_streaming!
+              buffer = []
+
+              tests.each do |test|
+                buffer << test
+
+                if buffer.size >= batch_size
+                  push_batch(buffer, random)
+                  buffer.clear
+                end
+              end
+
+              push_batch(buffer, random) unless buffer.empty?
+              finalize_streaming
+            end
+
+            puts "Streamed #{@total} tests in #{duration.round(2)}s."
+            $stdout.flush
+          end
+
+          register
+          redis.expire(key('workers'), config.redis_ttl)
+          self
+        rescue *CONNECTION_ERRORS
+          raise if @master
+        end
+
         def populated?
           !!defined?(@index)
         end
 
+        def total
+          return @total if defined?(@total) && @total
+
+          redis.get(key('total')).to_i
+        rescue *CONNECTION_ERRORS
+          @total || 0
+        end
+
         def shutdown!
           @shutdown_required = true
         end
@@ -51,13 +104,18 @@ module CI
         DEFAULT_SLEEP_SECONDS = 0.5
 
         def poll
-          wait_for_master
+          wait_for_master(timeout: config.queue_init_timeout, allow_streaming: true)
           attempt = 0
           until shutdown_required? || config.circuit_breakers.any?(&:open?) || exhausted? || max_test_failed?
-            if test = reserve
+            if entry = reserve
               attempt = 0
-              yield index.fetch(test)
+              yield resolve_entry(entry)
             else
+              if still_streaming?
+                raise LostMaster, "Streaming stalled for more than #{config.lazy_load_streaming_timeout}s" if streaming_stale?
+                sleep 0.1
+                next
+              end
               # Adding exponential backoff to avoid hammering Redis
               # we just stay online here in case a test gets retried or times out so we can afford to wait
               sleep_time = [DEFAULT_SLEEP_SECONDS * (2 ** attempt), Redis.max_sleep_time].min
@@ -89,6 +147,7 @@ module CI
         def retry_queue
           failures = build.failed_tests.to_set
           log = redis.lrange(key('worker', worker_id, 'queue'), 0, -1)
+          log = log.map { |entry| queue_entry_test_id(entry) }
           log.select! { |id| failures.include?(id) }
           log.uniq!
           log.reverse!
@@ -103,23 +162,38 @@ module CI
           @build ||= CI::Queue::Redis::BuildRecord.new(self, redis, config)
         end
 
+        def file_loader
+          @file_loader ||= CI::Queue::FileLoader.new
+        end
+
+        def worker_queue_length
+          redis.llen(key('worker', worker_id, 'queue'))
+        rescue *CONNECTION_ERRORS
+          nil
+        end
+
         def report_worker_error(error)
           build.report_worker_error(error)
         end
 
         def acknowledge(test_key, error: nil, pipeline: redis)
-          raise_on_mismatching_test(test_key)
+          test_id = normalize_test_id(test_key)
+          assert_reserved!(test_id)
+          entry = reserved_entries.fetch(test_id, queue_entry_for(test_key))
+          unreserve_entry(test_id)
           eval_script(
             :acknowledge,
-            keys: [key('running'), key('processed'), key('owners'), key('error-reports')],
-            argv: [test_key, error.to_s, config.redis_ttl],
+            keys: [key('running'), key('processed'), key('owners'), key('error-reports'), key('requeued-by')],
+            argv: [entry, test_id, error.to_s, config.redis_ttl],
             pipeline: pipeline,
           ) == 1
         end
 
         def requeue(test, offset: Redis.requeue_offset)
-          test_key = test.id
-          raise_on_mismatching_test(test_key)
+          test_id = normalize_test_id(test)
+          assert_reserved!(test_id)
+          entry = reserved_entries.fetch(test_id, queue_entry_for(test))
+          unreserve_entry(test_id)
           global_max_requeues = config.global_max_requeues(total)
 
           requeued = config.max_requeues > 0 && global_max_requeues > 0 && eval_script(
@@ -132,11 +206,16 @@ module CI
               key('worker', worker_id, 'queue'),
               key('owners'),
               key('error-reports'),
+              key('requeued-by'),
             ],
-            argv: [config.max_requeues, global_max_requeues, test_key, offset],
+            argv: [config.max_requeues, global_max_requeues, entry, test_id, offset, config.redis_ttl],
           ) == 1
 
-          reserved_tests << test_key unless requeued
+          unless requeued
+            reserved_tests << test_id
+            reserved_entries[test_id] = entry
+            reserved_entry_ids[entry] = test_id
+          end
           requeued
         end
 
@@ -157,19 +236,131 @@ module CI
           @reserved_tests ||= Concurrent::Set.new
         end
 
+        def reserved_entries
+          @reserved_entries ||= Concurrent::Map.new
+        end
+
+        def reserved_entry_ids
+          @reserved_entry_ids ||= Concurrent::Map.new
+        end
+
         def worker_id
           config.worker_id
         end
 
-        def raise_on_mismatching_test(test)
-          unless reserved_tests.delete?(test)
-            raise ReservationError, "Acknowledged #{test.inspect} but only #{reserved_tests.map(&:inspect).join(", ")} reserved"
+        def assert_reserved!(test_id)
+          unless reserved_tests.include?(test_id)
+            raise ReservationError, "Acknowledged #{test_id.inspect} but only #{reserved_tests.map(&:inspect).join(", ")} reserved"
+          end
+        end
+
+        def reserve_entry(entry)
+          test_id = queue_entry_test_id(entry)
+          reserved_tests << test_id
+          reserved_entries[test_id] = entry
+          reserved_entry_ids[entry] = test_id
+        end
+
+        def unreserve_entry(test_id)
+          entry = reserved_entries.delete(test_id)
+          reserved_tests.delete(test_id)
+          reserved_entry_ids.delete(entry) if entry
+        end
+
+        def normalize_test_id(test_key)
+          key = test_key.respond_to?(:id) ? test_key.id : test_key
+          if key.is_a?(String)
+            cached = reserved_entry_ids[key]
+            return cached if cached
+          end
+          queue_entry_test_id(key)
+        end
+
+        def queue_entry_test_id(entry)
+          CI::Queue::QueueEntry.test_id(entry)
+        end
+
+        def queue_entry_for(test)
+          return test.queue_entry if test.respond_to?(:queue_entry)
+          return test.id if test.respond_to?(:id)
+
+          test
+        end
+
+        def resolve_entry(entry)
+          test_id = reserved_entry_ids[entry] || queue_entry_test_id(entry)
+          if populated?
+            return index[test_id] if index.key?(test_id)
+          end
+
+          return entry_resolver.call(entry) if entry_resolver
+
+          entry
+        end
+
+        def still_streaming?
+          master_status == 'streaming'
+        end
+
+        def streaming_stale?
+          timeout = config.lazy_load_streaming_timeout.to_i
+          updated_at = redis.get(key('streaming-updated-at'))
+          return true unless updated_at
+
+          (CI::Queue.time_now.to_f - updated_at.to_f) > timeout
+        rescue *CONNECTION_ERRORS
+          false
+        end
+
+        def start_streaming!
+          timeout = config.lazy_load_streaming_timeout.to_i
+          with_redis_timeout(5) do
+            redis.multi do |transaction|
+              transaction.set(key('total'), 0)
+              transaction.set(key('master-status'), 'streaming')
+              transaction.set(key('streaming-updated-at'), CI::Queue.time_now.to_f)
+              transaction.expire(key('streaming-updated-at'), timeout)
+              transaction.expire(key('queue'), config.redis_ttl)
+              transaction.expire(key('total'), config.redis_ttl)
+              transaction.expire(key('master-status'), config.redis_ttl)
+            end
+          end
+        end
+
+        def push_batch(tests, random)
+          # Use plain shuffle instead of Queue.shuffle — the custom shuffler expects
+          # test objects with .id, but streaming entries are pre-formatted strings.
+          entries = tests.shuffle(random: random).map { |test| queue_entry_for(test) }
+          return if entries.empty?
+
+          @total += entries.size
+          timeout = config.lazy_load_streaming_timeout.to_i
+          redis.multi do |transaction|
+            transaction.lpush(key('queue'), entries)
+            transaction.incrby(key('total'), entries.size)
+            transaction.set(key('master-status'), 'streaming')
+            transaction.set(key('streaming-updated-at'), CI::Queue.time_now.to_f)
+            transaction.expire(key('streaming-updated-at'), timeout)
+            transaction.expire(key('queue'), config.redis_ttl)
+            transaction.expire(key('total'), config.redis_ttl)
+            transaction.expire(key('master-status'), config.redis_ttl)
+          end
+        end
+
+        def finalize_streaming
+          redis.multi do |transaction|
+            transaction.set(key('master-status'), 'ready')
+            transaction.expire(key('master-status'), config.redis_ttl)
+            transaction.del(key('streaming-updated-at'))
           end
         end
 
         def reserve
-          (try_to_reserve_lost_test || try_to_reserve_test).tap do |test|
-            reserved_tests << test if test
+          (try_to_reserve_lost_test || try_to_reserve_test).tap do |entry|
+            if entry
+              @first_reserve_at ||= Process.clock_gettime(Process::CLOCK_MONOTONIC)
+              reserve_entry(entry)
+            end
           end
         end
 
@@ -182,8 +373,10 @@ module CI
               key('processed'),
               key('worker', worker_id, 'queue'),
               key('owners'),
+              key('requeued-by'),
+              key('workers'),
             ],
-            argv: [CI::Queue.time_now.to_f],
+            argv: [CI::Queue.time_now.to_f, Redis.requeue_offset],
           )
         end
 
@@ -198,7 +391,7 @@ module CI
               key('worker', worker_id, 'queue'),
               key('owners'),
             ],
-            argv: [CI::Queue.time_now.to_f, timeout],
+            argv: [CI::Queue.time_now.to_f, timeout, CI::Queue::QueueEntry::DELIMITER],
           )
 
           if lost_test
@@ -208,8 +401,8 @@ module CI
           lost_test
         end
 
-        def push(tests)
-          @total = tests.size
+        def push(entries)
+          @total = entries.size
 
           # We set a unique value (worker_id) and read it back to make "SET if Not eXists" idempotent in case of a retry.
           value = key('setup', worker_id)
@@ -227,7 +420,7 @@ module CI
               with_redis_timeout(5) do
                 redis.without_reconnect do
                   redis.multi do |transaction|
-                    transaction.lpush(key('queue'), tests) unless tests.empty?
+                    transaction.lpush(key('queue'), entries) unless entries.empty?
                     transaction.set(key('total'), @total)
                     transaction.set(key('master-status'), 'ready')
 

data/lib/ci/queue/version.rb

@@ -2,7 +2,7 @@
 
 module CI
   module Queue
-    VERSION = '0.82.0'
+    VERSION = '0.83.0'
     DEV_SCRIPTS_ROOT = ::File.expand_path('../../../../../redis', __FILE__)
     RELEASE_SCRIPTS_ROOT = ::File.expand_path('../redis', __FILE__)
   end

data/lib/ci/queue.rb

@@ -14,6 +14,9 @@ require 'ci/queue/static'
 require 'ci/queue/file'
 require 'ci/queue/grind'
 require 'ci/queue/bisect'
+require 'ci/queue/queue_entry'
+require 'ci/queue/class_resolver'
+require 'ci/queue/file_loader'
 
 module CI
   module Queue
@@ -22,6 +25,18 @@ module CI
     attr_accessor :shuffler, :requeueable
 
     Error = Class.new(StandardError)
+    ClassNotFoundError = Class.new(Error)
+
+    class FileLoadError < Error
+      attr_reader :file_path, :original_error
+
+      def initialize(file_path, original_error)
+        @file_path = file_path
+        @original_error = original_error
+        super("Failed to load #{file_path}: #{original_error.class}: #{original_error.message}")
+        set_backtrace(original_error.backtrace)
+      end
+    end
 
     module Warnings
       RESERVED_LOST_TEST = :RESERVED_LOST_TEST
@@ -48,6 +63,17 @@ module CI
       end
     end
 
+    def debug?
+      return @debug if defined?(@debug)
+
+      value = ENV['CI_QUEUE_DEBUG']
+      @debug = !!(value && !value.strip.empty? && !%w[0 false].include?(value.strip.downcase))
+    end
+
+    def reset_debug!
+      remove_instance_variable(:@debug) if defined?(@debug)
+    end
+
     def from_uri(url, config)
       uri = URI(url)
       implementation = case uri.scheme
@@ -65,3 +91,4 @@ module CI
     end
   end
 end
+

data/lib/minitest/queue/junit_reporter.rb

@@ -136,9 +136,9 @@ module Minitest
         result[:time] = 0
         tests.each do |test|
           result[:"#{result(test)}_count"] += 1
-          result[:assertion_count] += test.assertions
+          result[:assertion_count] += test.assertions || 0
           result[:test_count] += 1
-          result[:time] += test.time
+          result[:time] += test.time || 0
         end
         result
       end

data/lib/minitest/queue/lazy_entry_resolver.rb

@@ -0,0 +1,55 @@
+# frozen_string_literal: true
+
+module Minitest
+  module Queue
+    class LazyEntryResolver
+      def initialize(loader:, resolver:)
+        @loader = loader
+        @resolver = resolver
+      end
+
+      def call(entry)
+        parsed = CI::Queue::QueueEntry.parse(entry)
+        class_name, method_name = parsed.fetch(:test_id).split('#', 2)
+        if CI::Queue::QueueEntry.load_error_payload?(parsed[:file_path])
+          payload = CI::Queue::QueueEntry.decode_load_error(parsed[:file_path])
+          if payload
+            error = StandardError.new("#{payload['error_class']}: #{payload['error_message']}")
+            error.set_backtrace(payload['backtrace']) if payload['backtrace']
+            load_error = CI::Queue::FileLoadError.new(payload['file_path'], error)
+            return Minitest::Queue::LazySingleExample.new(
+              class_name,
+              method_name,
+              payload['file_path'],
+              loader: @loader,
+              resolver: @resolver,
+              load_error: load_error,
+              queue_entry: entry,
+            )
+          else
+            error = StandardError.new("Corrupt load error payload for #{class_name}##{method_name}")
+            load_error = CI::Queue::FileLoadError.new("unknown", error)
+            return Minitest::Queue::LazySingleExample.new(
+              class_name,
+              method_name,
+              nil,
+              loader: @loader,
+              resolver: @resolver,
+              load_error: load_error,
+              queue_entry: entry,
+            )
+          end
+        end
+
+        Minitest::Queue::LazySingleExample.new(
+          class_name,
+          method_name,
+          parsed[:file_path],
+          loader: @loader,
+          resolver: @resolver,
+          queue_entry: entry,
+        )
+      end
+    end
+  end
+end