ci-queue 0.83.0 → 0.85.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: '08152aacde705d2472151b351e690db68a0c067c8930bf9574e1835825ee225b'
-  data.tar.gz: 37fc5a7e9174d9188ff39568a0e4a5bf4f1ae0e3bce0f08e4d9d66e2a5a5264b
+  metadata.gz: d7f6896ca23e9f9cd64dae9300539d0d5d88ced4c9b59a193d67824d353726cd
+  data.tar.gz: 969ba0b363dbe7e491561095587b802f24aa400685a96bcb5fda4b23e69dc02b
 SHA512:
-  metadata.gz: 0ca31505b3ca58115b632d1a23e7884e2bbc25ae2360f4228b08192705200303ddecd03220ff86ebb6c6c82b7e580f83f7d86e5f79d10d59cd6949c4b6947d42
-  data.tar.gz: f47440362656cc4ddf8ad52a9c49c9289b70f0a64312eb2c0c3f82e6bc9a980e6414a2c03ccb6613508975bf7b64e6b6232d1083fbbfc4c159b6eac0cb522e51
+  metadata.gz: dd6d91458d235f13c60df0f27374d09d3050d54354a7d87f0d427c21c558bac9338054a901a350eda42c37118af7893cfb763db96ab7be43b1f0d6a275ac2e6d
+  data.tar.gz: 833e87a9a0fd535589267462f98f81d134342f66432a43b246a0bc68632a64b85e9d80253a27d9ccf627898a5d846ea37d2c5bef7d2a602905d7c21195ddbff6

data/.ruby-version

@@ -1 +1 @@
-3.3.0
+4.0

data/Gemfile.lock

@@ -1,95 +1,105 @@
 PATH
   remote: .
   specs:
-    ci-queue (0.83.0)
+    ci-queue (0.85.0)
       logger
 
 GEM
   remote: https://rubygems.org/
   specs:
-    activesupport (7.1.3.2)
+    activesupport (8.1.3)
       base64
       bigdecimal
-      concurrent-ruby (~> 1.0, >= 1.0.2)
+      concurrent-ruby (~> 1.0, >= 1.3.1)
       connection_pool (>= 2.2.5)
       drb
       i18n (>= 1.6, < 2)
+      json
+      logger (>= 1.4.2)
       minitest (>= 5.1)
-      mutex_m
-      tzinfo (~> 2.0)
+      securerandom (>= 0.3)
+      tzinfo (~> 2.0, >= 2.0.5)
+      uri (>= 0.13.1)
     ansi (1.5.0)
-    ast (2.4.2)
-    base64 (0.2.0)
-    bigdecimal (3.1.7)
-    builder (3.2.4)
-    concurrent-ruby (1.2.3)
-    connection_pool (2.4.1)
-    diff-lcs (1.5.1)
-    docile (1.4.0)
-    drb (2.2.1)
-    i18n (1.14.4)
+    ast (2.4.3)
+    base64 (0.3.0)
+    benchmark (0.5.0)
+    bigdecimal (4.0.1)
+    builder (3.3.0)
+    concurrent-ruby (1.3.6)
+    connection_pool (3.0.2)
+    diff-lcs (1.6.2)
+    docile (1.4.1)
+    drb (2.2.3)
+    i18n (1.14.8)
       concurrent-ruby (~> 1.0)
-    json (2.7.1)
-    language_server-protocol (3.17.0.3)
-    logger (1.6.1)
-    minitest (5.22.3)
-    minitest-reporters (1.6.1)
+    json (2.19.3)
+    language_server-protocol (3.17.0.5)
+    lint_roller (1.1.0)
+    logger (1.7.0)
+    minitest (5.27.0)
+    minitest-reporters (1.7.1)
       ansi
       builder
       minitest (>= 5.0)
       ruby-progressbar
-    msgpack (1.7.2)
-    mutex_m (0.2.0)
-    parallel (1.24.0)
-    parser (3.3.0.5)
+    msgpack (1.8.0)
+    parallel (1.27.0)
+    parser (3.3.10.2)
       ast (~> 2.4.1)
       racc
-    racc (1.7.3)
+    prism (1.9.0)
+    racc (1.8.1)
     rainbow (3.1.1)
-    rake (13.1.0)
-    redis (5.1.0)
-      redis-client (>= 0.17.0)
-    redis-client (0.21.1)
+    rake (13.3.1)
+    redis (5.4.1)
+      redis-client (>= 0.22.0)
+    redis-client (0.28.0)
       connection_pool
-    regexp_parser (2.9.0)
-    rexml (3.3.8)
-    rspec (3.13.0)
+    regexp_parser (2.11.3)
+    rexml (3.4.4)
+    rspec (3.13.2)
       rspec-core (~> 3.13.0)
       rspec-expectations (~> 3.13.0)
       rspec-mocks (~> 3.13.0)
-    rspec-core (3.13.0)
+    rspec-core (3.13.6)
       rspec-support (~> 3.13.0)
-    rspec-expectations (3.13.0)
+    rspec-expectations (3.13.5)
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.13.0)
-    rspec-mocks (3.13.0)
+    rspec-mocks (3.13.8)
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.13.0)
-    rspec-support (3.13.1)
-    rubocop (1.62.1)
+    rspec-support (3.13.7)
+    rubocop (1.86.0)
       json (~> 2.3)
-      language_server-protocol (>= 3.17.0)
+      language_server-protocol (~> 3.17.0.2)
+      lint_roller (~> 1.1.0)
       parallel (~> 1.10)
       parser (>= 3.3.0.2)
       rainbow (>= 2.2.2, < 4.0)
-      regexp_parser (>= 1.8, < 3.0)
-      rexml (>= 3.2.5, < 4.0)
-      rubocop-ast (>= 1.31.1, < 2.0)
+      regexp_parser (>= 2.9.3, < 3.0)
+      rubocop-ast (>= 1.49.0, < 2.0)
       ruby-progressbar (~> 1.7)
-      unicode-display_width (>= 2.4.0, < 3.0)
-    rubocop-ast (1.31.2)
-      parser (>= 3.3.0.4)
+      unicode-display_width (>= 2.4.0, < 4.0)
+    rubocop-ast (1.49.1)
+      parser (>= 3.3.7.2)
+      prism (~> 1.7)
     ruby-progressbar (1.13.0)
+    securerandom (0.4.1)
     simplecov (0.22.0)
       docile (~> 1.1)
       simplecov-html (~> 0.11)
       simplecov_json_formatter (~> 0.1)
-    simplecov-html (0.12.3)
+    simplecov-html (0.13.2)
     simplecov_json_formatter (0.1.4)
-    snappy (0.4.0)
+    snappy (0.5.1)
     tzinfo (2.0.6)
       concurrent-ruby (~> 1.0)
-    unicode-display_width (2.5.0)
+    unicode-display_width (3.2.0)
+      unicode-emoji (~> 4.1)
+    unicode-emoji (4.2.0)
+    uri (1.1.1)
 
 PLATFORMS
   arm64-darwin-23
@@ -97,6 +107,7 @@ PLATFORMS
 
 DEPENDENCIES
   activesupport
+  benchmark
   bundler
   ci-queue!
   minitest (~> 5.11)
@@ -104,6 +115,7 @@ DEPENDENCIES
   msgpack
   rake
   redis
+  rexml
   rspec (~> 3.10)
   rubocop
   simplecov (~> 0.12)

data/ci-queue.gemspec

@@ -19,7 +19,7 @@ Gem::Specification.new do |spec|
   spec.homepage      = 'https://github.com/Shopify/ci-queue'
   spec.license       = 'MIT'
 
-  spec.required_ruby_version = '>= 2.7'
+  spec.required_ruby_version = '>= 3.1'
 
   spec.files         = lua_scripts + `git ls-files -z`.split("\x0").reject do |f|
     f.match(%r{^(test|spec|features)/})
@@ -43,5 +43,7 @@ Gem::Specification.new do |spec|
 
   spec.add_development_dependency 'snappy'
   spec.add_development_dependency 'msgpack'
+  spec.add_development_dependency 'benchmark'
+  spec.add_development_dependency 'rexml'
   spec.add_development_dependency 'rubocop'
 end

data/lib/ci/queue/build_record.rb

@@ -18,17 +18,17 @@ module CI
         @queue.exhausted?
       end
 
-      def record_error(id, payload, stat_delta: nil)
-        error_reports[id] = payload
+      def record_error(entry, payload, stat_delta: nil)
+        error_reports[entry] = payload
         true
       end
 
-      def record_success(id, skip_flaky_record: false, acknowledge: true)
-        error_reports.delete(id)
+      def record_success(entry, skip_flaky_record: false, acknowledge: true)
+        error_reports.delete(entry)
         true
       end
 
-      def record_requeue(id)
+      def record_requeue(entry)
         true
       end
 

data/lib/ci/queue/queue_entry.rb

@@ -6,26 +6,18 @@ require 'json'
 module CI
   module Queue
     module QueueEntry
-      DELIMITER = "\t"
       LOAD_ERROR_PREFIX = '__ciq_load_error__:'.freeze
 
       def self.test_id(entry)
-        pos = entry.index(DELIMITER)
-        pos ? entry[0, pos] : entry
+        JSON.parse(entry, symbolize_names: true)[:test_id]
       end
 
       def self.parse(entry)
-        return { test_id: entry, file_path: nil } unless entry.include?(DELIMITER)
-
-        test_id, file_path = entry.split(DELIMITER, 2)
-        file_path = nil if file_path == ""
-        { test_id: test_id, file_path: file_path }
+        JSON.parse(entry, symbolize_names: true)
       end
 
       def self.format(test_id, file_path)
-        return test_id if file_path.nil? || file_path == ""
-
-        "#{test_id}#{DELIMITER}#{file_path}"
+        JSON.dump({ test_id: test_id, file_path: file_path })
       end
 
       def self.load_error_payload?(file_path)

data/lib/ci/queue/redis/acknowledge.lua

@@ -4,18 +4,28 @@ local processed_key = KEYS[2]
 local owners_key = KEYS[3]
 local error_reports_key = KEYS[4]
 local requeued_by_key = KEYS[5]
+local leases_key = KEYS[6]
 
 local entry = ARGV[1]
-local test_id = ARGV[2]
-local error = ARGV[3]
-local ttl = ARGV[4]
-redis.call('zrem', zset_key, entry)
-redis.call('hdel', owners_key, entry)  -- Doesn't matter if it was reclaimed by another workers
+local error = ARGV[2]
+local ttl = ARGV[3]
+local lease_id = ARGV[4]
+
+-- Only the current lease holder can remove the entry from the running set.
+-- If the lease was transferred (e.g. via reserve_lost), the stale worker
+-- must not remove the running entry — that would let the supervisor think
+-- the queue is exhausted while the new lease holder is still processing.
+if tostring(redis.call('hget', leases_key, entry)) == lease_id then
+  redis.call('zrem', zset_key, entry)
+  redis.call('hdel', owners_key, entry)
+  redis.call('hdel', leases_key, entry)
+end
+
 redis.call('hdel', requeued_by_key, entry)
-local acknowledged = redis.call('sadd', processed_key, test_id) == 1
+local acknowledged = redis.call('sadd', processed_key, entry) == 1
 
 if acknowledged and error ~= "" then
-  redis.call('hset', error_reports_key, test_id, error)
+  redis.call('hset', error_reports_key, entry, error)
   redis.call('expire', error_reports_key, ttl)
 end
 

data/lib/ci/queue/redis/base.rb

@@ -60,10 +60,10 @@ module CI
           [0, 0, 0.1, 0.5, 1, 3, 5]
         end
 
-        def with_heartbeat(id)
+        def with_heartbeat(id, lease: nil)
           if heartbeat_enabled?
             ensure_heartbeat_thread_alive!
-            heartbeat_state.set(:tick, id)
+            heartbeat_state.set(:tick, id, lease)
           end
 
           yield
@@ -264,13 +264,11 @@ module CI
         end
 
         class HeartbeatProcess
-          def initialize(redis_url, zset_key, processed_key, owners_key, worker_queue_key, entry_delimiter:)
+          def initialize(redis_url, zset_key, owners_key, leases_key)
             @redis_url = redis_url
             @zset_key = zset_key
-            @processed_key = processed_key
             @owners_key = owners_key
-            @worker_queue_key = worker_queue_key
-            @entry_delimiter = entry_delimiter
+            @leases_key = leases_key
           end
 
           def boot!
@@ -282,10 +280,8 @@ module CI
               ::File.join(__dir__, "monitor.rb"),
               @redis_url,
               @zset_key,
-              @processed_key,
               @owners_key,
-              @worker_queue_key,
-              @entry_delimiter,
+              @leases_key,
               in: child_read,
               out: child_write,
             )
@@ -315,8 +311,8 @@ module CI
             end
           end
 
-          def tick!(id)
-            send_message(:tick!, id: id)
+          def tick!(id, lease)
+            send_message(:tick!, id: id, lease: lease.to_s)
           end
 
           private
@@ -357,10 +353,8 @@ module CI
           @heartbeat_process ||= HeartbeatProcess.new(
             @redis_url,
             key('running'),
-            key('processed'),
             key('owners'),
-            key('worker', worker_id, 'queue'),
-            entry_delimiter: CI::Queue::QueueEntry::DELIMITER,
+            key('leases'),
           )
         end
 
@@ -372,19 +366,16 @@ module CI
           Thread.current.name = "CI::Queue#heartbeat"
           Thread.current.abort_on_exception = true
 
-          timeout = config.timeout.to_i
           loop do
-            command = nil
             command = heartbeat_state.wait(1) # waits for max 1 second but wakes up immediately if we receive a command
 
             case command&.first
             when :tick
-              if timeout > 0
-                heartbeat_process.tick!(command.last)
-                timeout -= 1
-              end
+              # command = [:tick, entry_id, lease_id]
+              heartbeat_process.tick!(command[1], command[2])
             when :reset
-              timeout = config.timeout.to_i
+              # Test finished, stop ticking until next test starts
+              nil
             when :stop
               break
             end

data/lib/ci/queue/redis/build_record.rb

@@ -33,14 +33,14 @@ module CI
         end
 
         def failed_tests
-          redis.hkeys(key('error-reports'))
+          redis.hkeys(key('error-reports')).map { |entry| CI::Queue::QueueEntry.test_id(entry) }
         end
 
         TOTAL_KEY = "___total___"
         def requeued_tests
           requeues = redis.hgetall(key('requeues-count'))
           requeues.delete(TOTAL_KEY)
-          requeues
+          requeues.transform_keys { |entry| CI::Queue::QueueEntry.test_id(entry) }
         end
 
         def pop_warnings
@@ -56,39 +56,39 @@ module CI
           redis.rpush(key('warnings'), Marshal.dump([type, attributes]))
         end
 
-        def record_error(id, payload, stat_delta: nil)
+        def record_error(entry, payload, stat_delta: nil)
           # Run acknowledge first so we know whether we're the first to ack
-          acknowledged = @queue.acknowledge(id, error: payload)
+          acknowledged = @queue.acknowledge(entry, error: payload)
 
           if acknowledged
             # We were the first to ack; another worker already ack'd would get falsy from SADD
             @queue.increment_test_failed
             # Only the acknowledging worker's stats include this failure (others skip increment when ack=false).
             # Store so we can subtract it if another worker records success later.
-            store_error_report_delta(id, stat_delta) if stat_delta && stat_delta.any?
+            store_error_report_delta(entry, stat_delta) if stat_delta && stat_delta.any?
           end
           # Return so caller can roll back local counter when not acknowledged
           !!acknowledged
         end
 
-        def record_success(id, skip_flaky_record: false)
+        def record_success(entry, skip_flaky_record: false)
           acknowledged, error_reports_deleted_count, requeued_count, delta_json = redis.multi do |transaction|
-            @queue.acknowledge(id, pipeline: transaction)
-            transaction.hdel(key('error-reports'), id)
-            transaction.hget(key('requeues-count'), id)
-            transaction.hget(key('error-report-deltas'), id)
+            @queue.acknowledge(entry, pipeline: transaction)
+            transaction.hdel(key('error-reports'), entry)
+            transaction.hget(key('requeues-count'), entry)
+            transaction.hget(key('error-report-deltas'), entry)
           end
           # When we're replacing a failure, subtract the (single) acknowledging worker's stat contribution
           if error_reports_deleted_count.to_i > 0 && delta_json
             apply_error_report_delta_correction(delta_json)
-            redis.hdel(key('error-report-deltas'), id)
+            redis.hdel(key('error-report-deltas'), entry)
           end
-          record_flaky(id) if !skip_flaky_record && (error_reports_deleted_count.to_i > 0 || requeued_count.to_i > 0)
+          record_flaky(entry) if !skip_flaky_record && (error_reports_deleted_count.to_i > 0 || requeued_count.to_i > 0)
           # Count this run when we ack'd or when we replaced a failure (so stats delta is applied)
           !!(acknowledged || error_reports_deleted_count.to_i > 0)
         end
 
-        def record_requeue(id)
+        def record_requeue(entry)
           true
         end
 
@@ -142,11 +142,11 @@ module CI
         end
 
         def error_reports
-          redis.hgetall(key('error-reports'))
+          redis.hgetall(key('error-reports')).transform_keys { |entry| CI::Queue::QueueEntry.test_id(entry) }
         end
 
         def flaky_reports
-          redis.smembers(key('flaky-reports'))
+          redis.smembers(key('flaky-reports')).map { |entry| CI::Queue::QueueEntry.test_id(entry) }
         end
 
         def record_worker_profile(profile)
@@ -187,10 +187,10 @@ module CI
           ['build', config.build_id, *args].join(':')
         end
 
-        def store_error_report_delta(test_id, stat_delta)
+        def store_error_report_delta(entry, stat_delta)
           # Only the acknowledging worker's stats include this test; store their delta for correction on success
           payload = { 'worker_id' => config.worker_id.to_s }.merge(stat_delta)
-          redis.hset(key('error-report-deltas'), test_id, JSON.generate(payload))
+          redis.hset(key('error-report-deltas'), entry, JSON.generate(payload))
           redis.expire(key('error-report-deltas'), config.redis_ttl)
         end
 

data/lib/ci/queue/redis/heartbeat.lua

@@ -1,23 +1,17 @@
 -- AUTOGENERATED FILE DO NOT EDIT DIRECTLY
--- @include _entry_helpers
-
 local zset_key = KEYS[1]
-local processed_key = KEYS[2]
-local owners_key = KEYS[3]
-local worker_queue_key = KEYS[4]
+local leases_key = KEYS[2]
 
 local current_time = ARGV[1]
 local entry = ARGV[2]
-local entry_delimiter = ARGV[3]
-
-local test_id = test_id_from_entry(entry, entry_delimiter)
-
--- already processed, we do not need to bump the timestamp
-if redis.call('sismember', processed_key, test_id) == 1 then
-  return false
-end
+local lease_id = ARGV[3]
 
--- we're still the owner of the test, we can bump the timestamp
-if redis.call('hget', owners_key, entry) == worker_queue_key then
+-- Only the current lease holder can bump the timestamp.
+-- We intentionally do NOT check the processed set. A non-owner worker's
+-- acknowledge can add the entry to processed, which would poison the
+-- current lease holder's heartbeat if we checked it here.
+-- The lease check alone is sufficient — once the lease holder acknowledges,
+-- they zrem + hdel the lease, so the heartbeat will naturally stop.
+if tostring(redis.call('hget', leases_key, entry)) == lease_id then
   return redis.call('zadd', zset_key, current_time, entry)
 end

data/lib/ci/queue/redis/monitor.rb

@@ -13,12 +13,10 @@ module CI
         DEV_SCRIPTS_ROOT = ::File.expand_path('../../../../../../redis', __FILE__)
         RELEASE_SCRIPTS_ROOT = ::File.expand_path('../../redis', __FILE__)
 
-        def initialize(pipe, logger, redis_url, zset_key, processed_key, owners_key, worker_queue_key, entry_delimiter)
+        def initialize(pipe, logger, redis_url, zset_key, owners_key, leases_key)
           @zset_key = zset_key
-          @processed_key = processed_key
           @owners_key = owners_key
-          @worker_queue_key = worker_queue_key
-          @entry_delimiter = entry_delimiter
+          @leases_key = leases_key
           @logger = logger
           @redis = ::Redis.new(url: redis_url, reconnect_attempts: [0, 0, 0.1, 0.5, 1, 3, 5])
           @shutdown = false
@@ -37,11 +35,11 @@ module CI
           @self_pipe_writer << '.'
         end
 
-        def process_tick!(id:)
+        def process_tick!(id:, lease:)
           eval_script(
             :heartbeat,
-            keys: [@zset_key, @processed_key, @owners_key, @worker_queue_key],
-            argv: [Time.now.to_f, id, @entry_delimiter]
+            keys: [@zset_key, @leases_key],
+            argv: [Time.now.to_f, id, lease]
           )
         rescue => error
           @logger.info(error)
@@ -152,13 +150,11 @@ end
 
 redis_url = ARGV[0]
 zset_key = ARGV[1]
-processed_key = ARGV[2]
-owners_key = ARGV[3]
-worker_queue_key = ARGV[4]
-entry_delimiter = ARGV[5]
+owners_key = ARGV[2]
+leases_key = ARGV[3]
 
-logger.debug("Starting monitor: #{redis_url} #{zset_key} #{processed_key}")
-manager = CI::Queue::Redis::Monitor.new($stdin, logger, redis_url, zset_key, processed_key, owners_key, worker_queue_key, entry_delimiter)
+logger.debug("Starting monitor: #{redis_url} #{zset_key} #{leases_key}")
+manager = CI::Queue::Redis::Monitor.new($stdin, logger, redis_url, zset_key, owners_key, leases_key)
 
 # Notify the parent we're ready
 $stdout.puts(".")

data/lib/ci/queue/redis/release.lua

@@ -2,6 +2,7 @@
 local zset_key = KEYS[1]
 local worker_queue_key = KEYS[2]
 local owners_key = KEYS[3]
+local leases_key = KEYS[4]
 
 -- owned_tests = {"SomeTest", "worker:1", "SomeOtherTest", "worker:2", ...}
 local owned_tests = redis.call('hgetall', owners_key)
@@ -9,6 +10,7 @@ for index, owner_or_test in ipairs(owned_tests) do
   if owner_or_test == worker_queue_key then -- If we owned a test
     local test = owned_tests[index - 1]
     redis.call('zadd', zset_key, "0", test) -- We expire the lease immediately
+    redis.call('hdel', leases_key, test)
     return nil
   end
 end

data/lib/ci/queue/redis/requeue.lua

@@ -7,19 +7,23 @@ local worker_queue_key = KEYS[5]
 local owners_key = KEYS[6]
 local error_reports_key = KEYS[7]
 local requeued_by_key = KEYS[8]
+local leases_key = KEYS[9]
 
 local max_requeues = tonumber(ARGV[1])
 local global_max_requeues = tonumber(ARGV[2])
 local entry = ARGV[3]
-local test_id = ARGV[4]
-local offset = ARGV[5]
-local ttl = tonumber(ARGV[6])
-
-if redis.call('hget', owners_key, entry) == worker_queue_key then
-   redis.call('hdel', owners_key, entry)
+local offset = ARGV[4]
+local ttl = tonumber(ARGV[5])
+local lease_id = ARGV[6]
+
+-- Only the current lease holder can requeue a test.
+-- If the lease was transferred (e.g. via reserve_lost), reject the stale
+-- worker's requeue so the running entry stays intact for the new holder.
+if tostring(redis.call('hget', leases_key, entry)) ~= lease_id then
+  return false
 end
 
-if redis.call('sismember', processed_key, test_id) == 1 then
+if redis.call('sismember', processed_key, entry) == 1 then
   return false
 end
 
@@ -28,15 +32,15 @@ if global_requeues and global_requeues >= tonumber(global_max_requeues) then
   return false
 end
 
-local requeues = tonumber(redis.call('hget', requeues_count_key, test_id))
+local requeues = tonumber(redis.call('hget', requeues_count_key, entry))
 if requeues and requeues >= max_requeues then
   return false
 end
 
 redis.call('hincrby', requeues_count_key, '___total___', 1)
-redis.call('hincrby', requeues_count_key, test_id, 1)
+redis.call('hincrby', requeues_count_key, entry, 1)
 
-redis.call('hdel', error_reports_key, test_id)
+redis.call('hdel', error_reports_key, entry)
 
 local pivot = redis.call('lrange', queue_key, -1 - offset, 0 - offset)[1]
 if pivot then
@@ -50,6 +54,8 @@ if ttl and ttl > 0 then
   redis.call('expire', requeued_by_key, ttl)
 end
 
+redis.call('hdel', owners_key, entry)
+redis.call('hdel', leases_key, entry)
 redis.call('zrem', zset_key, entry)
 
 return true

data/lib/ci/queue/redis/reserve.lua

@@ -6,6 +6,8 @@ local worker_queue_key = KEYS[4]
 local owners_key = KEYS[5]
 local requeued_by_key = KEYS[6]
 local workers_key = KEYS[7]
+local leases_key = KEYS[8]
+local lease_counter_key = KEYS[9]
 
 local current_time = ARGV[1]
 local defer_offset = tonumber(ARGV[2]) or 0
@@ -20,6 +22,15 @@ local function insert_with_offset(test)
   end
 end
 
+local function claim_test(test)
+  local lease = redis.call('incr', lease_counter_key)
+  redis.call('zadd', zset_key, current_time, test)
+  redis.call('lpush', worker_queue_key, test)
+  redis.call('hset', owners_key, test, worker_queue_key)
+  redis.call('hset', leases_key, test, lease)
+  return {test, tostring(lease)}
+end
+
 for attempt = 1, max_skip_attempts do
   local test = redis.call('rpop', queue_key)
   if not test then
@@ -31,10 +42,7 @@ for attempt = 1, max_skip_attempts do
     -- If this build only has one worker, allow immediate self-pickup.
     if redis.call('scard', workers_key) <= 1 then
       redis.call('hdel', requeued_by_key, test)
-      redis.call('zadd', zset_key, current_time, test)
-      redis.call('lpush', worker_queue_key, test)
-      redis.call('hset', owners_key, test, worker_queue_key)
-      return test
+      return claim_test(test)
     end
 
     insert_with_offset(test)
@@ -47,10 +55,7 @@ for attempt = 1, max_skip_attempts do
     end
   else
     redis.call('hdel', requeued_by_key, test)
-    redis.call('zadd', zset_key, current_time, test)
-    redis.call('lpush', worker_queue_key, test)
-    redis.call('hset', owners_key, test, worker_queue_key)
-    return test
+    return claim_test(test)
   end
 end
 

data/lib/ci/queue/redis/reserve_lost.lua

@@ -1,23 +1,30 @@
 -- AUTOGENERATED FILE DO NOT EDIT DIRECTLY
--- @include _entry_helpers
-
 local zset_key = KEYS[1]
 local processed_key = KEYS[2]
 local worker_queue_key = KEYS[3]
 local owners_key = KEYS[4]
+local leases_key = KEYS[5]
+local lease_counter_key = KEYS[6]
 
 local current_time = ARGV[1]
 local timeout = ARGV[2]
-local entry_delimiter = ARGV[3]
 
 local lost_tests = redis.call('zrangebyscore', zset_key, 0, current_time - timeout)
 for _, test in ipairs(lost_tests) do
-  local test_id = test_id_from_entry(test, entry_delimiter)
-  if redis.call('sismember', processed_key, test_id) == 0 then
+  if redis.call('sismember', processed_key, test) == 0 then
+    local lease = redis.call('incr', lease_counter_key)
     redis.call('zadd', zset_key, current_time, test)
     redis.call('lpush', worker_queue_key, test)
-    redis.call('hset', owners_key, test, worker_queue_key) -- Take ownership
-    return test
+    redis.call('hset', owners_key, test, worker_queue_key)
+    redis.call('hset', leases_key, test, lease)
+    return {test, tostring(lease)}
+  else
+    -- Test is already processed but still in running (stale). This can happen when
+    -- a non-owner worker acknowledged the test (marking it processed) but could not
+    -- remove it from running due to the lease guard. Clean it up.
+    redis.call('zrem', zset_key, test)
+    redis.call('hdel', owners_key, test)
+    redis.call('hdel', leases_key, test)
   end
 end
 

data/lib/ci/queue/redis/retry.rb

@@ -12,6 +12,22 @@ module CI
           @build ||= CI::Queue::Redis::BuildRecord.new(self, redis, config)
         end
 
+        # Retry queue is pre-populated with failed test entries from the previous run.
+        # Don't replace them with the full preresolved/lazy test list.
+        # QueuePopulationStrategy#configure_lazy_queue will still set entry_resolver,
+        # so poll uses LazyEntryResolver to lazily load test files on demand.
+        # The random/batch_size params are intentionally ignored since we keep
+        # the existing queue contents as-is.
+        #
+        # Note: populate (non-stream) is intentionally NOT overridden here.
+        # RSpec and non-lazy Minitest retries call populate to build the
+        # @index mapping test IDs to runnable objects, which poll needs to
+        # yield proper test/example instances. In those paths, @queue contains
+        # bare test IDs that match @index keys, so populate works correctly.
+        def stream_populate(tests, random: nil, batch_size: nil)
+          self
+        end
+
         private
 
         attr_reader :redis

data/lib/ci/queue/redis/worker.rb

@@ -19,6 +19,7 @@ module CI
 
         def initialize(redis, config)
           @reserved_tests = Concurrent::Set.new
+          @reserved_leases = Concurrent::Map.new
           @shutdown_required = false
           @first_reserve_at = nil
           super(redis, config)
@@ -147,9 +148,10 @@ module CI
         def retry_queue
           failures = build.failed_tests.to_set
           log = redis.lrange(key('worker', worker_id, 'queue'), 0, -1)
-          log = log.map { |entry| queue_entry_test_id(entry) }
-          log.select! { |id| failures.include?(id) }
-          log.uniq!
+          # Keep full entries (test_id + file_path) so lazy loading can resolve them.
+          # Filter by test_id against failures without stripping file paths.
+          log.select! { |entry| failures.include?(CI::Queue::QueueEntry.test_id(entry)) }
+          log.uniq! { |entry| CI::Queue::QueueEntry.test_id(entry) }
           log.reverse!
           Retry.new(log, config, redis: redis)
         end
@@ -172,27 +174,34 @@ module CI
           nil
         end
 
+        def lease_for(entry)
+          test_id = CI::Queue::QueueEntry.test_id(entry)
+          @reserved_leases[test_id]
+        end
+
         def report_worker_error(error)
           build.report_worker_error(error)
         end
 
-        def acknowledge(test_key, error: nil, pipeline: redis)
-          test_id = normalize_test_id(test_key)
+        def acknowledge(entry, error: nil, pipeline: redis)
+          test_id = CI::Queue::QueueEntry.test_id(entry)
           assert_reserved!(test_id)
-          entry = reserved_entries.fetch(test_id, queue_entry_for(test_key))
+          entry = reserved_entries.fetch(test_id, entry)
+          lease = @reserved_leases.delete(test_id)
           unreserve_entry(test_id)
           eval_script(
             :acknowledge,
-            keys: [key('running'), key('processed'), key('owners'), key('error-reports'), key('requeued-by')],
-            argv: [entry, test_id, error.to_s, config.redis_ttl],
+            keys: [key('running'), key('processed'), key('owners'), key('error-reports'), key('requeued-by'), key('leases')],
+            argv: [entry, error.to_s, config.redis_ttl, lease.to_s],
             pipeline: pipeline,
           ) == 1
         end
 
-        def requeue(test, offset: Redis.requeue_offset)
-          test_id = normalize_test_id(test)
+        def requeue(entry, offset: Redis.requeue_offset)
+          test_id = CI::Queue::QueueEntry.test_id(entry)
           assert_reserved!(test_id)
-          entry = reserved_entries.fetch(test_id, queue_entry_for(test))
+          entry = reserved_entries.fetch(test_id, entry)
+          lease = @reserved_leases.delete(test_id)
           unreserve_entry(test_id)
           global_max_requeues = config.global_max_requeues(total)
 
@@ -207,14 +216,16 @@ module CI
               key('owners'),
               key('error-reports'),
               key('requeued-by'),
+              key('leases'),
             ],
-            argv: [config.max_requeues, global_max_requeues, entry, test_id, offset, config.redis_ttl],
+            argv: [config.max_requeues, global_max_requeues, entry, offset, config.redis_ttl, lease.to_s],
           ) == 1
 
           unless requeued
             reserved_tests << test_id
             reserved_entries[test_id] = entry
             reserved_entry_ids[entry] = test_id
+            @reserved_leases[test_id] = lease if lease
           end
           requeued
         end
@@ -222,7 +233,7 @@ module CI
         def release!
           eval_script(
             :release,
-            keys: [key('running'), key('worker', worker_id, 'queue'), key('owners')],
+            keys: [key('running'), key('worker', worker_id, 'queue'), key('owners'), key('leases')],
             argv: [],
           )
           nil
@@ -254,11 +265,12 @@ module CI
           end
         end
 
-        def reserve_entry(entry)
-          test_id = queue_entry_test_id(entry)
+        def reserve_entry(entry, lease = nil)
+          test_id = CI::Queue::QueueEntry.test_id(entry)
           reserved_tests << test_id
           reserved_entries[test_id] = entry
           reserved_entry_ids[entry] = test_id
+          @reserved_leases[test_id] = lease if lease
         end
 
         def unreserve_entry(test_id)
@@ -267,19 +279,6 @@ module CI
           reserved_entry_ids.delete(entry) if entry
         end
 
-        def normalize_test_id(test_key)
-          key = test_key.respond_to?(:id) ? test_key.id : test_key
-          if key.is_a?(String)
-            cached = reserved_entry_ids[key]
-            return cached if cached
-          end
-          queue_entry_test_id(key)
-        end
-
-        def queue_entry_test_id(entry)
-          CI::Queue::QueueEntry.test_id(entry)
-        end
-
         def queue_entry_for(test)
           return test.queue_entry if test.respond_to?(:queue_entry)
           return test.id if test.respond_to?(:id)
@@ -288,7 +287,7 @@ module CI
         end
 
         def resolve_entry(entry)
-          test_id = reserved_entry_ids[entry] || queue_entry_test_id(entry)
+          test_id = reserved_entry_ids[entry] || CI::Queue::QueueEntry.test_id(entry)
           if populated?
             return index[test_id] if index.key?(test_id)
           end
@@ -356,12 +355,12 @@ module CI
         end
 
         def reserve
-          (try_to_reserve_lost_test || try_to_reserve_test).tap do |entry|
-            if entry
-              @first_reserve_at ||= Process.clock_gettime(Process::CLOCK_MONOTONIC)
-              reserve_entry(entry)
-            end
+          entry, lease = try_to_reserve_lost_test || try_to_reserve_test || [nil, nil]
+          if entry
+            @first_reserve_at ||= Process.clock_gettime(Process::CLOCK_MONOTONIC)
+            reserve_entry(entry, lease)
           end
+          entry
         end
 
         def try_to_reserve_test
@@ -375,6 +374,8 @@ module CI
               key('owners'),
               key('requeued-by'),
               key('workers'),
+              key('leases'),
+              key('lease-counter'),
             ],
             argv: [CI::Queue.time_now.to_f, Redis.requeue_offset],
           )
@@ -383,22 +384,28 @@ module CI
         def try_to_reserve_lost_test
           timeout = config.max_missed_heartbeat_seconds ? config.max_missed_heartbeat_seconds : config.timeout
 
-          lost_test = eval_script(
+          result = eval_script(
             :reserve_lost,
             keys: [
               key('running'),
-              key('completed'),
+              key('processed'),
               key('worker', worker_id, 'queue'),
               key('owners'),
+              key('leases'),
+              key('lease-counter'),
             ],
-            argv: [CI::Queue.time_now.to_f, timeout, CI::Queue::QueueEntry::DELIMITER],
+            argv: [CI::Queue.time_now.to_f, timeout],
           )
 
-          if lost_test
-            build.record_warning(Warnings::RESERVED_LOST_TEST, test: lost_test, timeout: config.timeout)
+          if result
+            entry = result.is_a?(Array) ? result[0] : result
+            build.record_warning(Warnings::RESERVED_LOST_TEST, test: CI::Queue::QueueEntry.test_id(entry), timeout: config.timeout)
+            if CI::Queue.debug?
+              $stderr.puts "[ci-queue][reserve_lost] worker=#{worker_id} test_id=#{CI::Queue::QueueEntry.test_id(entry)}"
+            end
           end
 
-          lost_test
+          result
         end
 
         def push(entries)

data/lib/ci/queue/static.rb

@@ -16,6 +16,7 @@ module CI
       TEN_MINUTES = 60 * 10
 
       attr_reader :progress, :total
+      attr_accessor :entry_resolver
 
       def initialize(tests, config)
         @queue = tests
@@ -50,10 +51,24 @@ module CI
         self
       end
 
-      def with_heartbeat(id)
+      # Support lazy loading mode: accept an enumerator of entries and
+      # store them in queue order (no shuffling). This preserves the
+      # exact order from the input file for local reproduction.
+      def stream_populate(tests, random: nil, batch_size: nil)
+        @queue = []
+        tests.each { |entry| @queue << entry }
+        @total = @queue.size
+        self
+      end
+
+      def with_heartbeat(id, lease: nil)
         yield
       end
 
+      def lease_for(entry)
+        nil
+      end
+
       def ensure_heartbeat_thread_alive!; end
 
       def boot_heartbeat_process!; end
@@ -75,11 +90,15 @@ module CI
       end
 
       def populated?
-        !!defined?(@index)
+        !!defined?(@index) || @queue.any?
       end
 
       def to_a
-        @queue.map { |i| index.fetch(i) }
+        if defined?(@index) && @index
+          @queue.map { |i| index.fetch(i) }
+        else
+          @queue.dup
+        end
       end
 
       def size
@@ -97,9 +116,28 @@ module CI
       def poll
         while !@shutdown && config.circuit_breakers.none?(&:open?) && !max_test_failed? && reserved_test = @queue.shift
           reserved_tests << reserved_test
-          yield index.fetch(reserved_test)
+          if entry_resolver
+            resolved = entry_resolver.call(reserved_test)
+            # Track the original queue entry so requeue can push it back
+            # with its full payload (file path, load-error data, etc.).
+            reserved_entries[resolved.id] = reserved_test if resolved.respond_to?(:id)
+            yield resolved
+          elsif defined?(@index) && @index
+            # Queue entries may be JSON-formatted (with test_id + file_path) while
+            # the index is keyed by bare test_id from populate. Try the raw entry
+            # first, then fall back to extracting the test_id.
+            test_id = begin
+              CI::Queue::QueueEntry.test_id(reserved_test)
+            rescue JSON::ParserError
+              reserved_test
+            end
+            yield index.fetch(test_id)
+          else
+            yield reserved_test
+          end
         end
         reserved_tests.clear
+        reserved_entries.clear
       end
 
       def exhausted?
@@ -125,12 +163,15 @@ module CI
         test_failed >= config.max_test_failed
       end
 
-      def requeue(test)
-        test_key = test.id
-        return false unless should_requeue?(test_key)
+      def requeue(entry)
+        test_id = CI::Queue::QueueEntry.test_id(entry)
+        return false unless should_requeue?(test_id)
 
-        requeues[test_key] += 1
-        @queue.unshift(test_key)
+        requeues[test_id] += 1
+        # Push back the original queue entry (with file path / load-error payload)
+        # so entry_resolver can fully resolve it on the next poll iteration.
+        original_entry = reserved_entries.delete(test_id) || test_id
+        @queue.unshift(original_entry)
         true
       end
 
@@ -146,6 +187,10 @@ module CI
         @requeues ||= Hash.new(0)
       end
 
+      def reserved_entries
+        @reserved_entries ||= {}
+      end
+
       def reserved_tests
         @reserved_tests ||= Concurrent::Set.new
       end

data/lib/ci/queue/version.rb

@@ -2,7 +2,7 @@
 
 module CI
   module Queue
-    VERSION = '0.83.0'
+    VERSION = '0.85.0'
     DEV_SCRIPTS_ROOT = ::File.expand_path('../../../../../redis', __FILE__)
     RELEASE_SCRIPTS_ROOT = ::File.expand_path('../redis', __FILE__)
   end

data/lib/minitest/queue/build_status_recorder.rb

@@ -40,15 +40,15 @@ module Minitest
         self.total_time = Minitest.clock_time - start_time
         
         # Determine what type of result this is and record it
-        test_id = "#{test.klass}##{test.name}"
+        entry = test.queue_entry
         delta = delta_for(test)
 
         acknowledged = if (test.failure || test.error?) && !test.skipped?
-          build.record_error(test_id, dump(test), stat_delta: delta)
+          build.record_error(entry, dump(test), stat_delta: delta)
         elsif test.requeued?
-          build.record_requeue(test_id)
+          build.record_requeue(entry)
         else
-          build.record_success(test_id, skip_flaky_record: test.skipped?)
+          build.record_success(entry, skip_flaky_record: test.skipped?)
         end
 
         if acknowledged

data/lib/minitest/queue/test_data.rb

@@ -138,7 +138,7 @@ module Minitest
         @error_location ||= begin
           last_before_assertion = ''
           backtrace_for(exception).reverse_each do |s|
-            break if s =~ /in .(assert|refute|flunk|pass|fail|raise|must|wont)/
+            break if s =~ /in [`'](?:[\w:]*[#.])?(assert|refute|flunk|pass|fail|raise|must|wont)/
 
             last_before_assertion = s
           end

data/lib/minitest/queue.rb

@@ -163,7 +163,7 @@ module Minitest
         rescue_run_errors do
           begin
             queue.poll do |example|
-              result = queue.with_heartbeat(example.queue_entry) do
+              result = queue.with_heartbeat(example.queue_entry, lease: queue.lease_for(example.queue_entry)) do
                 example.run
               end
 
@@ -195,15 +195,18 @@ module Minitest
           # When we do a bisect, we don't care about the result other than the test we're running the bisect on
           result.mark_as_flaked!
           failed = false
+        end
+
+        if failed && CI::Queue.requeueable?(result) && queue.requeue(example.queue_entry)
+          result.requeue!
+          if CI::Queue.debug?
+            $stderr.puts "[ci-queue][requeue] test_id=#{example.id} error_class=#{result.failures.first&.class} error=#{result.failures.first&.message&.lines&.first&.chomp}"
+          end
         elsif failed
           queue.report_failure!
         else
           queue.report_success!
         end
-
-        if failed && CI::Queue.requeueable?(result) && queue.requeue(example)
-          result.requeue!
-        end
         reporter.record(result)
       end
 
@@ -327,7 +330,7 @@ module Minitest
       end
 
       def queue_entry
-        id
+        @queue_entry ||= CI::Queue::QueueEntry.format(id, nil)
       end
 
       def <=>(other)

data/lib/rspec/queue/build_status_recorder.rb

@@ -18,12 +18,14 @@ module RSpec
 
       def example_passed(notification)
         example = notification.example
-        build.record_success(example.id)
+        entry = CI::Queue::QueueEntry.format(example.id, nil)
+        build.record_success(entry)
       end
 
       def example_failed(notification)
         example = notification.example
-        build.record_error(example.id, dump(notification))
+        entry = CI::Queue::QueueEntry.format(example.id, nil)
+        build.record_error(entry, dump(notification))
       end
 
       private

data/lib/rspec/queue.rb

@@ -253,6 +253,10 @@ module RSpec
         example.id
       end
 
+      def queue_entry
+        @queue_entry ||= CI::Queue::QueueEntry.format(id, nil)
+      end
+
       def <=>(other)
         id <=> other.id
       end
@@ -411,7 +415,7 @@ module RSpec
       end
 
       def requeue
-        @queue.requeue(@example)
+        @queue.requeue(@example.queue_entry)
       end
 
       def cancel_run!
@@ -422,7 +426,7 @@ module RSpec
       end
 
       def acknowledge
-        @queue.acknowledge(@example)
+        @queue.acknowledge(@example.queue_entry)
       end
     end
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: ci-queue
 version: !ruby/object:Gem::Version
-  version: 0.83.0
+  version: 0.85.0
 platform: ruby
 authors:
 - Jean Boussier
@@ -149,6 +149,34 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: benchmark
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rexml
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: rubocop
   requirement: !ruby/object:Gem::Requirement
@@ -199,7 +227,6 @@ files:
 - lib/ci/queue/output_helpers.rb
 - lib/ci/queue/queue_entry.rb
 - lib/ci/queue/redis.rb
-- lib/ci/queue/redis/_entry_helpers.lua
 - lib/ci/queue/redis/acknowledge.lua
 - lib/ci/queue/redis/base.rb
 - lib/ci/queue/redis/build_record.rb
@@ -257,14 +284,14 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: '2.7'
+      version: '3.1'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 4.0.6
+rubygems_version: 4.0.8
 specification_version: 4
 summary: Distribute tests over many workers using a queue
 test_files: []

data/lib/ci/queue/redis/_entry_helpers.lua

@@ -1,10 +0,0 @@
--- AUTOGENERATED FILE DO NOT EDIT DIRECTLY
-local function test_id_from_entry(value, delimiter)
-  if delimiter then
-    local pos = string.find(value, delimiter, 1, true)
-    if pos then
-      return string.sub(value, 1, pos - 1)
-    end
-  end
-  return value
-end