ci-queue 0.83.0 → 0.84.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: '08152aacde705d2472151b351e690db68a0c067c8930bf9574e1835825ee225b'
-  data.tar.gz: 37fc5a7e9174d9188ff39568a0e4a5bf4f1ae0e3bce0f08e4d9d66e2a5a5264b
+  metadata.gz: c47a6b5450a21d7f4fb79a2b9a862ee53d6602d49e61a1e60d0fdaba92e9d0fd
+  data.tar.gz: 217ae043f06406663beff99e415dcf778a1a6cacd47ba6148d9e50504927dcf3
 SHA512:
-  metadata.gz: 0ca31505b3ca58115b632d1a23e7884e2bbc25ae2360f4228b08192705200303ddecd03220ff86ebb6c6c82b7e580f83f7d86e5f79d10d59cd6949c4b6947d42
-  data.tar.gz: f47440362656cc4ddf8ad52a9c49c9289b70f0a64312eb2c0c3f82e6bc9a980e6414a2c03ccb6613508975bf7b64e6b6232d1083fbbfc4c159b6eac0cb522e51
+  metadata.gz: ca7a1134775424386068df3e1b3c80738f90bbf5a353ed254715cd12194d6dfa39e1313c0fe4ea3e851f84a459ed0c61bbc52045164764fd4941b833ee6d71eb
+  data.tar.gz: 0ddca915e68afcfe1f6a99b41d34d9b46aa731499922ae1556de7db026b5e30dd48de08644df603c99426c430bebb782530825e626cc631f45dc94f0df2db200

data/.ruby-version

@@ -1 +1 @@
-3.3.0
+4.0

data/Gemfile.lock

@@ -1,95 +1,105 @@
 PATH
   remote: .
   specs:
-    ci-queue (0.83.0)
+    ci-queue (0.84.0)
       logger
 
 GEM
   remote: https://rubygems.org/
   specs:
-    activesupport (7.1.3.2)
+    activesupport (8.1.3)
       base64
       bigdecimal
-      concurrent-ruby (~> 1.0, >= 1.0.2)
+      concurrent-ruby (~> 1.0, >= 1.3.1)
       connection_pool (>= 2.2.5)
       drb
       i18n (>= 1.6, < 2)
+      json
+      logger (>= 1.4.2)
       minitest (>= 5.1)
-      mutex_m
-      tzinfo (~> 2.0)
+      securerandom (>= 0.3)
+      tzinfo (~> 2.0, >= 2.0.5)
+      uri (>= 0.13.1)
     ansi (1.5.0)
-    ast (2.4.2)
-    base64 (0.2.0)
-    bigdecimal (3.1.7)
-    builder (3.2.4)
-    concurrent-ruby (1.2.3)
-    connection_pool (2.4.1)
-    diff-lcs (1.5.1)
-    docile (1.4.0)
-    drb (2.2.1)
-    i18n (1.14.4)
+    ast (2.4.3)
+    base64 (0.3.0)
+    benchmark (0.5.0)
+    bigdecimal (4.0.1)
+    builder (3.3.0)
+    concurrent-ruby (1.3.6)
+    connection_pool (3.0.2)
+    diff-lcs (1.6.2)
+    docile (1.4.1)
+    drb (2.2.3)
+    i18n (1.14.8)
       concurrent-ruby (~> 1.0)
-    json (2.7.1)
-    language_server-protocol (3.17.0.3)
-    logger (1.6.1)
-    minitest (5.22.3)
-    minitest-reporters (1.6.1)
+    json (2.19.3)
+    language_server-protocol (3.17.0.5)
+    lint_roller (1.1.0)
+    logger (1.7.0)
+    minitest (5.27.0)
+    minitest-reporters (1.7.1)
       ansi
       builder
       minitest (>= 5.0)
       ruby-progressbar
-    msgpack (1.7.2)
-    mutex_m (0.2.0)
-    parallel (1.24.0)
-    parser (3.3.0.5)
+    msgpack (1.8.0)
+    parallel (1.27.0)
+    parser (3.3.10.2)
       ast (~> 2.4.1)
       racc
-    racc (1.7.3)
+    prism (1.9.0)
+    racc (1.8.1)
     rainbow (3.1.1)
-    rake (13.1.0)
-    redis (5.1.0)
-      redis-client (>= 0.17.0)
-    redis-client (0.21.1)
+    rake (13.3.1)
+    redis (5.4.1)
+      redis-client (>= 0.22.0)
+    redis-client (0.28.0)
       connection_pool
-    regexp_parser (2.9.0)
-    rexml (3.3.8)
-    rspec (3.13.0)
+    regexp_parser (2.11.3)
+    rexml (3.4.4)
+    rspec (3.13.2)
       rspec-core (~> 3.13.0)
       rspec-expectations (~> 3.13.0)
       rspec-mocks (~> 3.13.0)
-    rspec-core (3.13.0)
+    rspec-core (3.13.6)
       rspec-support (~> 3.13.0)
-    rspec-expectations (3.13.0)
+    rspec-expectations (3.13.5)
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.13.0)
-    rspec-mocks (3.13.0)
+    rspec-mocks (3.13.8)
       diff-lcs (>= 1.2.0, < 2.0)
       rspec-support (~> 3.13.0)
-    rspec-support (3.13.1)
-    rubocop (1.62.1)
+    rspec-support (3.13.7)
+    rubocop (1.86.0)
       json (~> 2.3)
-      language_server-protocol (>= 3.17.0)
+      language_server-protocol (~> 3.17.0.2)
+      lint_roller (~> 1.1.0)
       parallel (~> 1.10)
       parser (>= 3.3.0.2)
       rainbow (>= 2.2.2, < 4.0)
-      regexp_parser (>= 1.8, < 3.0)
-      rexml (>= 3.2.5, < 4.0)
-      rubocop-ast (>= 1.31.1, < 2.0)
+      regexp_parser (>= 2.9.3, < 3.0)
+      rubocop-ast (>= 1.49.0, < 2.0)
       ruby-progressbar (~> 1.7)
-      unicode-display_width (>= 2.4.0, < 3.0)
-    rubocop-ast (1.31.2)
-      parser (>= 3.3.0.4)
+      unicode-display_width (>= 2.4.0, < 4.0)
+    rubocop-ast (1.49.1)
+      parser (>= 3.3.7.2)
+      prism (~> 1.7)
     ruby-progressbar (1.13.0)
+    securerandom (0.4.1)
     simplecov (0.22.0)
       docile (~> 1.1)
       simplecov-html (~> 0.11)
       simplecov_json_formatter (~> 0.1)
-    simplecov-html (0.12.3)
+    simplecov-html (0.13.2)
     simplecov_json_formatter (0.1.4)
-    snappy (0.4.0)
+    snappy (0.5.1)
     tzinfo (2.0.6)
       concurrent-ruby (~> 1.0)
-    unicode-display_width (2.5.0)
+    unicode-display_width (3.2.0)
+      unicode-emoji (~> 4.1)
+    unicode-emoji (4.2.0)
+    uri (1.1.1)
 
 PLATFORMS
   arm64-darwin-23
@@ -97,6 +107,7 @@ PLATFORMS
 
 DEPENDENCIES
   activesupport
+  benchmark
   bundler
   ci-queue!
   minitest (~> 5.11)
@@ -104,6 +115,7 @@ DEPENDENCIES
   msgpack
   rake
   redis
+  rexml
   rspec (~> 3.10)
   rubocop
   simplecov (~> 0.12)

data/ci-queue.gemspec

@@ -19,7 +19,7 @@ Gem::Specification.new do |spec|
   spec.homepage      = 'https://github.com/Shopify/ci-queue'
   spec.license       = 'MIT'
 
-  spec.required_ruby_version = '>= 2.7'
+  spec.required_ruby_version = '>= 3.1'
 
   spec.files         = lua_scripts + `git ls-files -z`.split("\x0").reject do |f|
     f.match(%r{^(test|spec|features)/})
@@ -43,5 +43,7 @@ Gem::Specification.new do |spec|
 
   spec.add_development_dependency 'snappy'
   spec.add_development_dependency 'msgpack'
+  spec.add_development_dependency 'benchmark'
+  spec.add_development_dependency 'rexml'
   spec.add_development_dependency 'rubocop'
 end

data/lib/ci/queue/build_record.rb

@@ -18,17 +18,17 @@ module CI
         @queue.exhausted?
       end
 
-      def record_error(id, payload, stat_delta: nil)
-        error_reports[id] = payload
+      def record_error(entry, payload, stat_delta: nil)
+        error_reports[entry] = payload
         true
       end
 
-      def record_success(id, skip_flaky_record: false, acknowledge: true)
-        error_reports.delete(id)
+      def record_success(entry, skip_flaky_record: false, acknowledge: true)
+        error_reports.delete(entry)
         true
       end
 
-      def record_requeue(id)
+      def record_requeue(entry)
         true
       end
 

data/lib/ci/queue/queue_entry.rb

@@ -6,26 +6,18 @@ require 'json'
 module CI
   module Queue
     module QueueEntry
-      DELIMITER = "\t"
       LOAD_ERROR_PREFIX = '__ciq_load_error__:'.freeze
 
       def self.test_id(entry)
-        pos = entry.index(DELIMITER)
-        pos ? entry[0, pos] : entry
+        JSON.parse(entry, symbolize_names: true)[:test_id]
       end
 
       def self.parse(entry)
-        return { test_id: entry, file_path: nil } unless entry.include?(DELIMITER)
-
-        test_id, file_path = entry.split(DELIMITER, 2)
-        file_path = nil if file_path == ""
-        { test_id: test_id, file_path: file_path }
+        JSON.parse(entry, symbolize_names: true)
       end
 
       def self.format(test_id, file_path)
-        return test_id if file_path.nil? || file_path == ""
-
-        "#{test_id}#{DELIMITER}#{file_path}"
+        JSON.dump({ test_id: test_id, file_path: file_path })
       end
 
       def self.load_error_payload?(file_path)

data/lib/ci/queue/redis/acknowledge.lua

@@ -6,16 +6,15 @@ local error_reports_key = KEYS[4]
 local requeued_by_key = KEYS[5]
 
 local entry = ARGV[1]
-local test_id = ARGV[2]
-local error = ARGV[3]
-local ttl = ARGV[4]
+local error = ARGV[2]
+local ttl = ARGV[3]
 redis.call('zrem', zset_key, entry)
 redis.call('hdel', owners_key, entry)  -- Doesn't matter if it was reclaimed by another workers
 redis.call('hdel', requeued_by_key, entry)
-local acknowledged = redis.call('sadd', processed_key, test_id) == 1
+local acknowledged = redis.call('sadd', processed_key, entry) == 1
 
 if acknowledged and error ~= "" then
-  redis.call('hset', error_reports_key, test_id, error)
+  redis.call('hset', error_reports_key, entry, error)
   redis.call('expire', error_reports_key, ttl)
 end
 

data/lib/ci/queue/redis/base.rb

@@ -264,13 +264,12 @@ module CI
         end
 
         class HeartbeatProcess
-          def initialize(redis_url, zset_key, processed_key, owners_key, worker_queue_key, entry_delimiter:)
+          def initialize(redis_url, zset_key, processed_key, owners_key, worker_queue_key)
             @redis_url = redis_url
             @zset_key = zset_key
             @processed_key = processed_key
             @owners_key = owners_key
             @worker_queue_key = worker_queue_key
-            @entry_delimiter = entry_delimiter
           end
 
           def boot!
@@ -285,7 +284,6 @@ module CI
               @processed_key,
               @owners_key,
               @worker_queue_key,
-              @entry_delimiter,
               in: child_read,
               out: child_write,
             )
@@ -360,7 +358,6 @@ module CI
             key('processed'),
             key('owners'),
             key('worker', worker_id, 'queue'),
-            entry_delimiter: CI::Queue::QueueEntry::DELIMITER,
           )
         end
 

data/lib/ci/queue/redis/build_record.rb

@@ -33,14 +33,14 @@ module CI
         end
 
         def failed_tests
-          redis.hkeys(key('error-reports'))
+          redis.hkeys(key('error-reports')).map { |entry| CI::Queue::QueueEntry.test_id(entry) }
         end
 
         TOTAL_KEY = "___total___"
         def requeued_tests
           requeues = redis.hgetall(key('requeues-count'))
           requeues.delete(TOTAL_KEY)
-          requeues
+          requeues.transform_keys { |entry| CI::Queue::QueueEntry.test_id(entry) }
         end
 
         def pop_warnings
@@ -56,39 +56,39 @@ module CI
           redis.rpush(key('warnings'), Marshal.dump([type, attributes]))
         end
 
-        def record_error(id, payload, stat_delta: nil)
+        def record_error(entry, payload, stat_delta: nil)
           # Run acknowledge first so we know whether we're the first to ack
-          acknowledged = @queue.acknowledge(id, error: payload)
+          acknowledged = @queue.acknowledge(entry, error: payload)
 
           if acknowledged
             # We were the first to ack; another worker already ack'd would get falsy from SADD
             @queue.increment_test_failed
             # Only the acknowledging worker's stats include this failure (others skip increment when ack=false).
             # Store so we can subtract it if another worker records success later.
-            store_error_report_delta(id, stat_delta) if stat_delta && stat_delta.any?
+            store_error_report_delta(entry, stat_delta) if stat_delta && stat_delta.any?
           end
           # Return so caller can roll back local counter when not acknowledged
           !!acknowledged
         end
 
-        def record_success(id, skip_flaky_record: false)
+        def record_success(entry, skip_flaky_record: false)
           acknowledged, error_reports_deleted_count, requeued_count, delta_json = redis.multi do |transaction|
-            @queue.acknowledge(id, pipeline: transaction)
-            transaction.hdel(key('error-reports'), id)
-            transaction.hget(key('requeues-count'), id)
-            transaction.hget(key('error-report-deltas'), id)
+            @queue.acknowledge(entry, pipeline: transaction)
+            transaction.hdel(key('error-reports'), entry)
+            transaction.hget(key('requeues-count'), entry)
+            transaction.hget(key('error-report-deltas'), entry)
           end
           # When we're replacing a failure, subtract the (single) acknowledging worker's stat contribution
           if error_reports_deleted_count.to_i > 0 && delta_json
             apply_error_report_delta_correction(delta_json)
-            redis.hdel(key('error-report-deltas'), id)
+            redis.hdel(key('error-report-deltas'), entry)
           end
-          record_flaky(id) if !skip_flaky_record && (error_reports_deleted_count.to_i > 0 || requeued_count.to_i > 0)
+          record_flaky(entry) if !skip_flaky_record && (error_reports_deleted_count.to_i > 0 || requeued_count.to_i > 0)
           # Count this run when we ack'd or when we replaced a failure (so stats delta is applied)
           !!(acknowledged || error_reports_deleted_count.to_i > 0)
         end
 
-        def record_requeue(id)
+        def record_requeue(entry)
           true
         end
 
@@ -142,11 +142,11 @@ module CI
         end
 
         def error_reports
-          redis.hgetall(key('error-reports'))
+          redis.hgetall(key('error-reports')).transform_keys { |entry| CI::Queue::QueueEntry.test_id(entry) }
         end
 
         def flaky_reports
-          redis.smembers(key('flaky-reports'))
+          redis.smembers(key('flaky-reports')).map { |entry| CI::Queue::QueueEntry.test_id(entry) }
         end
 
         def record_worker_profile(profile)
@@ -187,10 +187,10 @@ module CI
           ['build', config.build_id, *args].join(':')
         end
 
-        def store_error_report_delta(test_id, stat_delta)
+        def store_error_report_delta(entry, stat_delta)
           # Only the acknowledging worker's stats include this test; store their delta for correction on success
           payload = { 'worker_id' => config.worker_id.to_s }.merge(stat_delta)
-          redis.hset(key('error-report-deltas'), test_id, JSON.generate(payload))
+          redis.hset(key('error-report-deltas'), entry, JSON.generate(payload))
           redis.expire(key('error-report-deltas'), config.redis_ttl)
         end
 

data/lib/ci/queue/redis/heartbeat.lua

@@ -1,6 +1,4 @@
 -- AUTOGENERATED FILE DO NOT EDIT DIRECTLY
--- @include _entry_helpers
-
 local zset_key = KEYS[1]
 local processed_key = KEYS[2]
 local owners_key = KEYS[3]
@@ -8,12 +6,9 @@ local worker_queue_key = KEYS[4]
 
 local current_time = ARGV[1]
 local entry = ARGV[2]
-local entry_delimiter = ARGV[3]
-
-local test_id = test_id_from_entry(entry, entry_delimiter)
 
 -- already processed, we do not need to bump the timestamp
-if redis.call('sismember', processed_key, test_id) == 1 then
+if redis.call('sismember', processed_key, entry) == 1 then
   return false
 end
 

data/lib/ci/queue/redis/monitor.rb

@@ -13,12 +13,11 @@ module CI
         DEV_SCRIPTS_ROOT = ::File.expand_path('../../../../../../redis', __FILE__)
         RELEASE_SCRIPTS_ROOT = ::File.expand_path('../../redis', __FILE__)
 
-        def initialize(pipe, logger, redis_url, zset_key, processed_key, owners_key, worker_queue_key, entry_delimiter)
+        def initialize(pipe, logger, redis_url, zset_key, processed_key, owners_key, worker_queue_key)
           @zset_key = zset_key
           @processed_key = processed_key
           @owners_key = owners_key
           @worker_queue_key = worker_queue_key
-          @entry_delimiter = entry_delimiter
           @logger = logger
           @redis = ::Redis.new(url: redis_url, reconnect_attempts: [0, 0, 0.1, 0.5, 1, 3, 5])
           @shutdown = false
@@ -41,7 +40,7 @@ module CI
           eval_script(
             :heartbeat,
             keys: [@zset_key, @processed_key, @owners_key, @worker_queue_key],
-            argv: [Time.now.to_f, id, @entry_delimiter]
+            argv: [Time.now.to_f, id]
           )
         rescue => error
           @logger.info(error)
@@ -155,10 +154,9 @@ zset_key = ARGV[1]
 processed_key = ARGV[2]
 owners_key = ARGV[3]
 worker_queue_key = ARGV[4]
-entry_delimiter = ARGV[5]
 
 logger.debug("Starting monitor: #{redis_url} #{zset_key} #{processed_key}")
-manager = CI::Queue::Redis::Monitor.new($stdin, logger, redis_url, zset_key, processed_key, owners_key, worker_queue_key, entry_delimiter)
+manager = CI::Queue::Redis::Monitor.new($stdin, logger, redis_url, zset_key, processed_key, owners_key, worker_queue_key)
 
 # Notify the parent we're ready
 $stdout.puts(".")

data/lib/ci/queue/redis/requeue.lua

@@ -11,15 +11,14 @@ local requeued_by_key = KEYS[8]
 local max_requeues = tonumber(ARGV[1])
 local global_max_requeues = tonumber(ARGV[2])
 local entry = ARGV[3]
-local test_id = ARGV[4]
-local offset = ARGV[5]
-local ttl = tonumber(ARGV[6])
+local offset = ARGV[4]
+local ttl = tonumber(ARGV[5])
 
 if redis.call('hget', owners_key, entry) == worker_queue_key then
    redis.call('hdel', owners_key, entry)
 end
 
-if redis.call('sismember', processed_key, test_id) == 1 then
+if redis.call('sismember', processed_key, entry) == 1 then
   return false
 end
 
@@ -28,15 +27,15 @@ if global_requeues and global_requeues >= tonumber(global_max_requeues) then
   return false
 end
 
-local requeues = tonumber(redis.call('hget', requeues_count_key, test_id))
+local requeues = tonumber(redis.call('hget', requeues_count_key, entry))
 if requeues and requeues >= max_requeues then
   return false
 end
 
 redis.call('hincrby', requeues_count_key, '___total___', 1)
-redis.call('hincrby', requeues_count_key, test_id, 1)
+redis.call('hincrby', requeues_count_key, entry, 1)
 
-redis.call('hdel', error_reports_key, test_id)
+redis.call('hdel', error_reports_key, entry)
 
 local pivot = redis.call('lrange', queue_key, -1 - offset, 0 - offset)[1]
 if pivot then

data/lib/ci/queue/redis/reserve_lost.lua

@@ -1,6 +1,4 @@
 -- AUTOGENERATED FILE DO NOT EDIT DIRECTLY
--- @include _entry_helpers
-
 local zset_key = KEYS[1]
 local processed_key = KEYS[2]
 local worker_queue_key = KEYS[3]
@@ -8,12 +6,10 @@ local owners_key = KEYS[4]
 
 local current_time = ARGV[1]
 local timeout = ARGV[2]
-local entry_delimiter = ARGV[3]
 
 local lost_tests = redis.call('zrangebyscore', zset_key, 0, current_time - timeout)
 for _, test in ipairs(lost_tests) do
-  local test_id = test_id_from_entry(test, entry_delimiter)
-  if redis.call('sismember', processed_key, test_id) == 0 then
+  if redis.call('sismember', processed_key, test) == 0 then
     redis.call('zadd', zset_key, current_time, test)
     redis.call('lpush', worker_queue_key, test)
     redis.call('hset', owners_key, test, worker_queue_key) -- Take ownership

data/lib/ci/queue/redis/worker.rb

@@ -147,8 +147,8 @@ module CI
         def retry_queue
           failures = build.failed_tests.to_set
           log = redis.lrange(key('worker', worker_id, 'queue'), 0, -1)
-          log = log.map { |entry| queue_entry_test_id(entry) }
-          log.select! { |id| failures.include?(id) }
+          log = log.map { |entry| CI::Queue::QueueEntry.test_id(entry) }
+          log.select! { |test_id| failures.include?(test_id) }
           log.uniq!
           log.reverse!
           Retry.new(log, config, redis: redis)
@@ -176,23 +176,23 @@ module CI
           build.report_worker_error(error)
         end
 
-        def acknowledge(test_key, error: nil, pipeline: redis)
-          test_id = normalize_test_id(test_key)
+        def acknowledge(entry, error: nil, pipeline: redis)
+          test_id = CI::Queue::QueueEntry.test_id(entry)
           assert_reserved!(test_id)
-          entry = reserved_entries.fetch(test_id, queue_entry_for(test_key))
+          entry = reserved_entries.fetch(test_id, entry)
           unreserve_entry(test_id)
           eval_script(
             :acknowledge,
             keys: [key('running'), key('processed'), key('owners'), key('error-reports'), key('requeued-by')],
-            argv: [entry, test_id, error.to_s, config.redis_ttl],
+            argv: [entry, error.to_s, config.redis_ttl],
             pipeline: pipeline,
           ) == 1
         end
 
-        def requeue(test, offset: Redis.requeue_offset)
-          test_id = normalize_test_id(test)
+        def requeue(entry, offset: Redis.requeue_offset)
+          test_id = CI::Queue::QueueEntry.test_id(entry)
           assert_reserved!(test_id)
-          entry = reserved_entries.fetch(test_id, queue_entry_for(test))
+          entry = reserved_entries.fetch(test_id, entry)
           unreserve_entry(test_id)
           global_max_requeues = config.global_max_requeues(total)
 
@@ -208,7 +208,7 @@ module CI
               key('error-reports'),
               key('requeued-by'),
             ],
-            argv: [config.max_requeues, global_max_requeues, entry, test_id, offset, config.redis_ttl],
+            argv: [config.max_requeues, global_max_requeues, entry, offset, config.redis_ttl],
           ) == 1
 
           unless requeued
@@ -255,7 +255,7 @@ module CI
         end
 
         def reserve_entry(entry)
-          test_id = queue_entry_test_id(entry)
+          test_id = CI::Queue::QueueEntry.test_id(entry)
           reserved_tests << test_id
           reserved_entries[test_id] = entry
           reserved_entry_ids[entry] = test_id
@@ -267,19 +267,6 @@ module CI
           reserved_entry_ids.delete(entry) if entry
         end
 
-        def normalize_test_id(test_key)
-          key = test_key.respond_to?(:id) ? test_key.id : test_key
-          if key.is_a?(String)
-            cached = reserved_entry_ids[key]
-            return cached if cached
-          end
-          queue_entry_test_id(key)
-        end
-
-        def queue_entry_test_id(entry)
-          CI::Queue::QueueEntry.test_id(entry)
-        end
-
         def queue_entry_for(test)
           return test.queue_entry if test.respond_to?(:queue_entry)
           return test.id if test.respond_to?(:id)
@@ -288,7 +275,7 @@ module CI
         end
 
         def resolve_entry(entry)
-          test_id = reserved_entry_ids[entry] || queue_entry_test_id(entry)
+          test_id = reserved_entry_ids[entry] || CI::Queue::QueueEntry.test_id(entry)
           if populated?
             return index[test_id] if index.key?(test_id)
           end
@@ -387,15 +374,18 @@ module CI
             :reserve_lost,
             keys: [
               key('running'),
-              key('completed'),
+              key('processed'),
               key('worker', worker_id, 'queue'),
               key('owners'),
             ],
-            argv: [CI::Queue.time_now.to_f, timeout, CI::Queue::QueueEntry::DELIMITER],
+            argv: [CI::Queue.time_now.to_f, timeout],
           )
 
           if lost_test
-            build.record_warning(Warnings::RESERVED_LOST_TEST, test: lost_test, timeout: config.timeout)
+            build.record_warning(Warnings::RESERVED_LOST_TEST, test: CI::Queue::QueueEntry.test_id(lost_test), timeout: config.timeout)
+            if CI::Queue.debug?
+              $stderr.puts "[ci-queue][reserve_lost] worker=#{worker_id} test_id=#{CI::Queue::QueueEntry.test_id(lost_test)}"
+            end
           end
 
           lost_test

data/lib/ci/queue/static.rb

@@ -125,12 +125,12 @@ module CI
         test_failed >= config.max_test_failed
       end
 
-      def requeue(test)
-        test_key = test.id
-        return false unless should_requeue?(test_key)
+      def requeue(entry)
+        test_id = CI::Queue::QueueEntry.test_id(entry)
+        return false unless should_requeue?(test_id)
 
-        requeues[test_key] += 1
-        @queue.unshift(test_key)
+        requeues[test_id] += 1
+        @queue.unshift(test_id)
         true
       end
 

data/lib/ci/queue/version.rb

@@ -2,7 +2,7 @@
 
 module CI
   module Queue
-    VERSION = '0.83.0'
+    VERSION = '0.84.0'
     DEV_SCRIPTS_ROOT = ::File.expand_path('../../../../../redis', __FILE__)
     RELEASE_SCRIPTS_ROOT = ::File.expand_path('../redis', __FILE__)
   end

data/lib/minitest/queue/build_status_recorder.rb

@@ -40,15 +40,15 @@ module Minitest
         self.total_time = Minitest.clock_time - start_time
         
         # Determine what type of result this is and record it
-        test_id = "#{test.klass}##{test.name}"
+        entry = test.queue_entry
         delta = delta_for(test)
 
         acknowledged = if (test.failure || test.error?) && !test.skipped?
-          build.record_error(test_id, dump(test), stat_delta: delta)
+          build.record_error(entry, dump(test), stat_delta: delta)
         elsif test.requeued?
-          build.record_requeue(test_id)
+          build.record_requeue(entry)
         else
-          build.record_success(test_id, skip_flaky_record: test.skipped?)
+          build.record_success(entry, skip_flaky_record: test.skipped?)
         end
 
         if acknowledged

data/lib/minitest/queue/test_data.rb

@@ -138,7 +138,7 @@ module Minitest
         @error_location ||= begin
           last_before_assertion = ''
           backtrace_for(exception).reverse_each do |s|
-            break if s =~ /in .(assert|refute|flunk|pass|fail|raise|must|wont)/
+            break if s =~ /in [`'](?:[\w:]*[#.])?(assert|refute|flunk|pass|fail|raise|must|wont)/
 
             last_before_assertion = s
           end

data/lib/minitest/queue.rb

@@ -195,15 +195,18 @@ module Minitest
           # When we do a bisect, we don't care about the result other than the test we're running the bisect on
           result.mark_as_flaked!
           failed = false
+        end
+
+        if failed && CI::Queue.requeueable?(result) && queue.requeue(example.queue_entry)
+          result.requeue!
+          if CI::Queue.debug?
+            $stderr.puts "[ci-queue][requeue] test_id=#{example.id} error_class=#{result.failures.first&.class} error=#{result.failures.first&.message&.lines&.first&.chomp}"
+          end
         elsif failed
           queue.report_failure!
         else
           queue.report_success!
         end
-
-        if failed && CI::Queue.requeueable?(result) && queue.requeue(example)
-          result.requeue!
-        end
         reporter.record(result)
       end
 
@@ -327,7 +330,7 @@ module Minitest
       end
 
       def queue_entry
-        id
+        @queue_entry ||= CI::Queue::QueueEntry.format(id, nil)
       end
 
       def <=>(other)

data/lib/rspec/queue/build_status_recorder.rb

@@ -18,12 +18,14 @@ module RSpec
 
       def example_passed(notification)
         example = notification.example
-        build.record_success(example.id)
+        entry = CI::Queue::QueueEntry.format(example.id, nil)
+        build.record_success(entry)
       end
 
       def example_failed(notification)
         example = notification.example
-        build.record_error(example.id, dump(notification))
+        entry = CI::Queue::QueueEntry.format(example.id, nil)
+        build.record_error(entry, dump(notification))
       end
 
       private

data/lib/rspec/queue.rb

@@ -253,6 +253,10 @@ module RSpec
         example.id
       end
 
+      def queue_entry
+        @queue_entry ||= CI::Queue::QueueEntry.format(id, nil)
+      end
+
       def <=>(other)
         id <=> other.id
       end
@@ -411,7 +415,7 @@ module RSpec
       end
 
       def requeue
-        @queue.requeue(@example)
+        @queue.requeue(@example.queue_entry)
       end
 
       def cancel_run!
@@ -422,7 +426,7 @@ module RSpec
       end
 
       def acknowledge
-        @queue.acknowledge(@example)
+        @queue.acknowledge(@example.queue_entry)
       end
     end
 

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: ci-queue
 version: !ruby/object:Gem::Version
-  version: 0.83.0
+  version: 0.84.0
 platform: ruby
 authors:
 - Jean Boussier
@@ -149,6 +149,34 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
+- !ruby/object:Gem::Dependency
+  name: benchmark
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rexml
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: rubocop
   requirement: !ruby/object:Gem::Requirement
@@ -199,7 +227,6 @@ files:
 - lib/ci/queue/output_helpers.rb
 - lib/ci/queue/queue_entry.rb
 - lib/ci/queue/redis.rb
-- lib/ci/queue/redis/_entry_helpers.lua
 - lib/ci/queue/redis/acknowledge.lua
 - lib/ci/queue/redis/base.rb
 - lib/ci/queue/redis/build_record.rb
@@ -257,14 +284,14 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: '2.7'
+      version: '3.1'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 4.0.6
+rubygems_version: 4.0.8
 specification_version: 4
 summary: Distribute tests over many workers using a queue
 test_files: []

data/lib/ci/queue/redis/_entry_helpers.lua

@@ -1,10 +0,0 @@
--- AUTOGENERATED FILE DO NOT EDIT DIRECTLY
-local function test_id_from_entry(value, delimiter)
-  if delimiter then
-    local pos = string.find(value, delimiter, 1, true)
-    if pos then
-      return string.sub(value, 1, pos - 1)
-    end
-  end
-  return value
-end