ci-queue 0.39.0 → 0.41.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: b107915399e844aa0692d7734747517860631865a6552d4ef3ca4ca8e4be70c5
-  data.tar.gz: ab35800b2df6a59fec8f6af18e3ab6240cc09818bad1de8fcb23f344541455fc
+  metadata.gz: 23effff409f076f80b7af3728d46a4122a59ca1a5a4c3bd1de5162250d60e72f
+  data.tar.gz: ae3c402de5003a6acb3ad002305b85c092ae76d4850634892cab935e03d4479b
 SHA512:
-  metadata.gz: 53a25b74ed491b2a1d62b8579f2416c7376ba1eaad0831bbb09a45a145177d68e4ac3b86de971e1645cdbabe74d1405109cc325e25932a130099f0996a0058c9
-  data.tar.gz: 499841246ebd2364397b54a068c033d90b0a5aa670e116678e51002af0eb3c6664bd4585b4e40c841714a2eb969d388707ae8b6a12db60d4ed22269c08c7aa85
+  metadata.gz: 0601ef053acbb457931feec6428c627cdc1ef3d8d3e76cdd54a98f02ef6f8f6f531a38be13dc3c40d2800ca0e9efe00c91c21a569470887978e8ba997880933a
+  data.tar.gz: c95d7058762b67a059c4868b1d69a8fd99d5f6098d70fa08d71ec7533f7674b8c0886d1090c41db0ba200273b0cbb1c5fbb0fc12ecadddab04f2fb6a9ec950f3

data/dev.yml

@@ -3,7 +3,7 @@
 name: ci-queue
 
 up:
-- ruby: 2.6.5
+- ruby: 3.3.0
 - bundler
 - redis
 

data/lib/ci/queue/configuration.rb

@@ -5,7 +5,7 @@ module CI
       attr_accessor :timeout, :worker_id, :max_requeues, :grind_count, :failure_file, :export_flaky_tests_file
       attr_accessor :requeue_tolerance, :namespace, :failing_test, :statsd_endpoint
       attr_accessor :max_test_duration, :max_test_duration_percentile, :track_test_duration
-      attr_accessor :max_test_failed, :redis_ttl
+      attr_accessor :max_test_failed, :redis_ttl, :warnings_file
       attr_reader :circuit_breakers
       attr_writer :seed, :build_id
       attr_writer :queue_init_timeout, :report_timeout, :inactive_workers_timeout
@@ -36,7 +36,7 @@ module CI
         grind_count: nil, max_duration: nil, failure_file: nil, max_test_duration: nil,
         max_test_duration_percentile: 0.5, track_test_duration: false, max_test_failed: nil,
         queue_init_timeout: nil, redis_ttl: 8 * 60 * 60, report_timeout: nil, inactive_workers_timeout: nil,
-        export_flaky_tests_file: nil
+        export_flaky_tests_file: nil, warnings_file: nil
       )
         @build_id = build_id
         @circuit_breakers = [CircuitBreaker::Disabled]
@@ -61,6 +61,7 @@ module CI
         @report_timeout = report_timeout
         @inactive_workers_timeout = inactive_workers_timeout
         @export_flaky_tests_file = export_flaky_tests_file
+        @warnings_file = warnings_file
       end
 
       def queue_init_timeout

data/lib/ci/queue/redis/base.rb

@@ -13,12 +13,16 @@ module CI
 
         def initialize(redis_url, config)
           @redis_url = redis_url
-          @redis = ::Redis.new(
-            url: redis_url,
-            # Booting a CI worker is costly, so in case of a Redis blip,
-            # it makes sense to retry for a while before giving up.
-            reconnect_attempts: [0, 0, 0.1, 0.5, 1, 3, 5],
-          )
+          if ::Redis::VERSION > "5.0.0"
+            @redis = ::Redis.new(
+              url: redis_url,
+              # Booting a CI worker is costly, so in case of a Redis blip,
+              # it makes sense to retry for a while before giving up.
+              reconnect_attempts: [0, 0, 0.1, 0.5, 1, 3, 5],
+            )
+          else
+            @redis = ::Redis.new(url: redis_url)
+          end
           @config = config
         end
 

data/lib/ci/queue/redis/build_record.rb

@@ -21,6 +21,13 @@ module CI
           redis.hkeys(key('error-reports'))
         end
 
+        TOTAL_KEY = "___total___"
+        def requeued_tests
+          requeues = redis.hgetall(key('requeues-count'))
+          requeues.delete(TOTAL_KEY)
+          requeues
+        end
+
         def pop_warnings
           warnings = redis.multi do |transaction|
             transaction.lrange(key('warnings'), 0, -1)

data/lib/ci/queue/redis.rb

@@ -1,6 +1,5 @@
 # frozen_string_literal: true
 
-gem 'redis', '>= 5'
 require 'redis'
 require 'ci/queue/redis/build_record'
 require 'ci/queue/redis/base'

data/lib/ci/queue/version.rb

@@ -2,7 +2,7 @@
 
 module CI
   module Queue
-    VERSION = '0.39.0'
+    VERSION = '0.41.0'
     DEV_SCRIPTS_ROOT = ::File.expand_path('../../../../../redis', __FILE__)
     RELEASE_SCRIPTS_ROOT = ::File.expand_path('../redis', __FILE__)
   end

data/lib/minitest/queue/build_status_reporter.rb

@@ -21,11 +21,21 @@ module Minitest
         build.flaky_reports
       end
 
+      def requeued_tests
+        build.requeued_tests
+      end
+
       def report
         puts aggregates
         errors = error_reports
         puts errors
 
+        requeued_tests.to_a.sort.each do |test_id, count|
+          puts yellow("REQUEUE")
+          puts "#{test_id} (requeued #{count} times)"
+          puts ""
+        end
+
         errors.empty?
       end
 

data/lib/minitest/queue/runner.rb

@@ -227,6 +227,8 @@ module Minitest
           end
 
           unless supervisor.exhausted?
+            reporter = BuildStatusReporter.new(build: supervisor.build)
+            reporter.report
             msg = "#{supervisor.size} tests weren't run."
             if supervisor.max_test_failed?
               puts('Encountered too many failed tests. Test run was ended early.')
@@ -296,17 +298,12 @@ module Minitest
       end
 
       def display_warnings(build)
-        build.pop_warnings.each do |type, attributes|
-          case type
-          when CI::Queue::Warnings::RESERVED_LOST_TEST
-            puts reopen_previous_step
-            puts yellow(
-              "[WARNING] #{attributes[:test]} was picked up by another worker because it didn't complete in the allocated #{attributes[:timeout]} seconds.\n" \
-              "You may want to either optimize this test or bump ci-queue timeout.\n" \
-              "It's also possible that the worker that was processing it was terminated without being able to report back.\n"
-            )
-          end
-        end
+        return unless queue_config.warnings_file
+
+        warnings = build.pop_warnings.map do |type, attributes|
+          attributes.merge(type: type)
+        end.compact
+        File.write(queue_config.warnings_file, warnings.to_json)
       end
 
       def run_tests_in_fork(queue)
@@ -526,6 +523,15 @@ module Minitest
             queue_config.export_flaky_tests_file = file
           end
 
+          help = <<~EOS
+            Defines a file where warnings during the execution are written to.
+            Defaults to disabled.
+          EOS
+          opts.separator ""
+          opts.on('--warnings-file FILE', help) do |file|
+            queue_config.warnings_file = file
+          end
+
           help = <<~EOS
             Defines after how many consecutive failures the worker will be considered unhealthy and terminate itself.
             Defaults to disabled.

data/lib/minitest/queue.rb

@@ -244,20 +244,17 @@ module Minitest
           queue.report_success!
         end
 
-        requeued = false
         if failed && CI::Queue.requeueable?(result) && queue.requeue(example)
-          requeued = true
           result.requeue!
           reporter.record(result)
-        elsif queue.acknowledge(example) || !failed
+        elsif queue.acknowledge(example)
+          reporter.record(result)
+          queue.increment_test_failed if failed
+        elsif !failed
           # If the test was already acknowledged by another worker (we timed out)
           # Then we only record it if it is successful.
           reporter.record(result)
         end
-
-        if !requeued && failed
-          queue.increment_test_failed
-        end
       end
     end
   end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: ci-queue
 version: !ruby/object:Gem::Version
-  version: 0.39.0
+  version: 0.41.0
 platform: ruby
 authors:
 - Jean Boussier
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2023-12-11 00:00:00.000000000 Z
+date: 2024-01-04 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -239,7 +239,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.4.22
+rubygems_version: 3.5.4
 signing_key: 
 specification_version: 4
 summary: Distribute tests over many workers using a queue