ci-queue 0.38.0 → 0.40.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 2d2c7f2bbcfac416404fdccc8e0ef588c4ea7e118e1df22ec2e9ab35df91b07e
-  data.tar.gz: d0ef379649b9918571d62b38c9c891403b65739f125a6d895dfba32f7f4ba327
+  metadata.gz: be4adb99c549118b56b3b989e226acd24276f2105cd407ebe7c60685b439712f
+  data.tar.gz: 53ad2297734965e9eb942710e6c6e0dcff81b2597d864f1c0336cbf88c5b792d
 SHA512:
-  metadata.gz: b01ba206b10a05262680a9e2b199b5710b584e36b746ae521d4bcafcb9e324c9abc0b5704a7e634393cdeca3fe32bfb2cfe3e6aa4489114907cb1638b428ab82
-  data.tar.gz: 14dde136243566c2d1a9c6e0bb2a594921f04399bcde2598ad95efd0127288e345b237e30d4e057d2038a8c236e0e723ae5b4b2c00c9d258ad13637915f82288
+  metadata.gz: c5748d3f7cb1b9a9dd9cd386394c6e812f900fe7883686e920b8d2375fbb4fbfaf6c1a1a78703866f1bc3a64bf704cb21188c9fe3b1ddf762490aa1a36344770
+  data.tar.gz: d8d146b0427253a80d1ad3739d976bef95bf43fbee151b59fb3810d75450cfb3f3dbddd2a0825c9aea966b44d2c3226874fb182940a473e27873f526f2d7564b

data/dev.yml

@@ -3,7 +3,7 @@
 name: ci-queue
 
 up:
-- ruby: 2.6.5
+- ruby: 3.2.0
 - bundler
 - redis
 

data/lib/ci/queue/configuration.rb

@@ -5,7 +5,7 @@ module CI
       attr_accessor :timeout, :worker_id, :max_requeues, :grind_count, :failure_file, :export_flaky_tests_file
       attr_accessor :requeue_tolerance, :namespace, :failing_test, :statsd_endpoint
       attr_accessor :max_test_duration, :max_test_duration_percentile, :track_test_duration
-      attr_accessor :max_test_failed, :redis_ttl
+      attr_accessor :max_test_failed, :redis_ttl, :warnings_file
       attr_reader :circuit_breakers
       attr_writer :seed, :build_id
       attr_writer :queue_init_timeout, :report_timeout, :inactive_workers_timeout
@@ -36,7 +36,7 @@ module CI
         grind_count: nil, max_duration: nil, failure_file: nil, max_test_duration: nil,
         max_test_duration_percentile: 0.5, track_test_duration: false, max_test_failed: nil,
         queue_init_timeout: nil, redis_ttl: 8 * 60 * 60, report_timeout: nil, inactive_workers_timeout: nil,
-        export_flaky_tests_file: nil
+        export_flaky_tests_file: nil, warnings_file: nil
       )
         @build_id = build_id
         @circuit_breakers = [CircuitBreaker::Disabled]
@@ -61,6 +61,7 @@ module CI
         @report_timeout = report_timeout
         @inactive_workers_timeout = inactive_workers_timeout
         @export_flaky_tests_file = export_flaky_tests_file
+        @warnings_file = warnings_file
       end
 
       def queue_init_timeout

data/lib/ci/queue/redis/base.rb

@@ -13,7 +13,12 @@ module CI
 
         def initialize(redis_url, config)
           @redis_url = redis_url
-          @redis = ::Redis.new(url: redis_url)
+          @redis = ::Redis.new(
+            url: redis_url,
+            # Booting a CI worker is costly, so in case of a Redis blip,
+            # it makes sense to retry for a while before giving up.
+            reconnect_attempts: [0, 0, 0.1, 0.5, 1, 3, 5],
+          )
           @config = config
         end
 

data/lib/ci/queue/redis/build_record.rb

@@ -21,6 +21,13 @@ module CI
           redis.hkeys(key('error-reports'))
         end
 
+        TOTAL_KEY = "___total___"
+        def requeued_tests
+          requeues = redis.hgetall(key('requeues-count'))
+          requeues.delete(TOTAL_KEY)
+          requeues
+        end
+
         def pop_warnings
           warnings = redis.multi do |transaction|
             transaction.lrange(key('warnings'), 0, -1)

data/lib/ci/queue/redis.rb

@@ -1,5 +1,6 @@
 # frozen_string_literal: true
 
+gem 'redis', '>= 5'
 require 'redis'
 require 'ci/queue/redis/build_record'
 require 'ci/queue/redis/base'

data/lib/ci/queue/version.rb

@@ -2,7 +2,7 @@
 
 module CI
   module Queue
-    VERSION = '0.38.0'
+    VERSION = '0.40.0'
     DEV_SCRIPTS_ROOT = ::File.expand_path('../../../../../redis', __FILE__)
     RELEASE_SCRIPTS_ROOT = ::File.expand_path('../redis', __FILE__)
   end

data/lib/minitest/queue/build_status_reporter.rb

@@ -21,11 +21,21 @@ module Minitest
         build.flaky_reports
       end
 
+      def requeued_tests
+        build.requeued_tests
+      end
+
       def report
         puts aggregates
         errors = error_reports
         puts errors
 
+        requeued_tests.to_a.sort.each do |test_id, count|
+          puts yellow("REQUEUE")
+          puts "#{test_id} (requeued #{count} times)"
+          puts ""
+        end
+
         errors.empty?
       end
 

data/lib/minitest/queue/runner.rb

@@ -230,6 +230,8 @@ module Minitest
             msg = "#{supervisor.size} tests weren't run."
             if supervisor.max_test_failed?
               puts('Encountered too many failed tests. Test run was ended early.')
+              reporter = BuildStatusReporter.new(build: supervisor.build)
+              reporter.report
               abort!(msg)
             else
               abort!(msg)
@@ -296,17 +298,17 @@ module Minitest
       end
 
       def display_warnings(build)
-        build.pop_warnings.each do |type, attributes|
+        return unless queue_config.warnings_file
+
+        warnings = build.pop_warnings.map do |type, attributes|
           case type
           when CI::Queue::Warnings::RESERVED_LOST_TEST
-            puts reopen_previous_step
-            puts yellow(
-              "[WARNING] #{attributes[:test]} was picked up by another worker because it didn't complete in the allocated #{attributes[:timeout]} seconds.\n" \
-              "You may want to either optimize this test or bump ci-queue timeout.\n" \
-              "It's also possible that the worker that was processing it was terminated without being able to report back.\n"
-            )
+            "[WARNING] #{attributes[:test]} was picked up by another worker because it didn't complete in the allocated #{attributes[:timeout]} seconds.\n" \
+            "You may want to either optimize this test or bump ci-queue timeout.\n" \
+            "It's also possible that the worker that was processing it was terminated without being able to report back.\n"
           end
-        end
+        end.compact
+        File.write(queue_config.warnings_file, warnings.join("\n"))
       end
 
       def run_tests_in_fork(queue)
@@ -526,6 +528,15 @@ module Minitest
             queue_config.export_flaky_tests_file = file
           end
 
+          help = <<~EOS
+            Defines a file where warnings during the execution are written to.
+            Defaults to disabled.
+          EOS
+          opts.separator ""
+          opts.on('--warnings-file FILE', help) do |file|
+            queue_config.warnings_file = file
+          end
+
           help = <<~EOS
             Defines after how many consecutive failures the worker will be considered unhealthy and terminate itself.
             Defaults to disabled.

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: ci-queue
 version: !ruby/object:Gem::Version
-  version: 0.38.0
+  version: 0.40.0
 platform: ruby
 authors:
 - Jean Boussier
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2023-10-24 00:00:00.000000000 Z
+date: 2023-12-21 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: bundler
@@ -239,7 +239,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.4.21
+rubygems_version: 3.4.22
 signing_key: 
 specification_version: 4
 summary: Distribute tests over many workers using a queue