ci-queue 0.96.0 → 0.97.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 8c3ced29dc916918cd28d47f312be55e014fd5836ab0c91d1e2eab17c9484951
-  data.tar.gz: 3ac890363a4a861ca2c3420c24490dff1ffd89dfb890b8d0fd71062ccf812003
+  metadata.gz: 4d17fbfbe9128059d59548415b65bf99e86f6b9a3345ffd35c3e694bd1f5d1ec
+  data.tar.gz: e864440baac5ff4f930455e565509d78e7900243c251f2b5368e5bd2d8e1e114
 SHA512:
-  metadata.gz: 27f198e8d9201f56112cc29bb3fcaa8518e7d5ddedc629d5eba13de06e0b936a8e809d47162d088f57d5ff614d3c4f92f003f786de62310ae421498e9913d8a9
-  data.tar.gz: 2c5f925d28012585afa8212cbb14afb1966cb0a2e5dbeaa9927486b2e96fb2c96d595f06ef7aa5ab6b97e5c238e5bca01e3572273a8a5fa9e8be357f54c44bf5
+  metadata.gz: 0a7ecb6a7c387b76fab9d317bd619308cc6d4aa1d46684aa4c8a01546308613e1681fecabbebc4340bd9eb2dc8f68a8ca1e7fe0c19202c8b4224d4d260519bf1
+  data.tar.gz: d215b3fc1ef18fefa67493f2e447bc01a500224efbc61bc07c23bbb332e3dfe3855a73de16671ad8cdb2d59eeb1704a1765555d765b2d9a50532249b4f4d0280

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    ci-queue (0.96.0)
+    ci-queue (0.97.0)
       logger
 
 GEM

data/README.md

@@ -36,7 +36,7 @@ Assuming you use one of the supported CI providers, the command can be as simple
 minitest-queue --queue redis://example.com run -Itest test/**/*_test.rb
 ```
 
-Additionally you can configure the requeue settings (see main README) with `--max-requeues` and `--requeue-tolerance`.
+Additionally you can configure the requeue settings (see main README) with `--max-requeues` and `--requeue-tolerance`. Worker-health circuit breakers are available through `--max-consecutive-failures` and `--max-consecutive-requeues`.
 
 #### Lazy loading (opt-in)
 
@@ -158,6 +158,30 @@ rspec-queue --queue redis://example.com --timeout 600 --report
 
 Because of how `ci-queue` executes the examples, `before(:all)` and `after(:all)` hooks are not supported. `rspec-queue` will explicitly reject them.
 
+### Worker circuit breakers
+
+Both runners support two independent, disabled-by-default circuit breakers:
+
+- `--max-consecutive-failures MAX` stops a worker after `MAX` consecutive final test failures.
+- `--max-consecutive-requeues MAX` stops a worker after `MAX` consecutive failures were accepted for requeueing.
+
+The requeue count is local to each worker and resets whenever that worker produces any non-requeued result, including a pass, skip, ignored flaky result, or final failure. An accepted requeue neither increments nor resets the consecutive-final-failure count. Requeue attempts rejected because of an ownership race or an exhausted retry budget follow the final-failure path and do not increment the requeue count.
+
+For example:
+
+```bash
+minitest-queue --max-consecutive-requeues 10 --queue redis://example.com run test/**/*_test.rb
+rspec-queue --max-consecutive-requeues 10 --queue redis://example.com
+```
+
+The breaker can also be configured programmatically:
+
+```ruby
+CI::Queue::Configuration.new(max_consecutive_requeues: 10)
+# or
+config.max_consecutive_requeues = 10
+```
+
 ## Releasing a New Version
 
 After merging changes to `main`, follow these steps to release and propagate the update:

data/lib/ci/queue/circuit_breaker.rb

@@ -8,6 +8,9 @@ module CI
         def report_failure!
         end
 
+        def report_requeue!
+        end
+
         def report_success!
         end
 
@@ -32,6 +35,9 @@ module CI
         def report_failure!
         end
 
+        def report_requeue!
+        end
+
         def report_success!
         end
 
@@ -60,6 +66,9 @@ module CI
           @consecutive_failures += 1
         end
 
+        def report_requeue!
+        end
+
         def report_success!
           @consecutive_failures = 0
         end
@@ -72,6 +81,33 @@ module CI
           'This worker is exiting early because it encountered too many consecutive test failures, probably because of some corrupted state.'
         end
       end
+
+      class MaxConsecutiveRequeues
+        def initialize(max_consecutive_requeues:)
+          @max = max_consecutive_requeues
+          @consecutive_requeues = 0
+        end
+
+        def report_failure!
+          @consecutive_requeues = 0
+        end
+
+        def report_requeue!
+          @consecutive_requeues += 1
+        end
+
+        def report_success!
+          @consecutive_requeues = 0
+        end
+
+        def open?
+          @consecutive_requeues >= @max
+        end
+
+        def message
+          'This worker is exiting early because it requeued too many consecutive tests, probably because of some corrupted state.'
+        end
+      end
     end
   end
 end

data/lib/ci/queue/common.rb

@@ -27,6 +27,10 @@ module CI
         config.circuit_breakers.each(&:report_failure!)
       end
 
+      def report_requeue!
+        config.circuit_breakers.each(&:report_requeue!)
+      end
+
       def report_success!
         config.circuit_breakers.each(&:report_success!)
       end

data/lib/ci/queue/configuration.rb

@@ -55,7 +55,7 @@ module CI
       def initialize(
         timeout: 30, build_id: nil, worker_id: nil, max_requeues: 0, requeue_tolerance: 0,
         namespace: nil, seed: nil, flaky_tests: [], statsd_endpoint: nil, max_consecutive_failures: nil,
-        grind_count: nil, max_duration: nil, failure_file: nil, max_test_duration: nil,
+        max_consecutive_requeues: nil, grind_count: nil, max_duration: nil, failure_file: nil, max_test_duration: nil,
         max_test_duration_percentile: 0.5, track_test_duration: false, max_test_failed: nil,
         queue_init_timeout: nil, redis_ttl: 8 * 60 * 60, report_timeout: nil, inactive_workers_timeout: nil,
         export_flaky_tests_file: nil, warnings_file: nil, debug_log: nil, max_missed_heartbeat_seconds: nil, heartbeat_max_test_duration: nil,
@@ -79,6 +79,7 @@ module CI
         @track_test_duration = track_test_duration
         @worker_id = worker_id
         self.max_consecutive_failures = max_consecutive_failures
+        self.max_consecutive_requeues = max_consecutive_requeues
         self.max_duration = max_duration
         @redis_ttl = redis_ttl
         @report_timeout = report_timeout
@@ -165,6 +166,12 @@ module CI
         end
       end
 
+      def max_consecutive_requeues=(max)
+        if max
+          @circuit_breakers << CircuitBreaker::MaxConsecutiveRequeues.new(max_consecutive_requeues: max)
+        end
+      end
+
       def max_duration=(duration)
         if duration
           @circuit_breakers << CircuitBreaker::Timeout.new(duration: duration)

data/lib/ci/queue/version.rb

@@ -2,7 +2,7 @@
 
 module CI
   module Queue
-    VERSION = '0.96.0'
+    VERSION = '0.97.0'
     DEV_SCRIPTS_ROOT = ::File.expand_path('../../../../../redis', __FILE__)
     RELEASE_SCRIPTS_ROOT = ::File.expand_path('../redis', __FILE__)
   end

data/lib/minitest/queue/runner.rb

@@ -692,6 +692,15 @@ module Minitest
             queue_config.max_consecutive_failures = max
           end
 
+          help = <<~EOS
+            Defines after how many consecutive accepted requeues the worker will be considered unhealthy and stop reserving tests.
+            Defaults to disabled.
+          EOS
+          opts.separator ""
+          opts.on('--max-consecutive-requeues MAX', Integer, help) do |max|
+            queue_config.max_consecutive_requeues = max
+          end
+
           help = <<~EOS
             Must set this option in report and report_grind command if you set --max-test-duration in the report_grind
           EOS

data/lib/minitest/queue.rb

@@ -198,6 +198,7 @@ module Minitest
         end
 
         if failed && CI::Queue.requeueable?(result) && queue.requeue(example.queue_entry)
+          queue.report_requeue!
           result.requeue!
           if CI::Queue.debug?
             $stderr.puts "[ci-queue][requeue] test_id=#{example.id} error_class=#{result.failures.first&.class} error=#{result.failures.first&.message&.lines&.first&.chomp}"
@@ -576,7 +577,7 @@ module Minitest
           Queue.run(*args)
 
           if queue.config.circuit_breakers.any?(&:open?)
-            STDERR.puts queue.config.circuit_breakers.map(&:message).join(' ').strip
+            STDERR.puts queue.config.circuit_breakers.select(&:open?).map(&:message).join(' ').strip
           end
 
           if queue.max_test_failed?

data/lib/rspec/queue.rb

@@ -158,6 +158,15 @@ module RSpec
           queue_config.max_consecutive_failures = Integer(max)
         end
 
+        help = <<~EOS
+          Defines after how many consecutive accepted requeues the worker will be considered unhealthy and stop reserving tests.
+          Defaults to disabled.
+        EOS
+        parser.separator ""
+        parser.on('--max-consecutive-requeues MAX', *help) do |max|
+          queue_config.max_consecutive_requeues = Integer(max)
+        end
+
         help = <<~EOS
           Defines how long the test report remain after the test run, in seconds.
           Defaults to 28,800 (8 hours)
@@ -215,22 +224,21 @@ module RSpec
 
       def finish(reporter, acknowledge: true)
         if acknowledge && reporter.respond_to?(:requeue)
-          if @exception
-            reporter.report_failure!
-          else
-            reporter.report_success!
-          end
-
           if @exception && CI::Queue.requeueable?(@exception) && reporter.requeue
+            reporter.report_requeue!
             reporter.cancel_run!
             dup.mark_as_requeued!(reporter)
             return true
+          end
+
+          if @exception
+            reporter.report_failure!
           else
-            super(reporter)
+            reporter.report_success!
           end
-        else
-          super(reporter)
         end
+
+        super(reporter)
       end
 
       def reset!
@@ -417,6 +425,10 @@ module RSpec
         @queue.report_failure!
       end
 
+      def report_requeue!
+        @queue.report_requeue!
+      end
+
       def requeue
         @queue.requeue(@example.queue_entry)
       end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: ci-queue
 version: !ruby/object:Gem::Version
-  version: 0.96.0
+  version: 0.97.0
 platform: ruby
 authors:
 - Jean Boussier
@@ -319,7 +319,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 4.0.11
+rubygems_version: 4.0.16
 specification_version: 4
 summary: Distribute tests over many workers using a queue
 test_files: []