ci-queue 0.12.1 → 0.13.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 9f86ba12b294a5698c6b1fa568d5482f549758e54ea85db004507a498a7477f1
-  data.tar.gz: bc54f7c6890c5c1d172a634cafa961a8a5bb76980eed482a7fb0ef8fe1dcaad8
+  metadata.gz: d494a80d73d0f5f6d8b478f30b40b8dc2ed90e24031ca6ae96dade3a1995a75e
+  data.tar.gz: 4504d20539f933366c50f96b094cbc0ae3623556d62516b763dd98f5a1c305df
 SHA512:
-  metadata.gz: bbc243b619402d0b4addf5cb85a9ac4bfa91668a9abd5b6d6c3ab48bc47a317271a2efa8bea88f4e3a9e733f3832ba731ba1afb0c21e8e330fda86f87218586b
-  data.tar.gz: d7b7f715153b663757f4ee60c3f2bd7236e3b7b9308e1577e8e95b26a4af8b7892965cfabc8ca61847c09c39eb73b5ce3463ccede6e209046f7d5e8e21bf474d
+  metadata.gz: 632de17b6e079142195e34d4b9ac0e3754091f786a430f3e0e82ee282fb4d7b7823c89b8c3ab30c397f228c80b37575420affc34da8f0b1ef5fe23f7e325494c
+  data.tar.gz: 00fa21726fcd0c584af82ffc32fa6a32fc78dbe0672ec2916350890e0c9fa6a00205af9a195e7382c3518183c6d63b9745106b6f1255b5672ab121f34f64f712

data/lib/ci/queue.rb

@@ -3,7 +3,9 @@ require 'cgi'
 
 require 'ci/queue/version'
 require 'ci/queue/output_helpers'
+require 'ci/queue/circuit_breaker'
 require 'ci/queue/configuration'
+require 'ci/queue/common'
 require 'ci/queue/build_record'
 require 'ci/queue/static'
 require 'ci/queue/file'

data/lib/ci/queue/circuit_breaker.rb

@@ -0,0 +1,36 @@
+module CI
+  module Queue
+    class CircuitBreaker
+      module Disabled
+        extend self
+
+        def report_failure!
+        end
+
+        def report_success!
+        end
+
+        def open?
+          false
+        end
+      end
+
+      def initialize(max_consecutive_failures:)
+        @max = max_consecutive_failures
+        @consecutive_failures = 0
+      end
+
+      def report_failure!
+        @consecutive_failures += 1
+      end
+
+      def report_success!
+        @consecutive_failures = 0
+      end
+
+      def open?
+        @consecutive_failures >= @max
+      end
+    end
+  end
+end

data/lib/ci/queue/common.rb

@@ -0,0 +1,19 @@
+module CI
+  module Queue
+    module Common
+      attr_reader :config
+
+      def flaky?(test)
+        @config.flaky?(test)
+      end
+
+      def report_failure!
+        config.circuit_breaker.report_failure!
+      end
+
+      def report_success!
+        config.circuit_breaker.report_success!
+      end
+    end
+  end
+end

data/lib/ci/queue/configuration.rb

@@ -1,9 +1,9 @@
-
 module CI
   module Queue
     class Configuration
       attr_accessor :timeout, :build_id, :worker_id, :max_requeues
       attr_accessor :requeue_tolerance, :namespace, :seed, :failing_test, :statsd_endpoint
+      attr_reader :circuit_breaker
 
       class << self
         def from_env(env)
@@ -26,7 +26,7 @@ module CI
 
       def initialize(
         timeout: 30, build_id: nil, worker_id: nil, max_requeues: 0, requeue_tolerance: 0,
-        namespace: nil, seed: nil, flaky_tests: [], statsd_endpoint: nil
+        namespace: nil, seed: nil, flaky_tests: [], statsd_endpoint: nil, max_consecutive_failures: nil
       )
         @namespace = namespace
         @timeout = timeout
@@ -37,6 +37,15 @@ module CI
         @seed = seed
         @flaky_tests = flaky_tests
         @statsd_endpoint = statsd_endpoint
+        self.max_consecutive_failures = max_consecutive_failures
+      end
+
+      def max_consecutive_failures=(max)
+        @circuit_breaker = if max
+          CircuitBreaker.new(max_consecutive_failures: max)
+        else
+          CircuitBreaker::Disabled
+        end
       end
 
       def flaky?(test)

data/lib/ci/queue/redis/base.rb

@@ -2,16 +2,14 @@ module CI
   module Queue
     module Redis
       class Base
+        include Common
+
         def initialize(redis_url, config)
           @redis_url = redis_url
           @redis = ::Redis.new(url: redis_url)
           @config = config
         end
 
-        def flaky?(test)
-          @config.flaky?(test)
-        end
-
         def exhausted?
           queue_initialized? && size == 0
         end
@@ -59,7 +57,7 @@ module CI
 
         private
 
-        attr_reader :redis, :config, :redis_url
+        attr_reader :redis, :redis_url
 
         def key(*args)
           ['build', build_id, *args].join(':')

data/lib/ci/queue/redis/worker.rb

@@ -44,7 +44,7 @@ module CI
 
         def poll
           wait_for_master
-          until shutdown_required? || exhausted?
+          until shutdown_required? || config.circuit_breaker.open? || exhausted?
             if test = reserve
               yield index.fetch(test)
             else

data/lib/ci/queue/static.rb

@@ -1,6 +1,7 @@
 module CI
   module Queue
     class Static
+      include Common
       class << self
         def from_uri(uri, config)
           tests = uri.opaque.split(':').map { |t| CGI.unescape(t) }
@@ -17,10 +18,6 @@ module CI
         @total = tests.size
       end
 
-      def flaky?(test)
-        @config.flaky?(test)
-      end
-
       def build
         @build ||= BuildRecord.new(self)
       end
@@ -51,7 +48,7 @@ module CI
       end
 
       def poll
-        while test = @queue.shift
+        while !config.circuit_breaker.open? && test = @queue.shift
           yield index.fetch(test)
           @progress += 1
         end
@@ -75,7 +72,7 @@ module CI
 
       private
 
-      attr_reader :index, :config
+      attr_reader :index
 
       def should_requeue?(key)
         requeues[key] < config.max_requeues && requeues.values.inject(0, :+) < config.global_max_requeues(total)

data/lib/ci/queue/version.rb

@@ -1,6 +1,6 @@
 module CI
   module Queue
-    VERSION = '0.12.1'
+    VERSION = '0.13.0'
     DEV_SCRIPTS_ROOT = ::File.expand_path('../../../../../redis', __FILE__)
     RELEASE_SCRIPTS_ROOT = ::File.expand_path('../redis', __FILE__)
   end

data/lib/minitest/queue.rb

@@ -143,6 +143,10 @@ module Minitest
     def __run(*args)
       if queue
         run_from_queue(*args)
+
+        if queue.config.circuit_breaker.open?
+          STDERR.puts "This worker is exiting early because it encountered too many consecutive test failures, probably because of some corrupted state."
+        end
       else
         super
       end
@@ -158,6 +162,12 @@ module Minitest
           failed = false
         end
 
+        if failed
+          queue.report_failure!
+        else
+          queue.report_success!
+        end
+
         if failed && queue.requeue(example)
           result.requeue!
           reporter.record(result)

data/lib/minitest/queue/runner.rb

@@ -271,6 +271,15 @@ module Minitest
             queue_config.requeue_tolerance = Float(ratio)
           end
 
+          help = split_heredoc(<<-EOS)
+            Defines after how many consecutive failures the worker will be considered unhealthy and terminate itself.
+            Defaults to disabled.
+          EOS
+          opts.separator ""
+          opts.on('--max-consecutive-failures MAX', *help) do |max|
+            queue_config.max_consecutive_failures = Integer(max)
+          end
+
           opts.separator ""
           opts.separator "    retry: Replays a previous run in the same order."
 

data/lib/rspec/queue.rb

@@ -135,6 +135,15 @@ module RSpec
           queue_config.requeue_tolerance = Float(ratio)
         end
 
+        help = split_heredoc(<<-EOS)
+          Defines after how many consecutive failures the worker will be considered unhealthy and terminate itself.
+          Defaults to disabled.
+        EOS
+        parser.separator ""
+        parser.on('--max-consecutive-failures MAX', *help) do |max|
+          queue_config.max_consecutive_failures = Integer(max)
+        end
+
         parser
       end
 
@@ -187,6 +196,12 @@ module RSpec
 
       def finish(reporter, acknowledge: true)
         if acknowledge && reporter.respond_to?(:requeue)
+          if @exception
+            reporter.report_failure!
+          else
+            reporter.report_success!
+          end
+
           if @exception && reporter.requeue
             reporter.cancel_run!
             dup.mark_as_requeued!(reporter)
@@ -301,6 +316,14 @@ module RSpec
         super(reporter)
       end
 
+      def report_success!
+        @queue.report_success!
+      end
+
+      def report_failure!
+        @queue.report_failure!
+      end
+
       def requeue
         @queue.requeue(@example)
       end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: ci-queue
 version: !ruby/object:Gem::Version
-  version: 0.12.1
+  version: 0.13.0
 platform: ruby
 authors:
 - Jean Boussier
 autorequire: 
 bindir: exe
 cert_chain: []
-date: 2018-04-06 00:00:00.000000000 Z
+date: 2018-04-23 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: ansi
@@ -174,6 +174,8 @@ files:
 - lib/ci/queue.rb
 - lib/ci/queue/bisect.rb
 - lib/ci/queue/build_record.rb
+- lib/ci/queue/circuit_breaker.rb
+- lib/ci/queue/common.rb
 - lib/ci/queue/configuration.rb
 - lib/ci/queue/file.rb
 - lib/ci/queue/output_helpers.rb