ci-queue 0.67.0 → 0.68.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: d3a701add5867e968b8dcc2c8d9fe301f88b54ede5b0f2f615171fa1d620e6bd
-  data.tar.gz: 401f2f2c45c09edbd609c177a077b1aec2fe361f376439d6f5b8a29770d7e488
+  metadata.gz: dfe678bd81982a8945fff3610d86021f2b32ab96f19cd00bbda28c7837374f0c
+  data.tar.gz: f71d4a43aa80012a863f3ec0c6fccebd3542dd1c92e4cc386ffe1bda80836254
 SHA512:
-  metadata.gz: f32ff8fa093d89df2194ac043943de43acfedbc5cbcf942c3638d0cc6ccf9659eb107dbf87ef04b3f19ba8329adf1b1275cb2125b8bc9c3cafd7072b49de3e5c
-  data.tar.gz: 01c75cfae762cb874df2160b04c7b96d14d7665631dfaaf642553587b14f7e86bd4d093aad6b9feefa3bee67b6626c3b2a143f3695f50b942579c59949eb16ed
+  metadata.gz: 7af733cedec42252b31b2851cdea9067d5dd3974e9549595c492edbacb366125a4655c0c3d7dccb817eba6ab4b8472e0f35c98bbc4bfeb86f6033f5ffd42d93b
+  data.tar.gz: 2ca6d5bf9b6b2d59c3907656afb536c84fb360eee42aee63c3e48136a7b880ef3c16584b50a7bcb0877917d0a62f1dc6f3a13d31277f1c6f7ec20718efd8c5a6

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    ci-queue (0.67.0)
+    ci-queue (0.68.0)
       logger
 
 GEM

data/lib/ci/queue/redis/worker.rb

@@ -7,14 +7,16 @@ module CI
     module Redis
       class << self
         attr_accessor :requeue_offset
+        attr_accessor :max_sleep_time
       end
       self.requeue_offset = 42
+      self.max_sleep_time = 2
 
       class Worker < Base
         attr_reader :total
 
         def initialize(redis, config)
-          @reserved_test = nil
+          @reserved_tests = Set.new
           @shutdown_required = false
           super(redis, config)
         end
@@ -46,13 +48,21 @@ module CI
           @master
         end
 
+        DEFAULT_SLEEP_SECONDS = 0.5
+
         def poll
           wait_for_master
+          attempt = 0
           until shutdown_required? || config.circuit_breakers.any?(&:open?) || exhausted? || max_test_failed?
             if test = reserve
+              attempt = 0
               yield index.fetch(test)
             else
-              sleep 0.05
+              # Adding exponential backoff to avoid hammering Redis
+              # we just stay online here in case a test gets retried or times out so we can afford to wait
+              sleep_time = [DEFAULT_SLEEP_SECONDS * (2 ** attempt), Redis.max_sleep_time].min
+              attempt += 1
+              sleep sleep_time
             end
           end
           redis.pipelined do |pipeline|
@@ -125,7 +135,7 @@ module CI
             argv: [config.max_requeues, global_max_requeues, test_key, offset],
           ) == 1
 
-          @reserved_test = test_key unless requeued
+          reserved_tests << test_key unless requeued
           requeued
         end
 
@@ -142,25 +152,24 @@ module CI
 
         attr_reader :index
 
+        def reserved_tests
+          @reserved_tests ||= Set.new
+        end
+
         def worker_id
           config.worker_id
         end
 
-        def raise_on_mismatching_test(test_key)
-          if @reserved_test == test_key
-            @reserved_test = nil
-          else
-            raise ReservationError, "Acknowledged #{test_key.inspect} but #{@reserved_test.inspect} was reserved"
+        def raise_on_mismatching_test(test)
+          unless reserved_tests.delete?(test)
+            raise ReservationError, "Acknowledged #{test.inspect} but only #{reserved_tests.map(&:inspect).join(", ")} reserved"
           end
         end
 
         def reserve
-          if @reserved_test
-            raise ReservationError, "#{@reserved_test.inspect} is already reserved. " \
-              "You have to acknowledge it before you can reserve another one"
-          end
-
-          @reserved_test = (try_to_reserve_lost_test || try_to_reserve_test)
+          test = (try_to_reserve_lost_test || try_to_reserve_test)
+          reserved_tests << test
+          test
         end
 
         def try_to_reserve_test

data/lib/ci/queue/static.rb

@@ -89,14 +89,15 @@ module CI
       end
 
       def running
-        @reserved_test ? 1 : 0
+        reserved_tests.empty? ? 0 : 1
       end
 
       def poll
-        while !@shutdown && config.circuit_breakers.none?(&:open?) && !max_test_failed? && @reserved_test = @queue.shift
-          yield index.fetch(@reserved_test)
+        while !@shutdown && config.circuit_breakers.none?(&:open?) && !max_test_failed? && reserved_test = @queue.shift
+          reserved_tests << reserved_test
+          yield index.fetch(reserved_test)
         end
-        @reserved_test = nil
+        reserved_tests.clear
       end
 
       def exhausted?
@@ -142,6 +143,10 @@ module CI
       def requeues
         @requeues ||= Hash.new(0)
       end
+
+      def reserved_tests
+        @reserved_tests ||= Set.new
+      end
     end
   end
 end

data/lib/ci/queue/version.rb

@@ -2,7 +2,7 @@
 
 module CI
   module Queue
-    VERSION = '0.67.0'
+    VERSION = '0.68.0'
     DEV_SCRIPTS_ROOT = ::File.expand_path('../../../../../redis', __FILE__)
     RELEASE_SCRIPTS_ROOT = ::File.expand_path('../redis', __FILE__)
   end

data/lib/minitest/queue/build_status_reporter.rb

@@ -108,7 +108,16 @@ module Minitest
         build.requeued_tests
       end
 
+      APPLICATION_ERROR_EXIT_CODE = 42
+      TIMED_OUT_EXIT_CODE = 43
+      TOO_MANY_FAILED_TESTS_EXIT_CODE = 44
+      WORKERS_DIED_EXIT_CODE = 45
+      SUCCESS_EXIT_CODE = 0
+      TEST_FAILURE_EXIT_CODE = 1
+
       def report
+        exit_code = TEST_FAILURE_EXIT_CODE
+
         if requeued_tests.to_a.any?
           step("Requeued #{requeued_tests.size} tests")
           requeued_tests.to_a.sort.each do |test_id, count|
@@ -131,10 +140,14 @@ module Minitest
           if remaining_tests.size > 10
             puts "  ..."
           end
+
+          exit_code = TIMED_OUT_EXIT_CODE
         elsif supervisor.time_left_with_no_workers.to_i <= 0
           puts red("All workers died.")
+          exit_code = WORKERS_DIED_EXIT_CODE
         elsif supervisor.max_test_failed?
           puts red("Encountered too many failed tests. Test run was ended early.")
+          exit_code = TOO_MANY_FAILED_TESTS_EXIT_CODE
         end
 
         puts
@@ -146,9 +159,10 @@ module Minitest
           puts red("Worker #{worker_id } crashed")
           puts error
           puts ""
+          exit_code = APPLICATION_ERROR_EXIT_CODE
         end
 
-        success?
+        success? ? SUCCESS_EXIT_CODE : exit_code
       end
 
       def success?

data/lib/minitest/queue/runner.rb

@@ -253,25 +253,24 @@ module Minitest
 
         unless supervisor.wait_for_workers { display_warnings(supervisor.build) }
           unless supervisor.queue_initialized?
-            abort! "No master was elected. Did all workers crash?", 40
+            abort! "No leader was elected. This typically means no worker was able to start. Were there any errors during application boot?", 40
           end
 
           unless supervisor.exhausted?
             reporter = BuildStatusReporter.new(supervisor: supervisor)
-            reporter.report
+            exit_code = reporter.report
             reporter.write_failure_file(queue_config.failure_file) if queue_config.failure_file
             reporter.write_flaky_tests_file(queue_config.export_flaky_tests_file) if queue_config.export_flaky_tests_file
 
-            abort!("#{supervisor.size} tests weren't run.")
+            abort!("#{supervisor.size} tests weren't run.", exit_code)
           end
         end
 
         reporter = BuildStatusReporter.new(supervisor: supervisor)
         reporter.write_failure_file(queue_config.failure_file) if queue_config.failure_file
         reporter.write_flaky_tests_file(queue_config.export_flaky_tests_file) if queue_config.export_flaky_tests_file
-        reporter.report
-
-        exit! reporter.success? ? 0 : 1
+        exit_code = reporter.report
+        exit! exit_code
       end
 
       def report_grind_command

data/lib/minitest/queue.rb

@@ -107,7 +107,7 @@ module Minitest
   end
 
   module Queue
-    include ::CI::Queue::OutputHelpers
+    extend ::CI::Queue::OutputHelpers
     attr_writer :run_command_formatter, :project_root
 
     def run_command_formatter
@@ -149,8 +149,79 @@ module Minitest
       path
     end
 
+    class << self
+      def queue
+        Minitest.queue
+      end
+
+      def run(reporter, *)
+        rescue_run_errors do
+          queue.poll do |example|
+            result = queue.with_heartbeat(example.id) do
+              example.run
+            end
+
+            handle_test_result(reporter, example, result)
+          end
+
+          queue.stop_heartbeat!
+        end
+      end
+
+      def handle_test_result(reporter, example, result)
+        failed = !(result.passed? || result.skipped?)
+
+        if example.flaky?
+          result.mark_as_flaked!
+          failed = false
+        end
+
+        if failed && queue.config.failing_test && queue.config.failing_test != example.id
+          # When we do a bisect, we don't care about the result other than the test we're running the bisect on
+          result.mark_as_flaked!
+          failed = false
+        elsif failed
+          queue.report_failure!
+        else
+          queue.report_success!
+        end
+
+        if failed && CI::Queue.requeueable?(result) && queue.requeue(example)
+          result.requeue!
+        end
+        reporter.record(result)
+      end
+
+      private
+
+      def rescue_run_errors(&block)
+        block.call
+      rescue Errno::EPIPE
+        # This happens when the heartbeat process dies
+        reopen_previous_step
+        puts red("The heartbeat process died. This worker is exiting early.")
+        exit!(41)
+      rescue CI::Queue::Error => error
+        reopen_previous_step
+        puts red("#{error.class}: #{error.message}")
+        error.backtrace.each do |frame|
+          puts red(frame)
+        end
+        exit!(41)
+      rescue => error
+        reopen_previous_step
+        Minitest.queue.report_worker_error(error)
+        puts red("This worker exited because of an uncaught application error:")
+        puts red("#{error.class}: #{error.message}")
+        error.backtrace.each do |frame|
+          puts red(frame)
+        end
+        exit!(42)
+      end
+    end
+
     class SingleExample
-      attr_reader :method_name
+      attr_reader :runnable, :method_name
 
       def initialize(runnable, method_name)
         @runnable = runnable
@@ -212,7 +283,7 @@ module Minitest
 
     def __run(*args)
       if queue
-        run_from_queue(*args)
+        Queue.run(*args)
 
         if queue.config.circuit_breakers.any?(&:open?)
           STDERR.puts queue.config.circuit_breakers.map(&:message).join(' ').strip
@@ -225,58 +296,6 @@ module Minitest
         super
       end
     end
-
-    def run_from_queue(reporter, *)
-      queue.poll do |example|
-        result = queue.with_heartbeat(example.id) do
-          example.run
-        end
-
-        failed = !(result.passed? || result.skipped?)
-
-        if example.flaky?
-          result.mark_as_flaked!
-          failed = false
-        end
-
-        if failed && queue.config.failing_test && queue.config.failing_test != example.id
-          # When we do a bisect, we don't care about the result other than the test we're running the bisect on
-          result.mark_as_flaked!
-          failed = false
-        elsif failed
-          queue.report_failure!
-        else
-          queue.report_success!
-        end
-
-        if failed && CI::Queue.requeueable?(result) && queue.requeue(example)
-          result.requeue!
-        end
-        reporter.record(result)
-      end
-      queue.stop_heartbeat!
-    rescue Errno::EPIPE
-      # This happens when the heartbeat process dies
-      reopen_previous_step
-      puts red("The heartbeat process died. This worker is exiting early.")
-      exit!(41)
-    rescue CI::Queue::Error => error
-      reopen_previous_step
-      puts red("#{error.class}: #{error.message}")
-      error.backtrace.each do |frame|
-        puts red(frame)
-      end
-      exit!(41)
-    rescue => error
-      reopen_previous_step
-      queue.report_worker_error(error)
-      puts red("This worker exited because of an uncaught application error:")
-      puts red("#{error.class}: #{error.message}")
-      error.backtrace.each do |frame|
-        puts red(frame)
-      end
-      exit!(42)
-    end
   end
 end
 

data/lib/rspec/queue.rb

@@ -283,7 +283,7 @@ module RSpec
 
         unless supervisor.wait_for_workers
           unless supervisor.queue_initialized?
-            abort! "No master was elected. Did all workers crash?"
+            abort! "No leader was elected. This typically means no worker was able to start. Were there any errors during application boot?"
           end
 
           unless supervisor.exhausted?

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: ci-queue
 version: !ruby/object:Gem::Version
-  version: 0.67.0
+  version: 0.68.0
 platform: ruby
 authors:
 - Jean Boussier