ci-queue 0.66.0 → 0.68.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: b78efa3839e0fdc3a3998ef7d664bab01f28d0cdde96c28b790f9c912964d33a
-  data.tar.gz: 4bc014eb8401ce329be509dc76f449a4d80696667563e3236c5f9634514b7df0
+  metadata.gz: dfe678bd81982a8945fff3610d86021f2b32ab96f19cd00bbda28c7837374f0c
+  data.tar.gz: f71d4a43aa80012a863f3ec0c6fccebd3542dd1c92e4cc386ffe1bda80836254
 SHA512:
-  metadata.gz: be5f84376a0a6bc93776eaafa4a725e48061e18ea13d79c407cb599b665201031fc189ae91cd89607f3c2fa062eb388caaa554411405cc7adf6b073610ee078f
-  data.tar.gz: 47057ca6d1b2d35eb0068be06330802a0ce614744f8498d069ef9bc1ffdd499a8a38861a3bfa7aa62e2b10becd0be5c99306026938c7ca704ede42396109367c
+  metadata.gz: 7af733cedec42252b31b2851cdea9067d5dd3974e9549595c492edbacb366125a4655c0c3d7dccb817eba6ab4b8472e0f35c98bbc4bfeb86f6033f5ffd42d93b
+  data.tar.gz: 2ca6d5bf9b6b2d59c3907656afb536c84fb360eee42aee63c3e48136a7b880ef3c16584b50a7bcb0877917d0a62f1dc6f3a13d31277f1c6f7ec20718efd8c5a6

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    ci-queue (0.66.0)
+    ci-queue (0.68.0)
       logger
 
 GEM

data/lib/ci/queue/redis/acknowledge.lua

@@ -2,9 +2,18 @@
 local zset_key = KEYS[1]
 local processed_key = KEYS[2]
 local owners_key = KEYS[3]
+local error_reports_key = KEYS[4]
 
 local test = ARGV[1]
-
+local error = ARGV[2]
+local ttl = ARGV[3]
 redis.call('zrem', zset_key, test)
 redis.call('hdel', owners_key, test)  -- Doesn't matter if it was reclaimed by another workers
-return redis.call('sadd', processed_key, test)
+local acknowledged = redis.call('sadd', processed_key, test)
+
+if acknowledged and error ~= "" then
+  redis.call('hset', error_reports_key, test, error)
+  redis.call('expire', error_reports_key, ttl)
+end
+
+return acknowledged

data/lib/ci/queue/redis/base.rb

@@ -181,8 +181,8 @@ module CI
           master_status == 'setup'
         end
 
-        def increment_test_failed
-          redis.incr(key('test_failed_count'))
+        def increment_test_failed(pipeline: redis)
+          pipeline.incr(key('test_failed_count'))
         end
 
         def test_failed
@@ -225,8 +225,8 @@ module CI
           redis.get(key('master-status'))
         end
 
-        def eval_script(script, *args)
-          redis.evalsha(load_script(script), *args)
+        def eval_script(script, keys:, argv:, pipeline: redis)
+          pipeline.evalsha(load_script(script), keys: keys, argv: argv)
         end
 
         def load_script(script)

data/lib/ci/queue/redis/build_record.rb

@@ -56,23 +56,22 @@ module CI
           redis.rpush(key('warnings'), Marshal.dump([type, attributes]))
         end
 
+        Test = Struct.new(:id) # Hack
+
         def record_error(id, payload, stats: nil)
           redis.pipelined do |pipeline|
-            pipeline.hset(
-              key('error-reports'),
-              id.dup.force_encoding(Encoding::BINARY),
-              payload.dup.force_encoding(Encoding::BINARY),
-            )
-            pipeline.expire(key('error-reports'), config.redis_ttl)
+            @queue.acknowledge(id, error: payload, pipeline: pipeline)
             record_stats(stats, pipeline: pipeline)
+            @queue.increment_test_failed(pipeline: pipeline)
           end
           nil
         end
 
-        def record_success(id, stats: nil, skip_flaky_record: false)
+        def record_success(id, stats: nil, skip_flaky_record: false, acknowledge: true)
+          @queue.acknowledge(id) if acknowledge
           error_reports_deleted_count, requeued_count, _ = redis.pipelined do |pipeline|
-            pipeline.hdel(key('error-reports'), id.dup.force_encoding(Encoding::BINARY))
-            pipeline.hget(key('requeues-count'), id.b)
+            pipeline.hdel(key('error-reports'), id)
+            pipeline.hget(key('requeues-count'), id)
             record_stats(stats, pipeline: pipeline)
           end
           record_flaky(id) if !skip_flaky_record && (error_reports_deleted_count.to_i > 0 || requeued_count.to_i > 0)

data/lib/ci/queue/redis/grind_record.rb

@@ -14,7 +14,7 @@ module CI
           redis.pipelined do |pipeline|
             pipeline.lpush(
               key('error-reports'),
-              payload.force_encoding(Encoding::BINARY),
+              payload,
             )
             pipeline.expire(key('error-reports'), config.redis_ttl)
             record_stats(stats, pipeline: pipeline)

data/lib/ci/queue/redis/test_time_record.rb

@@ -22,7 +22,7 @@ module CI
           redis.pipelined do |pipeline|
             pipeline.lpush(
               test_time_key(test_name),
-              duration.to_s.force_encoding(Encoding::BINARY),
+              duration.to_s,
             )
             pipeline.expire(test_time_key(test_name), config.redis_ttl)
           end
@@ -33,7 +33,7 @@ module CI
           redis.pipelined do |pipeline|
             pipeline.lpush(
               all_test_names_key,
-              test_name.dup.force_encoding(Encoding::BINARY),
+              test_name,
             )
             pipeline.expire(all_test_names_key, config.redis_ttl)
           end
@@ -53,11 +53,11 @@ module CI
         end
 
         def all_test_names_key
-          "build:#{config.build_id}:list_of_test_names".dup.force_encoding(Encoding::BINARY)
+          "build:#{config.build_id}:list_of_test_names"
         end
 
         def test_time_key(test_name)
-          "build:#{config.build_id}:#{test_name}".dup.force_encoding(Encoding::BINARY)
+          "build:#{config.build_id}:#{test_name}"
         end
       end
     end

data/lib/ci/queue/redis/worker.rb

@@ -7,14 +7,16 @@ module CI
     module Redis
       class << self
         attr_accessor :requeue_offset
+        attr_accessor :max_sleep_time
       end
       self.requeue_offset = 42
+      self.max_sleep_time = 2
 
       class Worker < Base
         attr_reader :total
 
         def initialize(redis, config)
-          @reserved_test = nil
+          @reserved_tests = Set.new
           @shutdown_required = false
           super(redis, config)
         end
@@ -46,13 +48,21 @@ module CI
           @master
         end
 
+        DEFAULT_SLEEP_SECONDS = 0.5
+
         def poll
           wait_for_master
+          attempt = 0
           until shutdown_required? || config.circuit_breakers.any?(&:open?) || exhausted? || max_test_failed?
             if test = reserve
+              attempt = 0
               yield index.fetch(test)
             else
-              sleep 0.05
+              # Adding exponential backoff to avoid hammering Redis
+              # we just stay online here in case a test gets retried or times out so we can afford to wait
+              sleep_time = [DEFAULT_SLEEP_SECONDS * (2 ** attempt), Redis.max_sleep_time].min
+              attempt += 1
+              sleep sleep_time
             end
           end
           redis.pipelined do |pipeline|
@@ -97,13 +107,13 @@ module CI
           build.report_worker_error(error)
         end
 
-        def acknowledge(test)
-          test_key = test.id
+        def acknowledge(test_key, error: nil, pipeline: redis)
           raise_on_mismatching_test(test_key)
           eval_script(
             :acknowledge,
-            keys: [key('running'), key('processed'), key('owners')],
-            argv: [test_key],
+            keys: [key('running'), key('processed'), key('owners'), key('error-reports')],
+            argv: [test_key, error.to_s, config.redis_ttl],
+            pipeline: pipeline,
           ) == 1
         end
 
@@ -125,7 +135,7 @@ module CI
             argv: [config.max_requeues, global_max_requeues, test_key, offset],
           ) == 1
 
-          @reserved_test = test_key unless requeued
+          reserved_tests << test_key unless requeued
           requeued
         end
 
@@ -142,25 +152,24 @@ module CI
 
         attr_reader :index
 
+        def reserved_tests
+          @reserved_tests ||= Set.new
+        end
+
         def worker_id
           config.worker_id
         end
 
         def raise_on_mismatching_test(test)
-          if @reserved_test == test
-            @reserved_test = nil
-          else
-            raise ReservationError, "Acknowledged #{test.inspect} but #{@reserved_test.inspect} was reserved"
+          unless reserved_tests.delete?(test)
+            raise ReservationError, "Acknowledged #{test.inspect} but only #{reserved_tests.map(&:inspect).join(", ")} reserved"
           end
         end
 
         def reserve
-          if @reserved_test
-            raise ReservationError, "#{@reserved_test.inspect} is already reserved. " \
-              "You have to acknowledge it before you can reserve another one"
-          end
-
-          @reserved_test = (try_to_reserve_lost_test || try_to_reserve_test)
+          test = (try_to_reserve_lost_test || try_to_reserve_test)
+          reserved_tests << test
+          test
         end
 
         def try_to_reserve_test

data/lib/ci/queue/redis.rb

@@ -1,5 +1,6 @@
 # frozen_string_literal: true
 
+gem "redis", "~> 5.0"
 require 'redis'
 require 'ci/queue/redis/build_record'
 require 'ci/queue/redis/base'

data/lib/ci/queue/static.rb

@@ -89,26 +89,27 @@ module CI
       end
 
       def running
-        @reserved_test ? 1 : 0
+        reserved_tests.empty? ? 0 : 1
       end
 
       def poll
-        while !@shutdown && config.circuit_breakers.none?(&:open?) && !max_test_failed? && @reserved_test = @queue.shift
-          yield index.fetch(@reserved_test)
+        while !@shutdown && config.circuit_breakers.none?(&:open?) && !max_test_failed? && reserved_test = @queue.shift
+          reserved_tests << reserved_test
+          yield index.fetch(reserved_test)
         end
-        @reserved_test = nil
+        reserved_tests.clear
       end
 
       def exhausted?
         @queue.empty?
       end
 
-      def acknowledge(test)
+      def acknowledge(...)
         @progress += 1
         true
       end
 
-      def increment_test_failed
+      def increment_test_failed(...)
         @test_failed = test_failed + 1
       end
 
@@ -142,6 +143,10 @@ module CI
       def requeues
         @requeues ||= Hash.new(0)
       end
+
+      def reserved_tests
+        @reserved_tests ||= Set.new
+      end
     end
   end
 end

data/lib/ci/queue/version.rb

@@ -2,7 +2,7 @@
 
 module CI
   module Queue
-    VERSION = '0.66.0'
+    VERSION = '0.68.0'
     DEV_SCRIPTS_ROOT = ::File.expand_path('../../../../../redis', __FILE__)
     RELEASE_SCRIPTS_ROOT = ::File.expand_path('../redis', __FILE__)
   end

data/lib/minitest/queue/build_status_recorder.rb

@@ -52,7 +52,7 @@ module Minitest
         if (test.failure || test.error?) && !test.skipped?
           build.record_error("#{test.klass}##{test.name}", dump(test), stats: stats)
         else
-          build.record_success("#{test.klass}##{test.name}", stats: stats, skip_flaky_record: test.skipped?)
+          build.record_success("#{test.klass}##{test.name}", stats: stats, skip_flaky_record: test.skipped?, acknowledge: !test.requeued?)
         end
       end
 

data/lib/minitest/queue/build_status_reporter.rb

@@ -108,7 +108,16 @@ module Minitest
         build.requeued_tests
       end
 
+      APPLICATION_ERROR_EXIT_CODE = 42
+      TIMED_OUT_EXIT_CODE = 43
+      TOO_MANY_FAILED_TESTS_EXIT_CODE = 44
+      WORKERS_DIED_EXIT_CODE = 45
+      SUCCESS_EXIT_CODE = 0
+      TEST_FAILURE_EXIT_CODE = 1
+
       def report
+        exit_code = TEST_FAILURE_EXIT_CODE
+
         if requeued_tests.to_a.any?
           step("Requeued #{requeued_tests.size} tests")
           requeued_tests.to_a.sort.each do |test_id, count|
@@ -131,10 +140,14 @@ module Minitest
           if remaining_tests.size > 10
             puts "  ..."
           end
+
+          exit_code = TIMED_OUT_EXIT_CODE
         elsif supervisor.time_left_with_no_workers.to_i <= 0
           puts red("All workers died.")
+          exit_code = WORKERS_DIED_EXIT_CODE
         elsif supervisor.max_test_failed?
           puts red("Encountered too many failed tests. Test run was ended early.")
+          exit_code = TOO_MANY_FAILED_TESTS_EXIT_CODE
         end
 
         puts
@@ -146,9 +159,10 @@ module Minitest
           puts red("Worker #{worker_id } crashed")
           puts error
           puts ""
+          exit_code = APPLICATION_ERROR_EXIT_CODE
         end
 
-        success?
+        success? ? SUCCESS_EXIT_CODE : exit_code
       end
 
       def success?

data/lib/minitest/queue/runner.rb

@@ -253,25 +253,24 @@ module Minitest
 
         unless supervisor.wait_for_workers { display_warnings(supervisor.build) }
           unless supervisor.queue_initialized?
-            abort! "No master was elected. Did all workers crash?", 40
+            abort! "No leader was elected. This typically means no worker was able to start. Were there any errors during application boot?", 40
           end
 
           unless supervisor.exhausted?
             reporter = BuildStatusReporter.new(supervisor: supervisor)
-            reporter.report
+            exit_code = reporter.report
             reporter.write_failure_file(queue_config.failure_file) if queue_config.failure_file
             reporter.write_flaky_tests_file(queue_config.export_flaky_tests_file) if queue_config.export_flaky_tests_file
 
-            abort!("#{supervisor.size} tests weren't run.")
+            abort!("#{supervisor.size} tests weren't run.", exit_code)
           end
         end
 
         reporter = BuildStatusReporter.new(supervisor: supervisor)
         reporter.write_failure_file(queue_config.failure_file) if queue_config.failure_file
         reporter.write_flaky_tests_file(queue_config.export_flaky_tests_file) if queue_config.export_flaky_tests_file
-        reporter.report
-
-        exit! reporter.success? ? 0 : 1
+        exit_code = reporter.report
+        exit! exit_code
       end
 
       def report_grind_command

data/lib/minitest/queue.rb

@@ -107,7 +107,7 @@ module Minitest
   end
 
   module Queue
-    include ::CI::Queue::OutputHelpers
+    extend ::CI::Queue::OutputHelpers
     attr_writer :run_command_formatter, :project_root
 
     def run_command_formatter
@@ -149,7 +149,79 @@ module Minitest
       path
     end
 
+    class << self
+      def queue
+        Minitest.queue
+      end
+
+      def run(reporter, *)
+        rescue_run_errors do
+          queue.poll do |example|
+            result = queue.with_heartbeat(example.id) do
+              example.run
+            end
+
+            handle_test_result(reporter, example, result)
+          end
+
+          queue.stop_heartbeat!
+        end
+      end
+
+      def handle_test_result(reporter, example, result)
+        failed = !(result.passed? || result.skipped?)
+
+        if example.flaky?
+          result.mark_as_flaked!
+          failed = false
+        end
+
+        if failed && queue.config.failing_test && queue.config.failing_test != example.id
+          # When we do a bisect, we don't care about the result other than the test we're running the bisect on
+          result.mark_as_flaked!
+          failed = false
+        elsif failed
+          queue.report_failure!
+        else
+          queue.report_success!
+        end
+
+        if failed && CI::Queue.requeueable?(result) && queue.requeue(example)
+          result.requeue!
+        end
+        reporter.record(result)
+      end
+
+      private
+
+      def rescue_run_errors(&block)
+        block.call
+      rescue Errno::EPIPE
+        # This happens when the heartbeat process dies
+        reopen_previous_step
+        puts red("The heartbeat process died. This worker is exiting early.")
+        exit!(41)
+      rescue CI::Queue::Error => error
+        reopen_previous_step
+        puts red("#{error.class}: #{error.message}")
+        error.backtrace.each do |frame|
+          puts red(frame)
+        end
+        exit!(41)
+      rescue => error
+        reopen_previous_step
+        Minitest.queue.report_worker_error(error)
+        puts red("This worker exited because of an uncaught application error:")
+        puts red("#{error.class}: #{error.message}")
+        error.backtrace.each do |frame|
+          puts red(frame)
+        end
+        exit!(42)
+      end
+    end
+
     class SingleExample
+      attr_reader :runnable, :method_name
 
       def initialize(runnable, method_name)
         @runnable = runnable
@@ -211,7 +283,7 @@ module Minitest
 
     def __run(*args)
       if queue
-        run_from_queue(*args)
+        Queue.run(*args)
 
         if queue.config.circuit_breakers.any?(&:open?)
           STDERR.puts queue.config.circuit_breakers.map(&:message).join(' ').strip
@@ -224,65 +296,6 @@ module Minitest
         super
       end
     end
-
-    def run_from_queue(reporter, *)
-      queue.poll do |example|
-        result = queue.with_heartbeat(example.id) do
-          example.run
-        end
-
-        failed = !(result.passed? || result.skipped?)
-
-        if example.flaky?
-          result.mark_as_flaked!
-          failed = false
-        end
-
-        if failed && queue.config.failing_test && queue.config.failing_test != example.id
-          # When we do a bisect, we don't care about the result other than the test we're running the bisect on
-          result.mark_as_flaked!
-          failed = false
-        elsif failed
-          queue.report_failure!
-        else
-          queue.report_success!
-        end
-
-        if failed && CI::Queue.requeueable?(result) && queue.requeue(example)
-          result.requeue!
-          reporter.record(result)
-        elsif queue.acknowledge(example)
-          reporter.record(result)
-          queue.increment_test_failed if failed
-        elsif !failed
-          # If the test was already acknowledged by another worker (we timed out)
-          # Then we only record it if it is successful.
-          reporter.record(result)
-        end
-      end
-      queue.stop_heartbeat!
-    rescue Errno::EPIPE
-      # This happens when the heartbeat process dies
-      reopen_previous_step
-      puts red("The heartbeat process died. This worker is exiting early.")
-      exit!(41)
-    rescue CI::Queue::Error => error
-      reopen_previous_step
-      puts red("#{error.class}: #{error.message}")
-      error.backtrace.each do |frame|
-        puts red(frame)
-      end
-      exit!(41)
-    rescue => error
-      reopen_previous_step
-      queue.report_worker_error(error)
-      puts red("This worker exited because of an uncaught application error:")
-      puts red("#{error.class}: #{error.message}")
-      error.backtrace.each do |frame|
-        puts red(frame)
-      end
-      exit!(42)
-    end
   end
 end
 

data/lib/rspec/queue.rb

@@ -224,13 +224,8 @@ module RSpec
             reporter.cancel_run!
             dup.mark_as_requeued!(reporter)
             return true
-          elsif reporter.acknowledge || !@exception
-            # If the test was already acknowledged by another worker (we timed out)
-            # Then we only record it if it is successful.
-            super(reporter)
           else
-            reporter.cancel_run!
-            return
+            super(reporter)
           end
         else
           super(reporter)
@@ -288,7 +283,7 @@ module RSpec
 
         unless supervisor.wait_for_workers
           unless supervisor.queue_initialized?
-            abort! "No master was elected. Did all workers crash?"
+            abort! "No leader was elected. This typically means no worker was able to start. Were there any errors during application boot?"
           end
 
           unless supervisor.exhausted?

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: ci-queue
 version: !ruby/object:Gem::Version
-  version: 0.66.0
+  version: 0.68.0
 platform: ruby
 authors:
 - Jean Boussier
@@ -254,7 +254,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.6.8
+rubygems_version: 3.6.9
 specification_version: 4
 summary: Distribute tests over many workers using a queue
 test_files: []