minitest-distributed 0.2.11 → 0.2.13

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 3755320df95c6d8e5ba5e32b04b44351043588d43c309d4d746cc36f96567132
-  data.tar.gz: 785503f39eadcfab72b7fc3df6e8f19d3ad0fbaea793e68f9f233ed08c4f6452
+  metadata.gz: 1605e583ce4427da1909789380c1ac20ffe629a5f1ddf96452f93e45d50188f2
+  data.tar.gz: 052ba1210bee26b878052bb28207b800a67682f023e6fdcc1e03bebaea758376
 SHA512:
-  metadata.gz: fd42edc7fd402f38c08f22ff17fa8430b65cae6d1c53b27afa55446ab5cb6e90f0c0d8a92c6fa7a7665d7aba4a81768a52d9a5c1007da2ebf6407414c2538510
-  data.tar.gz: e3b5119c06eaec10a7f8befacec8b3fde636e17eb9c8a4b539ae4aedf95d00217a97911ccf067e14f08850e1dbc3a6d072723811b565ea06aff46be8fb2607ae
+  metadata.gz: fb550faaa06a888561a3557b10f35fc867ca341e6e4008a49eb5c5137aa79b3490e6a5b24444d67c85ffc79c2e683aff7a21f948e7867b1b78bbafeccdebe9b8
+  data.tar.gz: ad60584710485b804af453696912a2676d56905e69642ead265111689ea10212768925f5c48017fd3c6da7547a94e8450b181dd2f89b5db82dc480d79d843d10

data/dev.yml

@@ -5,3 +5,4 @@ type: ruby
 up:
   - ruby
   - bundler
+  - redis

data/lib/minitest/distributed/coordinators/redis_coordinator.rb

@@ -3,6 +3,7 @@
 
 require "redis"
 require "set"
+require "logger"
 
 module Minitest
   module Distributed
@@ -152,63 +153,65 @@ module Minitest
 
           return if consumer_group_exists
 
-          tests = T.let([], T::Array[Minitest::Runnable])
-          tests = if initial_attempt
-            # If this is the first attempt for this run ID, we will schedule the full
-            # test suite as returned by the test selector to run.
-
-            tests_from_selector = test_selector.tests
-            adjust_combined_results(ResultAggregate.new(size: tests_from_selector.size))
-            tests_from_selector
-
-          elsif configuration.retry_failures
-            # Before starting a retry attempt, we first check if the previous attempt
-            # was aborted before it was completed. If this is the case, we cannot use
-            # retry mode, and should immediately fail the attempt.
-            if combined_results.abort?
-              # We mark this run as aborted, which causes this worker to not be successful.
-              @aborted = true
-
-              # We still publish an empty size run to Redis, so if there are any followers,
-              # they will wind down normally. Only the leader will exit
-              # with a non-zero exit status and fail the build; any follower will
-              # exit with status 0.
-              adjust_combined_results(ResultAggregate.new(size: 0))
-              T.let([], T::Array[Minitest::Runnable])
-            else
-              previous_failures, previous_errors, _deleted = redis.multi do |pipeline|
-                pipeline.lrange(list_key(ResultType::Failed.serialize), 0, -1)
-                pipeline.lrange(list_key(ResultType::Error.serialize), 0, -1)
-                pipeline.del(list_key(ResultType::Failed.serialize), list_key(ResultType::Error.serialize))
-              end
+          tests = T.let(
+            if initial_attempt
+              # If this is the first attempt for this run ID, we will schedule the full
+              # test suite as returned by the test selector to run.
+
+              tests_from_selector = test_selector.tests
+              adjust_combined_results(ResultAggregate.new(size: tests_from_selector.size))
+              tests_from_selector
+
+            elsif configuration.retry_failures
+              # Before starting a retry attempt, we first check if the previous attempt
+              # was aborted before it was completed. If this is the case, we cannot use
+              # retry mode, and should immediately fail the attempt.
+              if combined_results.abort?
+                # We mark this run as aborted, which causes this worker to not be successful.
+                @aborted = true
+
+                # We still publish an empty size run to Redis, so if there are any followers,
+                # they will wind down normally. Only the leader will exit
+                # with a non-zero exit status and fail the build; any follower will
+                # exit with status 0.
+                adjust_combined_results(ResultAggregate.new(size: 0))
+                []
+              else
+                previous_failures, previous_errors, _deleted = redis.multi do |pipeline|
+                  pipeline.lrange(list_key(ResultType::Failed.serialize), 0, -1)
+                  pipeline.lrange(list_key(ResultType::Error.serialize), 0, -1)
+                  pipeline.del(list_key(ResultType::Failed.serialize), list_key(ResultType::Error.serialize))
+                end
 
-              # We set the `size` key to the number of tests we are planning to schedule.
-              # We also adjust the number of failures and errors back to 0.
-              # We set the number of requeues to the number of tests that failed, so the
-              # run statistics will reflect that we retried some failed test.
-              #
-              # However, normally requeues are not acked, as we expect the test to be acked
-              # by another worker later. This makes the test loop think iot is already done.
-              # To prevent this, we initialize the number of acks negatively, so it evens out
-              # in the statistics.
-              total_failures = previous_failures.length + previous_errors.length
-              adjust_combined_results(ResultAggregate.new(
-                size: total_failures,
-                failures: -previous_failures.length,
-                errors: -previous_errors.length,
-                requeues: total_failures,
-              ))
-
-              # For subsequent attempts, we check the list of previous failures and
-              # errors, and only schedule to re-run those tests. This allows for faster
-              # retries of potentially flaky tests.
-              test_identifiers_to_retry = T.let(previous_failures + previous_errors, T::Array[String])
-              test_identifiers_to_retry.map { |identifier| DefinedRunnable.from_identifier(identifier) }
-            end
-          else
-            adjust_combined_results(ResultAggregate.new(size: 0))
-            T.let([], T::Array[Minitest::Runnable])
-          end
+                # We set the `size` key to the number of tests we are planning to schedule.
+                # We also adjust the number of failures and errors back to 0.
+                # We set the number of requeues to the number of tests that failed, so the
+                # run statistics will reflect that we retried some failed test.
+                #
+                # However, normally requeues are not acked, as we expect the test to be acked
+                # by another worker later. This makes the test loop think iot is already done.
+                # To prevent this, we initialize the number of acks negatively, so it evens out
+                # in the statistics.
+                total_failures = previous_failures.length + previous_errors.length
+                adjust_combined_results(ResultAggregate.new(
+                  size: total_failures,
+                  failures: -previous_failures.length,
+                  errors: -previous_errors.length,
+                  requeues: total_failures,
+                ))
+
+                # For subsequent attempts, we check the list of previous failures and
+                # errors, and only schedule to re-run those tests. This allows for faster
+                # retries of potentially flaky tests.
+                test_identifiers_to_retry = T.let(previous_failures + previous_errors, T::Array[String])
+                test_identifiers_to_retry.map { |identifier| DefinedRunnable.from_identifier(identifier) }
+              end
+            else
+              adjust_combined_results(ResultAggregate.new(size: 0))
+              []
+            end,
+            T::Array[Minitest::Runnable],
+          )
 
           redis.pipelined do |pipeline|
             tests.each do |test|
@@ -242,10 +245,17 @@ module Minitest
             # To make sure we don't end up in a busy loop overwhelming Redis with commands
             # when there is no work to do, we increase the blocking time exponentially,
             # and reset it to the initial value if we processed any tests.
-            if stale_runnables.empty? && fresh_runnables.empty?
-              exponential_backoff <<= 1
+            #
+            # The backoff is capped at MAX_BACKOFF to bound how long a worker can sit
+            # inside a single XREADGROUP BLOCK call. Without a cap, after ~15 empty
+            # iterations the worker is blocked in Redis for 5+ minutes and cannot
+            # re-check `complete?` / `abort?` until the BLOCK returns, which manifests
+            # as a long post-100% teardown hang when pipelined XACKs race the progress
+            # reporter.
+            exponential_backoff = if stale_runnables.empty? && fresh_runnables.empty?
+              next_backoff(exponential_backoff)
             else
-              exponential_backoff = INITIAL_BACKOFF
+              INITIAL_BACKOFF
             end
           end
 
@@ -268,7 +278,25 @@ module Minitest
 
         sig { returns(Redis) }
         def redis
-          @redis ||= Redis.new(url: configuration.coordinator_uri.to_s)
+          @redis ||= Redis.new(
+            url: configuration.coordinator_uri.to_s,
+            middlewares: custom_middlewares,
+            custom: custom_config,
+            timeout: 2,
+          )
+        end
+
+        sig { returns(T.nilable(T::Array[Module])) }
+        def custom_middlewares
+          return unless ENV.key?("MINITEST_DISTRIBUTED_REDIS_LOG")
+
+          require_relative "redis_instrumentation_middleware"
+          [RedisInstrumentationMiddleware]
+        end
+
+        sig { returns(T.nilable(T::Hash[Symbol, File])) }
+        def custom_config
+          { log_file: logger }.compact
         end
 
         sig { returns(String) }
@@ -517,8 +545,27 @@ module Minitest
           adjust_combined_results(batch_result_aggregate)
         end
 
+        sig { returns(T.nilable(Logger)) }
+        def logger
+          return unless (log_path = ENV["MINITEST_DISTRIBUTED_REDIS_LOG"])
+
+          @logger ||= T.let(Logger.new(log_path), T.nilable(Logger))
+        end
+
+        sig { params(backoff: Integer).returns(Integer) }
+        def next_backoff(backoff)
+          [backoff << 1, MAX_BACKOFF].min
+        end
+
         INITIAL_BACKOFF = 10 # milliseconds
         private_constant :INITIAL_BACKOFF
+
+        # Cap on the XREADGROUP BLOCK timeout used by `consume`. Reached after roughly
+        # 9 consecutive empty iterations (10 ms * 2^9 = 5120 ms). Bounds the worst-case
+        # time a worker can be unresponsive to `complete?` / `abort?` after the queue
+        # is drained.
+        MAX_BACKOFF = 5_000 # milliseconds
+        private_constant :MAX_BACKOFF
       end
     end
   end

data/lib/minitest/distributed/coordinators/redis_instrumentation_middleware.rb

@@ -0,0 +1,39 @@
+# typed: strict
+# frozen_string_literal: true
+
+require "redis"
+
+module Minitest
+  module Distributed
+    module Coordinators
+      # Redis middleware that logs all Redis commands to a file for debugging.
+      module RedisInstrumentationMiddleware
+        extend T::Sig
+
+        sig { params(command: T::Array[T.untyped], redis_config: T.untyped).returns(T.untyped) }
+        def call(command, redis_config)
+          log_file = redis_config.custom[:log_file]
+          log_file.info("EXEC: #{command.inspect}")
+          result = super
+          log_file.info("RESULT: #{result.inspect}")
+          result
+        rescue => e
+          log_file.info("ERROR: #{e.class}")
+          Kernel.raise
+        end
+
+        sig { params(commands: T::Array[T.untyped], redis_config: T.untyped).returns(T.untyped) }
+        def call_pipelined(commands, redis_config)
+          log_file = redis_config.custom[:log_file]
+          log_file.info("EXEC PIPELINED: #{commands.inspect}")
+          result = super
+          log_file.info("RESULT PIPELINED: #{result.inspect}")
+          result
+        rescue => e
+          log_file.info("ERROR PIPELINED: #{e.class}")
+          Kernel.raise
+        end
+      end
+    end
+  end
+end

data/lib/minitest/distributed/version.rb

@@ -3,6 +3,6 @@
 
 module Minitest
   module Distributed
-    VERSION = "0.2.11"
+    VERSION = "0.2.13"
   end
 end

metadata

@@ -1,14 +1,13 @@
 --- !ruby/object:Gem::Specification
 name: minitest-distributed
 version: !ruby/object:Gem::Version
-  version: 0.2.11
+  version: 0.2.13
 platform: ruby
 authors:
 - Willem van Bergen
-autorequire:
 bindir: exe
 cert_chain: []
-date: 2024-09-16 00:00:00.000000000 Z
+date: 1980-01-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: minitest
@@ -116,6 +115,7 @@ files:
 - lib/minitest/distributed/coordinators/coordinator_interface.rb
 - lib/minitest/distributed/coordinators/memory_coordinator.rb
 - lib/minitest/distributed/coordinators/redis_coordinator.rb
+- lib/minitest/distributed/coordinators/redis_instrumentation_middleware.rb
 - lib/minitest/distributed/enqueued_runnable.rb
 - lib/minitest/distributed/filters/exclude_file_filter.rb
 - lib/minitest/distributed/filters/exclude_filter.rb
@@ -169,7 +169,6 @@ metadata:
   allowed_push_host: https://rubygems.org
   homepage_uri: https://github.com/Shopify/minitest-distributed
   source_code_uri: https://github.com/Shopify/minitest-distributed
-post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -184,8 +183,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.5.18
-signing_key:
+rubygems_version: 4.0.12
 specification_version: 4
 summary: Distributed test executor plugin for Minitest
 test_files: []