minitest-distributed 0.1.2 → 0.2.0

data/lib/minitest/distributed/enqueued_runnable.rb

@@ -3,85 +3,237 @@
 
 module Minitest
   module Distributed
+    class PendingExecution < T::Struct
+      extend T::Sig
+
+      const :worker_id, String
+      const :entry_id, String
+      const :elapsed_time_ms, Integer
+      const :attempt, Integer
+
+      sig { returns(String) }
+      def attempt_id
+        "#{entry_id}/#{attempt}"
+      end
+
+      sig { params(xpending_result: T::Hash[String, T.untyped]).returns(T.attached_class) }
+      def self.from_xpending(xpending_result)
+        new(
+          worker_id: xpending_result.fetch('consumer'),
+          entry_id: xpending_result.fetch('entry_id'),
+          elapsed_time_ms: xpending_result.fetch('elapsed'),
+          attempt: xpending_result.fetch('count'),
+        )
+      end
+    end
+
+    # This module defines some helper methods to deal with Minitest::Runnable
+    module DefinedRunnable
+      extend T::Sig
+
+      sig { params(name: String).returns(T.class_of(Minitest::Runnable)) }
+      def self.find_class(name)
+        name.split('::')
+          .reduce(Object) { |ns, const| ns.const_get(const) } # rubocop:disable Sorbet/ConstantsFromStrings
+      end
+
+      sig { params(runnable: Minitest::Runnable).returns(String) }
+      def self.identifier(runnable)
+        "#{T.must(runnable.class.name)}##{runnable.name}"
+      end
+
+      sig { params(identifier: String).returns(Minitest::Runnable) }
+      def self.from_identifier(identifier)
+        class_name, method_name = identifier.split('#', 2)
+        find_class(T.must(class_name)).new(T.must(method_name))
+      end
+    end
+
     class EnqueuedRunnable < T::Struct
-      class << self
+      class Result < T::Struct
+        class Commit
+          extend T::Sig
+
+          sig { params(block: T.proc.returns(T::Boolean)).void }
+          def initialize(&block)
+            @block = block
+          end
+
+          sig { returns(T::Boolean) }
+          def success?
+            @success = T.let(@success, T.nilable(T::Boolean))
+            @success ||= @block.call
+          end
+
+          sig { returns(T::Boolean) }
+          def failure?
+            !success?
+          end
+
+          sig { returns(Commit) }
+          def self.success
+            @success = T.let(@success, T.nilable(Commit))
+            @success ||= new { true }
+          end
+
+          sig { returns(Commit) }
+          def self.failure
+            @failure = T.let(@failure, T.nilable(Commit))
+            @failure ||= new { false }
+          end
+        end
+
         extend T::Sig
 
-        sig { params(identifier: String).returns(T.attached_class) }
-        def from_identifier(identifier)
-          class_name, method_name = identifier.split('#', 2)
-          new(
-            class_name: T.must(class_name),
-            method_name: T.must(method_name),
-          )
+        const :enqueued_runnable, EnqueuedRunnable
+        const :initial_result, Minitest::Result
+        const :commit, Commit
+
+        sig { returns(String) }
+        def entry_id
+          enqueued_runnable.entry_id
+        end
+
+        sig { returns(T::Boolean) }
+        def final?
+          !requeue?
         end
 
-        sig { params(runnable: Minitest::Runnable).returns(T.attached_class) }
-        def from_runnable(runnable)
-          new(
-            class_name: T.must(runnable.class.name),
-            method_name: runnable.name,
-          )
+        sig { returns(T::Boolean) }
+        def requeue?
+          ResultType.of(initial_result) == ResultType::Requeued
         end
 
-        sig { params(result: Minitest::Result).returns(T.attached_class) }
-        def from_result(result)
-          new(class_name: result.class_name, method_name: result.name)
+        sig { returns(Minitest::Result) }
+        def committed_result
+          @committed_result = T.let(@committed_result, T.nilable(Minitest::Result))
+          @committed_result ||= if final? && commit.failure?
+            # If a runnable result is final, but the acked failed, we will discard the result.
+            Minitest::Discard.wrap(initial_result, test_timeout_seconds: enqueued_runnable.test_timeout_seconds)
+          else
+            initial_result
+          end
         end
+      end
+
+      class << self
+        extend T::Sig
+
+        sig do
+          params(
+            claims: T::Array[[String, T::Hash[String, String]]],
+            pending_messages: T::Hash[String, PendingExecution],
+            configuration: Configuration,
+          ).returns(T::Array[T.attached_class])
+        end
+        def from_redis_stream_claim(claims, pending_messages = {}, configuration:)
+          claims.map do |entry_id, runnable_method_info|
+            # `attempt` will be set to the current attempt of a different worker that has timed out.
+            # The attempt we are going to try will be the next one, so add one.
+            attempt = pending_messages.key?(entry_id) ? pending_messages.fetch(entry_id).attempt + 1 : 1
 
-        sig { params(claims: T::Array[[String, T::Hash[String, String]]]).returns(T::Array[T.attached_class]) }
-        def from_redis_stream_claim(claims)
-          claims.map do |id, runnable_method_info|
             new(
               class_name: runnable_method_info.fetch('class_name'),
               method_name: runnable_method_info.fetch('method_name'),
-              execution_id: id,
+              entry_id: entry_id,
+              attempt: attempt,
+              max_attempts: configuration.max_attempts,
+              test_timeout_seconds: configuration.test_timeout_seconds,
             )
           end
         end
-
-        sig { params(name: String).returns(T.class_of(Minitest::Runnable)) }
-        def find_runnable_class(name)
-          name.split('::')
-            .reduce(Object) { |ns, const| ns.const_get(const) } # rubocop:disable Sorbet/ConstantsFromStrings
-        end
       end
 
       extend T::Sig
 
       const :class_name, String
       const :method_name, String
-      const :execution_id, T.nilable(String), dont_store: true
-
-      # By setting canned failure, we will not actually run the runnable,
-      # but immediately return a result with the canned assertion.x
-      prop :canned_failure, T.nilable(Minitest::Assertion), dont_store: true
+      const :entry_id, String, factory: -> { SecureRandom.uuid }, dont_store: true
+      const :attempt, Integer, default: 1, dont_store: true
+      const :max_attempts, Integer, dont_store: true
+      const :test_timeout_seconds, Float, dont_store: true
 
       sig { returns(String) }
       def identifier
         "#{class_name}##{method_name}"
       end
 
+      sig { returns(String) }
+      def attempt_id
+        "#{entry_id}/#{attempt}"
+      end
+
       sig { returns(T.class_of(Minitest::Runnable)) }
       def runnable_class
-        self.class.find_runnable_class(class_name)
+        DefinedRunnable.find_class(class_name)
       end
 
       sig { returns(Minitest::Runnable) }
-      def runnable
+      def instantiate_runnable
         runnable_class.new(method_name)
       end
 
+      sig { returns(T::Boolean) }
+      def attempts_exhausted?
+        attempt > max_attempts
+      end
+
+      sig { returns(T::Boolean) }
+      def final_attempt?
+        attempt == max_attempts
+      end
+
       sig { returns(Minitest::Result) }
-      def run
-        if canned_failure
-          canned_runnable = runnable
-          canned_runnable.time = 0.0
-          canned_runnable.failures << canned_failure
-          Minitest::Result.from(canned_runnable)
+      def attempts_exhausted_result
+        assertion = Minitest::AttemptsExhausted.new(<<~EOM.chomp)
+          This test takes too long to run (> #{test_timeout_seconds}s).
+
+          We have tried running this test #{max_attempts} on different workers, but every time the worker has not reported back a result within #{test_timeout_seconds}s.
+          Try to make the test faster, or increase the test timeout.
+        EOM
+        assertion.set_backtrace(caller)
+
+        runnable = instantiate_runnable
+        runnable.time = 0.0
+        runnable.failures = [assertion]
+
+        Minitest::Result.from(runnable)
+      end
+
+      sig do
+        params(
+          block: T.proc.params(arg0: Minitest::Result).returns(EnqueuedRunnable::Result::Commit)
+        ).returns(EnqueuedRunnable::Result)
+      end
+      def run(&block)
+        initial_result = if attempts_exhausted?
+          attempts_exhausted_result
         else
-          Minitest.run_one_method(runnable_class, method_name)
+          result = Minitest.run_one_method(runnable_class, method_name)
+          result_type = ResultType.of(result)
+          if (result_type == ResultType::Error || result_type == ResultType::Failed) && !final_attempt?
+            result = Minitest::Requeue.wrap(result, attempt: attempt, max_attempts: max_attempts)
+          end
+          result
         end
+
+        EnqueuedRunnable::Result.new(
+          enqueued_runnable: self,
+          initial_result: initial_result,
+          commit: block.call(initial_result),
+        )
+      end
+
+      sig { returns(T.self_type) }
+      def next_attempt
+        self.class.new(
+          class_name: class_name,
+          method_name: method_name,
+          entry_id: entry_id,
+          attempt: attempt + 1,
+          max_attempts: max_attempts,
+          test_timeout_seconds: test_timeout_seconds,
+        )
       end
     end
   end

data/lib/minitest/distributed/filters/exclude_filter.rb

@@ -19,13 +19,13 @@ module Minitest
           end
         end
 
-        sig { override.params(enqueued_runnable: EnqueuedRunnable).returns(T::Array[EnqueuedRunnable]) }
-        def call(enqueued_runnable)
+        sig { override.params(runnable: Minitest::Runnable).returns(T::Array[Runnable]) }
+        def call(runnable)
           # rubocop:disable Style/CaseEquality
-          if filter === enqueued_runnable.method_name || filter === enqueued_runnable.identifier
+          if filter === runnable.name || filter === DefinedRunnable.identifier(runnable)
             []
           else
-            [enqueued_runnable]
+            [runnable]
           end
           # rubocop:enable Style/CaseEquality
         end

data/lib/minitest/distributed/filters/filter_interface.rb

@@ -9,7 +9,7 @@ module Minitest
       # array of runnables.
       #
       # - If it returns an empty array, the runnable will not be run.
-      # - If it returns a single elemnt array with the passed ion runnable to make no changes.
+      # - If it returns a single element array with the passed ion runnable to make no changes.
       # - It can return an array of enumerables to expand the number of runnables in this test run,
       #   We use this for grinding tests, for instance.
       module FilterInterface
@@ -17,8 +17,8 @@ module Minitest
         extend T::Helpers
         interface!
 
-        sig { abstract.params(runnable_method: EnqueuedRunnable).returns(T::Array[EnqueuedRunnable]) }
-        def call(runnable_method); end
+        sig { abstract.params(runnable: Minitest::Runnable).returns(T::Array[Minitest::Runnable]) }
+        def call(runnable); end
       end
     end
   end

data/lib/minitest/distributed/filters/include_filter.rb

@@ -19,11 +19,11 @@ module Minitest
           end
         end
 
-        sig { override.params(enqueued_runnable: EnqueuedRunnable).returns(T::Array[EnqueuedRunnable]) }
-        def call(enqueued_runnable)
+        sig { override.params(runnable: Minitest::Runnable).returns(T::Array[Minitest::Runnable]) }
+        def call(runnable)
           # rubocop:disable Style/CaseEquality
-          if filter === enqueued_runnable.method_name || filter === enqueued_runnable.identifier
-            [enqueued_runnable]
+          if filter === runnable.name || filter === DefinedRunnable.identifier(runnable)
+            [runnable]
           else
             []
           end

data/lib/minitest/distributed/reporters/distributed_progress_reporter.rb

@@ -50,9 +50,9 @@ module Minitest
           case (result_type = ResultType.of(result))
           when ResultType::Passed
             # TODO: warn for tests that are slower than the test timeout.
-          when ResultType::Skipped
+          when ResultType::Skipped, ResultType::Discarded
             io.puts("#{result}\n") if options[:verbose]
-          when ResultType::Error, ResultType::Failed
+          when ResultType::Error, ResultType::Failed, ResultType::Requeued
             io.puts("#{result}\n")
           else
             T.absurd(result_type)

data/lib/minitest/distributed/reporters/distributed_summary_reporter.rb

@@ -7,14 +7,10 @@ module Minitest
       class DistributedSummaryReporter < Minitest::Reporter
         extend T::Sig
 
-        sig { returns(Coordinators::CoordinatorInterface) }
-        attr_reader :coordinator
-
         sig { params(io: IO, options: T::Hash[Symbol, T.untyped]).void }
         def initialize(io, options)
           super
           io.sync = true
-          @coordinator = T.let(options[:distributed].coordinator, Coordinators::CoordinatorInterface)
           @start_time = T.let(0.0, Float)
         end
 
@@ -26,21 +22,64 @@ module Minitest
 
         sig { override.void }
         def report
-          duration = format("(in %0.3fs)", Minitest.clock_time - @start_time)
+          print_discard_warning if local_results.discards > 0
+
+          if configuration.coordinator.aborted?
+            io.puts("Cannot retry a run that was cut short during the previous attempt.")
+            io.puts
+          elsif combined_results.abort?
+            io.puts("The run was cut short after reaching the limit of #{configuration.max_failures} test failures.")
+            io.puts
+          end
 
-          local_results = coordinator.local_results
-          combined_results = coordinator.combined_results
+          formatted_duration = format("(in %0.3fs)", Minitest.clock_time - @start_time)
           if combined_results == local_results
-            io.puts("Results: #{combined_results} #{duration}")
+            io.puts("Results: #{combined_results} #{formatted_duration}")
           else
-            io.puts("This worker:      #{local_results} #{duration}")
+            io.puts("This worker:      #{local_results} #{formatted_duration}")
             io.puts("Combined results: #{combined_results}")
           end
         end
 
         sig { override.returns(T::Boolean) }
         def passed?
-          coordinator.combined_results.passed?
+          return false if configuration.coordinator.aborted?
+
+          # Generally, we want the workers to fail that had at least one failed or errored
+          # test. We have to trust that another worker will fail (and fail the build) if it
+          # encountered a failed test. We trust that the other worker will do this correctly,
+          # but we do verify that the statistics for the complete run are valid,
+          # to have some protection against unknown edge cases and bugs.
+          local_results.passed? && combined_results.valid?
+        end
+
+        protected
+
+        sig { void }
+        def print_discard_warning
+          io.puts(<<~WARNING)
+            WARNING: This worker was not able to ack all the tests it ran with the coordinator,
+            and had to discard the results of those tests. This means that some of your tests may
+            take too long to run. Make sure that all your tests complete well within #{configuration.test_timeout_seconds}s.
+
+          WARNING
+        end
+
+        sig { returns(ResultAggregate) }
+        def local_results
+          @local_results = T.let(@local_results, T.nilable(ResultAggregate))
+          @local_results ||= configuration.coordinator.local_results
+        end
+
+        sig { returns(ResultAggregate) }
+        def combined_results
+          @combined_results = T.let(@combined_results, T.nilable(ResultAggregate))
+          @combined_results ||= configuration.coordinator.combined_results
+        end
+
+        sig { returns(Configuration) }
+        def configuration
+          T.let(options[:distributed], Configuration)
         end
       end
     end

data/lib/minitest/distributed/reporters/redis_coordinator_warnings_reporter.rb

@@ -9,9 +9,10 @@ module Minitest
 
         sig { override.void }
         def report
-          [reclaim_warning, missing_acks_warning].compact.each do |warning|
-            io.puts
+          warnings = [reclaim_timeout_warning, reclaim_failed_warning].compact
+          warnings.each do |warning|
             io.puts(warning)
+            io.puts
           end
         end
 
@@ -28,30 +29,24 @@ module Minitest
         end
 
         sig { returns(T.nilable(String)) }
-        def reclaim_warning
-          if redis_coordinator.reclaimed_tests.any?
+        def reclaim_timeout_warning
+          if redis_coordinator.reclaimed_timeout_tests.any?
             <<~WARNING
               WARNING: The following tests were reclaimed from another worker:
-              #{redis_coordinator.reclaimed_tests.map { |test| "- #{test.identifier}" }.join("\n")}
+              #{redis_coordinator.reclaimed_timeout_tests.map { |test| "- #{test.identifier}" }.join("\n")}
 
-              The original worker did not complete running this test in #{configuration.test_timeout}ms.
+              The original worker did not complete running these tests in #{configuration.test_timeout_seconds}s.
               This either means that the worker unexpectedly went away, or that the test is too slow.
             WARNING
           end
         end
 
         sig { returns(T.nilable(String)) }
-        def missing_acks_warning
-          local_results = redis_coordinator.local_results
-          if local_results.acks < local_results.size
+        def reclaim_failed_warning
+          if redis_coordinator.reclaimed_failed_tests.any?
             <<~WARNING
-              WARNING: This worker was not able to ack all the test it ran with the coordinator (#{local_results.acks}/#{local_results.size}).
-
-              This means that this worker took too long to report the status of one or more tests,
-              and these tests were claimed by other workers. As a result, the total number of
-              reported runs may be larger than the size of the test suite.
-
-              Make sure that all your tests complete within #{configuration.test_timeout}ms.
+              WARNING: The following tests were reclaimed from another worker because they failed:
+              #{redis_coordinator.reclaimed_failed_tests.map { |test| "- #{test.identifier}" }.join("\n")}
             WARNING
           end
         end