minitest-distributed 0.1.0

data/lib/minitest/distributed/enqueued_runnable.rb

@@ -0,0 +1,88 @@
+# typed: strict
+# frozen_string_literal: true
+
+module Minitest
+  module Distributed
+    class EnqueuedRunnable < T::Struct
+      class << self
+        extend T::Sig
+
+        sig { params(identifier: String).returns(T.attached_class) }
+        def from_identifier(identifier)
+          class_name, method_name = identifier.split('#', 2)
+          new(
+            class_name: T.must(class_name),
+            method_name: T.must(method_name),
+          )
+        end
+
+        sig { params(runnable: Minitest::Runnable).returns(T.attached_class) }
+        def from_runnable(runnable)
+          new(
+            class_name: T.must(runnable.class.name),
+            method_name: runnable.name,
+          )
+        end
+
+        sig { params(result: Minitest::Result).returns(T.attached_class) }
+        def from_result(result)
+          new(class_name: result.class_name, method_name: result.name)
+        end
+
+        sig { params(claims: T::Array[[String, T::Hash[String, String]]]).returns(T::Array[T.attached_class]) }
+        def from_redis_stream_claim(claims)
+          claims.map do |id, runnable_method_info|
+            new(
+              class_name: runnable_method_info.fetch('class_name'),
+              method_name: runnable_method_info.fetch('method_name'),
+              execution_id: id,
+            )
+          end
+        end
+
+        sig { params(name: String).returns(T.class_of(Minitest::Runnable)) }
+        def find_runnable_class(name)
+          name.split('::')
+            .reduce(Object) { |ns, const| ns.const_get(const) } # rubocop:disable Sorbet/ConstantsFromStrings
+        end
+      end
+
+      extend T::Sig
+
+      const :class_name, String
+      const :method_name, String
+      const :execution_id, T.nilable(String), dont_store: true
+
+      # By setting canned failure, we will not actually run the runnable,
+      # but immediately return a result with the canned assertion.x
+      prop :canned_failure, T.nilable(Minitest::Assertion), dont_store: true
+
+      sig { returns(String) }
+      def identifier
+        "#{class_name}##{method_name}"
+      end
+
+      sig { returns(T.class_of(Minitest::Runnable)) }
+      def runnable_class
+        self.class.find_runnable_class(class_name)
+      end
+
+      sig { returns(Minitest::Runnable) }
+      def runnable
+        runnable_class.new(method_name)
+      end
+
+      sig { returns(Minitest::Result) }
+      def run
+        if canned_failure
+          canned_runnable = runnable
+          canned_runnable.time = 0.0
+          canned_runnable.failures << canned_failure
+          Minitest::Result.from(canned_runnable)
+        else
+          Minitest.run_one_method(runnable_class, method_name)
+        end
+      end
+    end
+  end
+end

data/lib/minitest/distributed/filters/exclude_filter.rb

@@ -0,0 +1,35 @@
+# typed: strict
+# frozen_string_literal: true
+
+module Minitest
+  module Distributed
+    module Filters
+      class ExcludeFilter
+        extend T::Sig
+        include FilterInterface
+
+        sig { returns(T.any(String, Regexp)) }
+        attr_reader :filter
+
+        sig { params(filter: T.any(String, Regexp)).void }
+        def initialize(filter)
+          @filter = filter
+          if filter.is_a?(String) && (match_info = filter.match(%r%/(.*)/%))
+            @filter = Regexp.new(T.must(match_info[1]))
+          end
+        end
+
+        sig { override.params(enqueued_runnable: EnqueuedRunnable).returns(T::Array[EnqueuedRunnable]) }
+        def call(enqueued_runnable)
+          # rubocop:disable Style/CaseEquality
+          if filter === enqueued_runnable.method_name || filter === enqueued_runnable.identifier
+            []
+          else
+            [enqueued_runnable]
+          end
+          # rubocop:enable Style/CaseEquality
+        end
+      end
+    end
+  end
+end

data/lib/minitest/distributed/filters/filter_interface.rb

@@ -0,0 +1,25 @@
+# typed: strict
+# frozen_string_literal: true
+
+module Minitest
+  module Distributed
+    module Filters
+      # A filter proc is a callable object that changes the list of runnables that will
+      # be executed during the test run. For every runnable, it should return an
+      # array of runnables.
+      #
+      # - If it returns an empty array, the runnable will not be run.
+      # - If it returns a single elemnt array with the passed ion runnable to make no changes.
+      # - It can return an array of enumerables to expand the number of runnables in this test run,
+      #   We use this for grinding tests, for instance.
+      module FilterInterface
+        extend T::Sig
+        extend T::Helpers
+        interface!
+
+        sig { abstract.params(runnable_method: EnqueuedRunnable).returns(T::Array[EnqueuedRunnable]) }
+        def call(runnable_method); end
+      end
+    end
+  end
+end

data/lib/minitest/distributed/filters/include_filter.rb

@@ -0,0 +1,35 @@
+# typed: strict
+# frozen_string_literal: true
+
+module Minitest
+  module Distributed
+    module Filters
+      class IncludeFilter
+        extend T::Sig
+        include FilterInterface
+
+        sig { returns(T.any(String, Regexp)) }
+        attr_reader :filter
+
+        sig { params(filter: T.any(String, Regexp)).void }
+        def initialize(filter)
+          @filter = filter
+          if filter.is_a?(String) && (match_info = filter.match(%r%/(.*)/%))
+            @filter = Regexp.new(T.must(match_info[1]))
+          end
+        end
+
+        sig { override.params(enqueued_runnable: EnqueuedRunnable).returns(T::Array[EnqueuedRunnable]) }
+        def call(enqueued_runnable)
+          # rubocop:disable Style/CaseEquality
+          if filter === enqueued_runnable.method_name || filter === enqueued_runnable.identifier
+            [enqueued_runnable]
+          else
+            []
+          end
+          # rubocop:enable Style/CaseEquality
+        end
+      end
+    end
+  end
+end

data/lib/minitest/distributed/reporters/distributed_progress_reporter.rb

@@ -0,0 +1,76 @@
+# typed: strict
+# frozen_string_literal: true
+
+require 'io/console'
+
+module Minitest
+  module Distributed
+    module Reporters
+      class DistributedPogressReporter < Minitest::Reporter
+        extend T::Sig
+
+        sig { returns(Coordinators::CoordinatorInterface) }
+        attr_reader :coordinator
+
+        sig { params(io: IO, options: T::Hash[Symbol, T.untyped]).void }
+        def initialize(io, options)
+          super
+          if io.tty?
+            io.sync = true
+          end
+          @coordinator = T.let(options[:distributed].coordinator, Coordinators::CoordinatorInterface)
+        end
+
+        # Note: due to batching and parallel tests, we have no guarantee that `prerecord`
+        # and `record` will be called in succession for the same test without calls to
+        # either method being interjected for other tests.
+        #
+        # As a result we have no idea what will be on the last line of the console.
+        # We always clear the full line before printing output.
+
+        sig { override.params(klass: T.class_of(Runnable), name: String).void }
+        def prerecord(klass, name)
+          if io.tty?
+            line_width = clear_current_line
+            io.print("[#{results.acks}/#{results.size}] #{klass}##{name}".slice(0...line_width))
+          end
+        end
+
+        sig { override.params(result: Minitest::Result).void }
+        def record(result)
+          clear_current_line if io.tty?
+
+          case (result_type = ResultType.of(result))
+          when ResultType::Passed
+            # TODO: warn for tests that are slower than the test timeout.
+          when ResultType::Skipped
+            io.puts("#{result}\n") if options[:verbose]
+          when ResultType::Error, ResultType::Failed
+            io.puts("#{result}\n")
+          else
+            T.absurd(result_type)
+          end
+        end
+
+        sig { override.void }
+        def report
+          clear_current_line if io.tty?
+        end
+
+        private
+
+        sig { returns(Integer) }
+        def clear_current_line
+          _height, width = IO.console.winsize
+          io.print("\r" + (' ' * width) + "\r")
+          width
+        end
+
+        sig { returns(ResultAggregate) }
+        def results
+          coordinator.combined_results
+        end
+      end
+    end
+  end
+end

data/lib/minitest/distributed/reporters/distributed_summary_reporter.rb

@@ -0,0 +1,48 @@
+# typed: strict
+# frozen_string_literal: true
+
+module Minitest
+  module Distributed
+    module Reporters
+      class DistributedSummaryReporter < Minitest::Reporter
+        extend T::Sig
+
+        sig { returns(Coordinators::CoordinatorInterface) }
+        attr_reader :coordinator
+
+        sig { params(io: IO, options: T::Hash[Symbol, T.untyped]).void }
+        def initialize(io, options)
+          super
+          io.sync = true
+          @coordinator = T.let(options[:distributed].coordinator, Coordinators::CoordinatorInterface)
+          @start_time = T.let(0.0, Float)
+        end
+
+        sig { override.void }
+        def start
+          @start_time = Minitest.clock_time
+          io.puts("Run options: #{options[:args]}\n\n")
+        end
+
+        sig { override.void }
+        def report
+          duration = format("(in %0.3fs)", Minitest.clock_time - @start_time)
+
+          local_results = coordinator.local_results
+          combined_results = coordinator.combined_results
+          if combined_results == local_results
+            io.puts("Results: #{combined_results} #{duration}")
+          else
+            io.puts("This worker:      #{local_results} #{duration}")
+            io.puts("Combined results: #{combined_results}")
+          end
+        end
+
+        sig { override.returns(T::Boolean) }
+        def passed?
+          coordinator.combined_results.passed?
+        end
+      end
+    end
+  end
+end

data/lib/minitest/distributed/reporters/redis_coordinator_warnings_reporter.rb

@@ -0,0 +1,61 @@
+# typed: strict
+# frozen_string_literal: true
+
+module Minitest
+  module Distributed
+    module Reporters
+      class RedisCoordinatorWarningsReporter < Minitest::Reporter
+        extend T::Sig
+
+        sig { override.void }
+        def report
+          [reclaim_warning, missing_acks_warning].compact.each do |warning|
+            io.puts
+            io.puts(warning)
+          end
+        end
+
+        private
+
+        sig { returns(Configuration) }
+        def configuration
+          options[:distributed]
+        end
+
+        sig { returns(Coordinators::RedisCoordinator) }
+        def redis_coordinator
+          T.cast(configuration.coordinator, Coordinators::RedisCoordinator)
+        end
+
+        sig { returns(T.nilable(String)) }
+        def reclaim_warning
+          if redis_coordinator.reclaimed_tests.any?
+            <<~WARNING
+              WARNING: The following tests were reclaimed from another worker:
+              #{redis_coordinator.reclaimed_tests.map { |test| "- #{test.identifier}" }.join("\n")}
+
+              The original worker did not complete running this test in #{configuration.test_timeout}ms.
+              This either means that the worker unexpectedly went away, or that the test is too slow.
+            WARNING
+          end
+        end
+
+        sig { returns(T.nilable(String)) }
+        def missing_acks_warning
+          local_results = redis_coordinator.local_results
+          if local_results.acks < local_results.size
+            <<~WARNING
+              WARNING: This worker was not able to ack all the test it ran with the coordinator (#{local_results.acks}/#{local_results.size}).
+
+              This means that this worker took too long to report the status of one or more tests,
+              and these tests were claimed by other workers. As a result, the total number of
+              reported runs may be larger than the size of the test suite.
+
+              Make sure that all your tests complete within #{configuration.test_timeout}ms.
+            WARNING
+          end
+        end
+      end
+    end
+  end
+end

data/lib/minitest/distributed/result_aggregate.rb

@@ -0,0 +1,67 @@
+# typed: strict
+# frozen_string_literal: true
+
+module Minitest
+  module Distributed
+    class ResultAggregate < T::Struct
+      extend T::Sig
+
+      prop :runs, Integer, default: 0
+      prop :assertions, Integer, default: 0
+      prop :passes, Integer, default: 0
+      prop :failures, Integer, default: 0
+      prop :errors, Integer, default: 0
+      prop :skips, Integer, default: 0
+      prop :reruns, Integer, default: 0
+      prop :acks, Integer, default: 0
+      prop :size, Integer, default: 0
+
+      sig { params(result: Minitest::Result).void }
+      def update_with_result(result)
+        case (result_type = ResultType.of(result))
+        when ResultType::Passed then self.passes += 1
+        when ResultType::Failed then self.failures += 1
+        when ResultType::Error then self.errors += 1
+        when ResultType::Skipped then self.skips += 1
+        else T.absurd(result_type)
+        end
+
+        self.runs += 1
+        self.assertions += result.assertions
+      end
+
+      sig { returns(String) }
+      def to_s
+        str = +"#{runs} runs, #{assertions} assertions, #{passes} passes, #{failures} failures, #{errors} errors"
+        str << ", #{skips} skips" if skips > 0
+        str << ", #{reruns} re-runs" if reruns > 0
+        str
+      end
+
+      sig { returns(Integer) }
+      def unique_runs
+        runs - reruns
+      end
+
+      sig { returns(Integer) }
+      def reported_results
+        passes + failures + errors + skips
+      end
+
+      sig { returns(T::Boolean) }
+      def completed?
+        acks == size
+      end
+
+      sig { returns(T::Boolean) }
+      def valid?
+        unique_runs == reported_results
+      end
+
+      sig { returns(T::Boolean) }
+      def passed?
+        completed? && valid? && self.failures == 0 && self.errors == 0
+      end
+    end
+  end
+end

data/lib/minitest/distributed/result_type.rb

@@ -0,0 +1,28 @@
+# typed: strict
+# frozen_string_literal: true
+
+module Minitest
+  module Distributed
+    class ResultType < T::Enum
+      enums do
+        Passed = new
+        Failed = new
+        Error = new
+        Skipped = new
+      end
+
+      sig { params(result: Minitest::Result).returns(ResultType) }
+      def self.of(result)
+        if result.passed?
+          Passed
+        elsif result.error?
+          Error
+        elsif result.skipped?
+          Skipped
+        else
+          Failed
+        end
+      end
+    end
+  end
+end