minitest-distributed 0.1.0

data/lib/minitest/distributed/test_runner.rb

@@ -0,0 +1,37 @@
+# typed: strict
+# frozen_string_literal: true
+
+module Minitest
+  module Distributed
+    class TestRunner
+      extend T::Sig
+
+      sig { returns(T::Hash[Symbol, T.untyped]) }
+      attr_reader :options
+
+      sig { returns(Configuration) }
+      attr_reader :configuration
+
+      sig { returns(TestSelector) }
+      attr_reader :test_selector
+
+      sig { returns(Coordinators::CoordinatorInterface) }
+      attr_reader :coordinator
+
+      sig { params(options: T::Hash[Symbol, T.untyped]).void }
+      def initialize(options)
+        @options = options
+
+        @configuration = T.let(@options[:distributed], Configuration)
+        @coordinator = T.let(configuration.coordinator, Coordinators::CoordinatorInterface)
+        @test_selector = T.let(TestSelector.new(options), TestSelector)
+      end
+
+      sig { params(reporter: AbstractReporter).void }
+      def run(reporter)
+        coordinator.produce(test_selector: test_selector)
+        coordinator.consume(reporter: reporter)
+      end
+    end
+  end
+end

data/lib/minitest/distributed/test_selector.rb

@@ -0,0 +1,54 @@
+# typed: strict
+# frozen_string_literal: true
+
+module Minitest
+  module Distributed
+    class TestSelector
+      extend T::Sig
+
+      sig { returns(T::Hash[Symbol, T.untyped]) }
+      attr_reader :options
+
+      sig { returns(T::Array[Filters::FilterInterface]) }
+      attr_reader :filters
+
+      sig { params(options: T::Hash[Symbol, T.untyped]).void }
+      def initialize(options)
+        @options = options
+
+        @filters = T.let([], T::Array[Filters::FilterInterface])
+        initialize_filters
+      end
+
+      sig { void }
+      def initialize_filters
+        @filters << Filters::IncludeFilter.new(options[:filter]) if options[:filter]
+        @filters << Filters::ExcludeFilter.new(options[:exclude]) if options[:exclude]
+      end
+
+      sig { returns(T::Array[EnqueuedRunnable]) }
+      def discover_tests
+        Minitest::Runnable.runnables.flat_map do |runnable|
+          runnable.runnable_methods.map do |method_name|
+            EnqueuedRunnable.new(class_name: runnable.name, method_name: method_name)
+          end
+        end
+      end
+
+      sig { params(tests: T::Array[EnqueuedRunnable]).returns(T::Array[EnqueuedRunnable]) }
+      def select_tests(tests)
+        return tests if filters.empty?
+        tests.flat_map do |runnable_method|
+          filters.flat_map do |filter|
+            filter.call(runnable_method)
+          end
+        end.compact
+      end
+
+      sig { returns(T::Array[EnqueuedRunnable]) }
+      def tests
+        select_tests(discover_tests)
+      end
+    end
+  end
+end

data/lib/minitest/distributed/version.rb

@@ -0,0 +1,8 @@
+# typed: strict
+# frozen_string_literal: true
+
+module Minitest
+  module Distributed
+    VERSION = "0.1.0"
+  end
+end

data/lib/minitest/distributed_plugin.rb

@@ -0,0 +1,51 @@
+# typed: true
+# frozen_string_literal: true
+
+require_relative './distributed'
+
+module Minitest
+  class << self
+    extend T::Sig
+
+    def plugin_distributed_options(opts, options)
+      options[:distributed] = Minitest::Distributed::Configuration.from_env
+
+      opts.on('--coordinator=URI', "The URI pointing to the coordinator") do |uri|
+        options[:distributed].coordinator_uri = URI.parse(uri)
+      end
+
+      opts.on('--test-timeout=TIMEOUT', "The maximum run time for a single test in seconds") do |timeout|
+        options[:distributed].test_timeout = Integer(timeout)
+      end
+
+      opts.on('--max-attempts=ATTEMPTS', "The maximum number of attempts to run a test") do |attempts|
+        options[:distributed].max_attempts = Integer(attempts)
+      end
+
+      opts.on('--test-batch-size=NUMBER', "The number of tests to process per batch") do |batch_size|
+        options[:distributed].test_batch_size = Integer(batch_size)
+      end
+
+      opts.on('--run-id=ID', "The ID for this run shared between coordinated workers") do |id|
+        options[:distributed].run_id = id
+      end
+
+      opts.on('--worker-id=ID', "The unique ID for this worker") do |id|
+        options[:distributed].worker_id = id
+      end
+    end
+
+    def plugin_distributed_init(options)
+      Minitest.singleton_class.prepend(Minitest::Distributed::TestRunnerPatch)
+      options[:distributed].coordinator.register_reporters(reporter: reporter, options: options)
+
+      if reporter.reporters.reject! { |reporter| reporter.is_a?(Minitest::ProgressReporter) }
+        reporter << Minitest::Distributed::Reporters::DistributedPogressReporter.new(options[:io], options)
+      end
+
+      if reporter.reporters.reject! { |reporter| reporter.is_a?(Minitest::SummaryReporter) }
+        reporter << Minitest::Distributed::Reporters::DistributedSummaryReporter.new(options[:io], options)
+      end
+    end
+  end
+end

data/minitest-distributed.gemspec

@@ -0,0 +1,50 @@
+require_relative 'lib/minitest/distributed/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "minitest-distributed"
+  spec.version       = Minitest::Distributed::VERSION
+  spec.authors       = ["Willem van Bergen"]
+  spec.email         = ["willem@vanbergen.org"]
+
+  spec.summary       = "Distributed test executor plugin for Minitest"
+  spec.description   = <<~EOD
+    minitest-distributed is a plugin for minitest for executing tests on a
+    distributed set of unreliable workers.
+
+    When a test suite grows large enough, it inevitable gets too slow to run
+    on a single machine to give timely feedback to developers. This plugins
+    combats this issue by distributing the full test suite to a set of workers.
+    Every worker is a consuming from a single queue, so the tests get evenly
+    distributed and all workers will finish around the same time. Redis is used
+    as coordinator, but when using this plugin without having access to Redis,
+    it will use an in-memory coordinator.
+
+    Using multiple (virtual) machines for a test run is an (additional) source
+    of flakiness. To combat flakiness, minitest-distributed implements resiliency
+    patterns, like re-running a test on a different worker on failure, and
+    a circuit breaker for misbehaving workers.
+  EOD
+
+  spec.homepage      = "https://github.com/Shopify/minitest-distributed"
+  spec.license       = "MIT"
+  spec.required_ruby_version = Gem::Requirement.new(">= 2.3.0")
+
+  spec.metadata["allowed_push_host"] = "https://rubygems.org"
+
+  spec.metadata["homepage_uri"] = spec.homepage
+  spec.metadata["source_code_uri"] = "https://github.com/Shopify/minitest-distributed"
+  # spec.metadata["changelog_uri"] = "TODO: Put your gem's CHANGELOG.md URL here."
+
+  # Specify which files should be added to the gem when it is released.
+  # The `git ls-files -z` loads the files in the RubyGem that have been added into git.
+  spec.files         = Dir.chdir(File.expand_path('..', __FILE__)) do
+    `git ls-files -z`.split("\x0").reject { |f| f.match(%r{^(test|spec|features)/}) }
+  end
+  spec.bindir        = "exe"
+  spec.executables   = spec.files.grep(%r{^exe/}) { |f| File.basename(f) }
+  spec.require_paths = ["lib"]
+
+  spec.add_dependency('minitest', '~> 5.12')
+  spec.add_dependency('redis', '~> 4.1')
+  spec.add_dependency('sorbet-runtime')
+end

data/sorbet/config

@@ -0,0 +1,2 @@
+--dir
+.

data/sorbet/rbi/minitest.rbi

@@ -0,0 +1,238 @@
+# This file is autogenerated. Do not edit it by hand. Regenerate it with:
+#   srb rbi sorbet-typed
+#
+# If you would like to make changes to this file, great! Please upstream any changes you make here:
+#
+#   https://github.com/sorbet/sorbet-typed/edit/master/lib/minitest/all/minitest.rbi
+#
+# typed: strong
+
+module Minitest
+  class Runnable
+    def self.run_one_method(klass, method_name, reporter); end
+    def self.runnables; end
+
+    def name; end
+    def time; end
+    def time=(duration); end
+    def failures; end
+
+    def initialize(method_name); end
+  end
+
+  class Test < Runnable
+    include Minitest::Assertions
+  end
+
+  class Result < Runnable
+    sig { returns(String) }
+    def name; end
+
+    sig { returns(String) }
+    def klass; end
+
+    sig { returns(String) }
+    def class_name; end
+
+    sig { returns(T.nilable(Minitest::Assertion)) }
+    def failure; end
+
+    sig { returns(T::Boolean) }
+    def error?; end
+
+    sig { returns(T::Boolean) }
+    def skipped?; end
+
+    sig { returns(T::Boolean) }
+    def passed?; end
+
+    sig { returns(Integer) }
+    def assertions; end
+
+    sig { params(runnable: Runnable).returns(T.attached_class) }
+    def self.from(runnable); end
+  end
+
+  class Assertion < Exception
+    sig { returns(String) }
+    def result_label; end
+
+    sig { returns(String) }
+    def result_code; end
+  end
+
+  class Skip < Exception
+  end
+
+  class UnexpectedError < Assertion
+  end
+
+  class AbstractReporter
+    sig { void }
+    def start; end
+
+    sig { params(runnable: T.class_of(Runnable), method_name: String).void }
+    def prerecord(runnable, method_name); end
+
+    sig { params(result: Minitest::Result).void }
+    def record(result); end
+
+    sig { void }
+    def report; end
+
+    sig { returns(T::Boolean) }
+    def passed?; end
+  end
+
+  class Reporter < AbstractReporter
+    sig { params(io: IO, options: T::Hash[Symbol, T.untyped]).void }
+    def initialize(io, options); end
+
+    sig { returns(T::Hash[Symbol, T.untyped]) }
+    def options; end
+
+    sig { returns(IO) }
+    def io; end
+  end
+
+  class StatisticsReporter < Reporter
+  end
+
+  class SummaryReporter < StatisticsReporter
+  end
+
+  class ProgressReporter < Reporter
+  end
+
+  class CompositeReporter < AbstractReporter
+    sig { params(reporter: AbstractReporter).returns(T.self_type) }
+    def <<(reporter); end
+
+    sig { returns(T::Array[AbstractReporter]) }
+    def reporters; end
+  end
+
+  sig { returns(CompositeReporter) }
+  def self.reporter; end
+
+  sig { void }
+  def self.autorun; end
+
+  sig { params(args: T::Array[String]).returns(T::Boolean) }
+  def self.run(args = []); end
+
+  sig { params(klass: T.class_of(Runnable), method_name: String).returns(Minitest::Result) }
+  def self.run_one_method(klass, method_name); end
+
+  sig { returns(Float) }
+  def self.clock_time; end
+end
+
+module Minitest::Assertions
+  extend T::Sig
+
+  sig { params(msg: T.nilable(String)).returns(TrueClass) }
+  def pass(msg = nil); end
+
+  sig { params(msg: T.nilable(String)).returns(FalseClass) }
+  def flunk(msg = nil); end
+
+  sig { params(test: T.untyped, msg: T.nilable(String)).returns(TrueClass) }
+  def assert(test, msg = nil); end
+
+  sig do
+    params(
+      exp: BasicObject,
+      msg: T.nilable(String)
+    ).returns(TrueClass)
+  end
+  def assert_empty(exp, msg = nil); end
+
+  sig do
+    params(
+      exp: BasicObject,
+      act: BasicObject,
+      msg: T.nilable(String)
+    ).returns(TrueClass)
+  end
+  def assert_equal(exp, act, msg = nil); end
+
+  sig do
+    params(
+      collection: T::Enumerable[T.untyped],
+      obj: BasicObject,
+      msg: T.nilable(String)
+    ).returns(TrueClass)
+  end
+  def assert_includes(collection, obj, msg = nil); end
+
+  sig do
+    params(
+      obj: BasicObject,
+      msg: T.nilable(String)
+    ).returns(TrueClass)
+  end
+  def assert_nil(obj, msg = nil); end
+
+  sig do
+    params(
+      exp: T.untyped
+    ).returns(TrueClass)
+  end
+  def assert_raises(*exp); end
+
+  sig do
+    params(
+      obj: BasicObject,
+      predicate: Symbol,
+      msg: T.nilable(String)
+    ).returns(TrueClass)
+  end
+  def assert_predicate(obj, predicate, msg = nil); end
+
+  sig { params(test: T.untyped, msg: T.nilable(String)).returns(TrueClass) }
+  def refute(test, msg = nil); end
+
+  sig do
+    params(
+      exp: BasicObject,
+      msg: T.nilable(String)
+    ).returns(TrueClass)
+  end
+  def refute_empty(exp, msg = nil); end
+
+  sig do
+    params(
+      exp: BasicObject,
+      act: BasicObject,
+      msg: T.nilable(String)
+    ).returns(TrueClass)
+  end
+  def refute_equal(exp, act, msg = nil); end
+
+  sig do
+    params(
+      collection: T::Enumerable[T.untyped],
+      obj: BasicObject,
+      msg: T.nilable(String)
+    ).returns(TrueClass)
+  end
+  def refute_includes(collection, obj, msg = nil); end
+
+  sig do
+    params(
+      obj: BasicObject,
+      msg: T.nilable(String)
+    ).returns(TrueClass)
+  end
+  def refute_nil(obj, msg = nil); end
+
+  sig do
+    params(
+      obj: BasicObject,
+      predicate: Symbol,
+      msg: T.nilable(String)
+    ).returns(TrueClass)
+  end
+  def refute_predicate(obj, predicate, msg = nil); end
+end