minitest-distributed 0.1.2 → 0.2.2

data/lib/minitest/distributed/reporters/junitxml_reporter.rb

@@ -0,0 +1,150 @@
+# typed: strict
+# frozen_string_literal: true
+
+require 'rexml/document'
+require 'fileutils'
+
+module Minitest
+  module Distributed
+    module Reporters
+      # Reporter that generates a JUnit XML report of the results it is presented.
+      #
+      # The JUnitXML schema is not very well standardized, and many implementations deviate
+      # from the schema (see https://www.ibm.com/support/knowledgecenter/SSQ2R2_14.2.0/com.ibm.rsar.analysis.codereview.cobol.doc/topics/cac_useresults_junit.html).
+      #
+      # This JunitXML importer embraces this flexibility, and extends the format with some additional
+      # information that we can use to create more meaningful annotations. For instance, the information
+      # can be use to set annotations on your build system or for annotations using the GitHub checks API.
+      #
+      # For the implementation, we use REXML to prevent the need of additional dependencies on this gem.
+      # We also use XML 1.1, which allows more characters to be valid. We are primarily interested in
+      # this so \e is an allowed character, which is used for ANSI color coding.
+      class JUnitXMLReporter < Minitest::Reporter
+        extend T::Sig
+
+        sig { returns(T::Hash[String, T::Array[Minitest::Result]]) }
+        attr_reader :results
+
+        sig { params(io: IO, options: T::Hash[Symbol, T.untyped]).void }
+        def initialize(io, options)
+          @io = io
+          @report_path = T.let(options.fetch(:junitxml), String)
+          @results = T.let(Hash.new { |hash, key| hash[key] = [] }, T::Hash[String, T::Array[Minitest::Result]])
+        end
+
+        sig { override.params(result: Minitest::Result).void }
+        def record(result)
+          case (result_type = ResultType.of(result))
+          when ResultType::Passed, ResultType::Failed, ResultType::Error
+            T.must(results[result.klass]) << result
+          when ResultType::Skipped, ResultType::Requeued, ResultType::Discarded
+            # We will not include skipped, requeued, and discarded tests in JUnitXML reports,
+            # because they will not fail builds, but also didn't pass.
+          else
+            T.absurd(result_type)
+          end
+        end
+
+        sig { override.void }
+        def report
+          FileUtils.mkdir_p(File.dirname(@report_path))
+          File.open(@report_path, 'w+') do |file|
+            format_document(generate_document, file)
+          end
+        end
+
+        sig { returns(REXML::Document) }
+        def generate_document
+          doc = REXML::Document.new(nil, prologue_quote: :quote, attribute_quote: :quote)
+          doc << REXML::XMLDecl.new('1.1', 'utf-8')
+
+          testsuites = doc.add_element('testsuites')
+          results.each do |suite, tests|
+            add_tests_to(testsuites, suite, tests)
+          end
+          doc
+        end
+
+        sig { params(doc: REXML::Document, io: IO).void }
+        def format_document(doc, io)
+          formatter = REXML::Formatters::Pretty.new
+          formatter.write(doc, io)
+          io << "\n"
+        end
+
+        private
+
+        sig { params(testsuites: REXML::Element, suite: String, results: T::Array[Minitest::Result]).void }
+        def add_tests_to(testsuites, suite, results)
+          # TODO: make path relative to project root
+          relative_path = T.must(results.first).source_location.first
+          lineno = T.must(results.first).source_location.last
+
+          testsuite = testsuites.add_element(
+            'testsuite',
+            { 'name' => suite, 'filepath' => relative_path }.merge(aggregate_suite_results(results))
+          )
+
+          results.each do |test|
+            attributes = {
+              'name' => test.name,
+              'classname' => suite,
+              'assertions' => test.assertions,
+              'time' => test.time,
+              # 'run-command' => ... # TODO
+            }
+            attributes['lineno'] = lineno if lineno != -1
+
+            testcase_tag = testsuite.add_element('testcase', attributes)
+            add_failure_tag_if_needed(testcase_tag, test)
+          end
+        end
+
+        sig { params(testcase: REXML::Element, result: Minitest::Result).void }
+        def add_failure_tag_if_needed(testcase, result)
+          case (result_type = ResultType.of(result))
+          when ResultType::Passed, ResultType::Skipped, ResultType::Requeued, ResultType::Discarded
+            # noop
+          when ResultType::Error, ResultType::Failed
+            failure = T.must(result.failure)
+            failure_tag = testcase.add_element('failure',
+              'type' => result_type.serialize,
+              'message' => truncate_message(failure.message))
+            failure_tag.add_text(REXML::CData.new(result.to_s))
+          else
+            T.absurd(result_type)
+          end
+        end
+
+        sig { params(message: String).returns(String) }
+        def truncate_message(message)
+          T.must(message.lines.first).chomp.gsub(/\e\[[^m]+m/, '')
+        end
+
+        sig { params(results: T::Array[Minitest::Result]).returns(T::Hash[String, Numeric]) }
+        def aggregate_suite_results(results)
+          aggregate = Hash.new(0)
+          results.each do |result|
+            aggregate['assertions'] += result.assertions
+            aggregate['failures'] += 1 if failure?(ResultType.of(result))
+            aggregate['tests'] += 1
+            aggregate['time'] += result.time
+          end
+          aggregate
+        end
+
+        sig { params(result_type: ResultType).returns(T::Boolean) }
+        def failure?(result_type)
+          case result_type
+          when ResultType::Failed, ResultType::Error
+            true
+          when ResultType::Passed, ResultType::Skipped, ResultType::Discarded, ResultType::Requeued
+            false
+          else
+            T.absurd(result_type)
+          end
+        end
+      end
+    end
+  end
+end

data/lib/minitest/distributed/reporters/redis_coordinator_warnings_reporter.rb

@@ -9,9 +9,10 @@ module Minitest
 
         sig { override.void }
         def report
-          [reclaim_warning, missing_acks_warning].compact.each do |warning|
-            io.puts
+          warnings = [reclaim_timeout_warning, reclaim_failed_warning].compact
+          warnings.each do |warning|
             io.puts(warning)
+            io.puts
           end
         end
 
@@ -28,30 +29,24 @@ module Minitest
         end
 
         sig { returns(T.nilable(String)) }
-        def reclaim_warning
-          if redis_coordinator.reclaimed_tests.any?
+        def reclaim_timeout_warning
+          if redis_coordinator.reclaimed_timeout_tests.any?
             <<~WARNING
               WARNING: The following tests were reclaimed from another worker:
-              #{redis_coordinator.reclaimed_tests.map { |test| "- #{test.identifier}" }.join("\n")}
+              #{redis_coordinator.reclaimed_timeout_tests.map { |test| "- #{test.identifier}" }.join("\n")}
 
-              The original worker did not complete running this test in #{configuration.test_timeout}ms.
+              The original worker did not complete running these tests in #{configuration.test_timeout_seconds}s.
               This either means that the worker unexpectedly went away, or that the test is too slow.
             WARNING
           end
         end
 
         sig { returns(T.nilable(String)) }
-        def missing_acks_warning
-          local_results = redis_coordinator.local_results
-          if local_results.acks < local_results.size
+        def reclaim_failed_warning
+          if redis_coordinator.reclaimed_failed_tests.any?
             <<~WARNING
-              WARNING: This worker was not able to ack all the test it ran with the coordinator (#{local_results.acks}/#{local_results.size}).
-
-              This means that this worker took too long to report the status of one or more tests,
-              and these tests were claimed by other workers. As a result, the total number of
-              reported runs may be larger than the size of the test suite.
-
-              Make sure that all your tests complete within #{configuration.test_timeout}ms.
+              WARNING: The following tests were reclaimed from another worker because they failed:
+              #{redis_coordinator.reclaimed_failed_tests.map { |test| "- #{test.identifier}" }.join("\n")}
             WARNING
           end
         end

data/lib/minitest/distributed/result_aggregate.rb

@@ -6,41 +6,51 @@ module Minitest
     class ResultAggregate < T::Struct
       extend T::Sig
 
+      const :max_failures, T.nilable(Integer)
+
+      # These are maintained between different attempts for the same run ID
       prop :runs, Integer, default: 0
       prop :assertions, Integer, default: 0
       prop :passes, Integer, default: 0
       prop :failures, Integer, default: 0
       prop :errors, Integer, default: 0
       prop :skips, Integer, default: 0
-      prop :reruns, Integer, default: 0
+      prop :requeues, Integer, default: 0
+      prop :discards, Integer, default: 0
+
+      # These are reset between different attempts for the same run ID
       prop :acks, Integer, default: 0
       prop :size, Integer, default: 0
 
-      sig { params(result: Minitest::Result).void }
-      def update_with_result(result)
-        case (result_type = ResultType.of(result))
+      sig { params(runnable_result: EnqueuedRunnable::Result).void }
+      def update_with_result(runnable_result)
+        case (result_type = ResultType.of(runnable_result.committed_result))
         when ResultType::Passed then self.passes += 1
         when ResultType::Failed then self.failures += 1
         when ResultType::Error then self.errors += 1
         when ResultType::Skipped then self.skips += 1
+        when ResultType::Discarded then self.discards += 1
+        when ResultType::Requeued then self.requeues += 1
         else T.absurd(result_type)
         end
 
+        self.acks += 1 if runnable_result.final? && runnable_result.commit.success?
         self.runs += 1
-        self.assertions += result.assertions
+        self.assertions += runnable_result.committed_result.assertions
       end
 
       sig { returns(String) }
       def to_s
         str = +"#{runs} runs, #{assertions} assertions, #{passes} passes, #{failures} failures, #{errors} errors"
         str << ", #{skips} skips" if skips > 0
-        str << ", #{reruns} re-runs" if reruns > 0
+        str << ", #{requeues} re-queued" if requeues > 0
+        str << ", #{discards} discarded" if discards > 0
         str
       end
 
       sig { returns(Integer) }
       def unique_runs
-        runs - reruns
+        runs - requeues - discards
       end
 
       sig { returns(Integer) }
@@ -49,18 +59,37 @@ module Minitest
       end
 
       sig { returns(T::Boolean) }
-      def completed?
+      def complete?
         acks == size
       end
 
+      sig { returns(T::Boolean) }
+      def abort?
+        if (max = max_failures)
+          total_failures >= max
+        else
+          false
+        end
+      end
+
       sig { returns(T::Boolean) }
       def valid?
+        all_runs_reported? && (complete? || abort?)
+      end
+
+      sig { returns(T::Boolean) }
+      def all_runs_reported?
         unique_runs == reported_results
       end
 
+      sig { returns(Integer) }
+      def total_failures
+        failures + errors
+      end
+
       sig { returns(T::Boolean) }
       def passed?
-        completed? && valid? && self.failures == 0 && self.errors == 0
+        total_failures == 0
       end
     end
   end

data/lib/minitest/distributed/result_type.rb

@@ -2,6 +2,74 @@
 # frozen_string_literal: true
 
 module Minitest
+  class Discard < Minitest::Skip
+    extend T::Sig
+
+    sig { returns(Minitest::Result) }
+    attr_reader :original_result
+
+    sig { params(message: String, original_result: Minitest::Result).void }
+    def initialize(message, original_result:)
+      @original_result = original_result
+      super(message)
+    end
+
+    sig { override.returns(String) }
+    def result_label
+      "Discarded"
+    end
+
+    sig { params(result: Minitest::Result, test_timeout_seconds: Float).returns(Minitest::Result) }
+    def self.wrap(result, test_timeout_seconds:)
+      message = +"This test result was discarded, because it could not be committed to the test run coordinator."
+      if result.time > test_timeout_seconds
+        message << format(
+          "\n\nThe test took %0.3fs to run, longer than the test timeout which is configured to be %0.1fs.\n" \
+          "Another worker likely claimed ownership of this test, and will commit the result instead.\n" \
+          "For best results, make sure that all your tests finish within %0.1fs.",
+          result.time, test_timeout_seconds, test_timeout_seconds
+        )
+      end
+
+      discard_assertion = Minitest::Discard.new(message, original_result: result)
+      discard_assertion.set_backtrace(caller)
+      discarded_result = result.dup
+      discarded_result.failures = [discard_assertion]
+      discarded_result
+    end
+  end
+
+  class Requeue < Minitest::Skip
+    extend T::Sig
+
+    sig { params(message: String, original_result: Minitest::Result).void }
+    def initialize(message, original_result:)
+      @original_result = original_result
+      super(message)
+    end
+
+    sig { override.returns(String) }
+    def result_label
+      "Requeued"
+    end
+
+    sig { params(result: Minitest::Result, attempt: Integer, max_attempts: Integer).returns(Minitest::Result) }
+    def self.wrap(result, attempt:, max_attempts:)
+      failure = T.must(result.failure)
+
+      message = "#{failure.message}\n\nThe test will be retried (attempt #{attempt} of #{max_attempts})"
+      requeue_assertion = Minitest::Requeue.new(message, original_result: result)
+      requeue_assertion.set_backtrace(failure.backtrace)
+
+      requeued_result = result.dup
+      requeued_result.failures = [requeue_assertion]
+      requeued_result
+    end
+  end
+
+  class AttemptsExhausted < Minitest::Assertion
+  end
+
   module Distributed
     class ResultType < T::Enum
       extend T::Sig
@@ -11,16 +79,22 @@ module Minitest
         Failed = new
         Error = new
         Skipped = new
+        Discarded = new
+        Requeued = new
       end
 
       sig { params(result: Minitest::Result).returns(ResultType) }
       def self.of(result)
         if result.passed?
           Passed
-        elsif result.error?
-          Error
+        elsif result.failure.is_a?(Minitest::Requeue)
+          Requeued
+        elsif result.failure.is_a?(Minitest::Discard)
+          Discarded
         elsif result.skipped?
           Skipped
+        elsif result.error?
+          Error
         else
           Failed
         end

data/lib/minitest/distributed/test_selector.rb

@@ -1,6 +1,8 @@
 # typed: strict
 # frozen_string_literal: true
 
+require 'pathname'
+
 module Minitest
   module Distributed
     class TestSelector
@@ -24,18 +26,22 @@ module Minitest
       def initialize_filters
         @filters << Filters::IncludeFilter.new(options[:filter]) if options[:filter]
         @filters << Filters::ExcludeFilter.new(options[:exclude]) if options[:exclude]
+
+        exclude_file = options[:distributed].exclude_file
+        @filters << Filters::ExcludeFileFilter.new(Pathname.new(exclude_file)) if exclude_file
+
+        include_file = options[:distributed].include_file
+        @filters << Filters::IncludeFileFilter.new(Pathname.new(include_file)) if include_file
       end
 
-      sig { returns(T::Array[EnqueuedRunnable]) }
+      sig { returns(T::Array[Minitest::Runnable]) }
       def discover_tests
         Minitest::Runnable.runnables.flat_map do |runnable|
-          runnable.runnable_methods.map do |method_name|
-            EnqueuedRunnable.new(class_name: runnable.name, method_name: method_name)
-          end
+          runnable.runnable_methods.map { |method_name| runnable.new(method_name) }
         end
       end
 
-      sig { params(tests: T::Array[EnqueuedRunnable]).returns(T::Array[EnqueuedRunnable]) }
+      sig { params(tests: T::Array[Minitest::Runnable]).returns(T::Array[Minitest::Runnable]) }
       def select_tests(tests)
         return tests if filters.empty?
         tests.flat_map do |runnable_method|
@@ -45,7 +51,7 @@ module Minitest
         end.compact
       end
 
-      sig { returns(T::Array[EnqueuedRunnable]) }
+      sig { returns(T::Array[Minitest::Runnable]) }
       def tests
         select_tests(discover_tests)
       end

data/lib/minitest/distributed/version.rb

@@ -3,6 +3,6 @@
 
 module Minitest
   module Distributed
-    VERSION = "0.1.2"
+    VERSION = "0.2.2"
   end
 end

data/lib/minitest/distributed.rb

@@ -13,6 +13,9 @@ require "minitest/distributed/result_aggregate"
 require "minitest/distributed/filters/filter_interface"
 require "minitest/distributed/filters/include_filter"
 require "minitest/distributed/filters/exclude_filter"
+require "minitest/distributed/filters/file_filter_base"
+require "minitest/distributed/filters/exclude_file_filter"
+require "minitest/distributed/filters/include_file_filter"
 require "minitest/distributed/coordinators/coordinator_interface"
 require "minitest/distributed/coordinators/memory_coordinator"
 require "minitest/distributed/coordinators/redis_coordinator"

data/lib/minitest/distributed_plugin.rb

@@ -12,31 +12,7 @@ module Minitest
         options[:disable_distributed] = true
       end
 
-      options[:distributed] = Minitest::Distributed::Configuration.from_env
-
-      opts.on('--coordinator=URI', "The URI pointing to the coordinator") do |uri|
-        options[:distributed].coordinator_uri = URI.parse(uri)
-      end
-
-      opts.on('--test-timeout=TIMEOUT', "The maximum run time for a single test in seconds") do |timeout|
-        options[:distributed].test_timeout = Float(timeout)
-      end
-
-      opts.on('--max-attempts=ATTEMPTS', "The maximum number of attempts to run a test") do |attempts|
-        options[:distributed].max_attempts = Integer(attempts)
-      end
-
-      opts.on('--test-batch-size=NUMBER', "The number of tests to process per batch") do |batch_size|
-        options[:distributed].test_batch_size = Integer(batch_size)
-      end
-
-      opts.on('--run-id=ID', "The ID for this run shared between coordinated workers") do |id|
-        options[:distributed].run_id = id
-      end
-
-      opts.on('--worker-id=ID', "The unique ID for this worker") do |id|
-        options[:distributed].worker_id = id
-      end
+      options[:distributed] = Minitest::Distributed::Configuration.from_command_line_options(opts, options)
     end
 
     def plugin_distributed_init(options)

data/lib/minitest/junitxml_plugin.rb

@@ -0,0 +1,21 @@
+# typed: true
+# frozen_string_literal: true
+
+module Minitest
+  class << self
+    def plugin_junitxml_options(opts, options)
+      options[:junitxml] = ENV['MINITEST_JUNITXML']
+
+      opts.on('--junitxml=PATH', "Generate a JUnitXML report at the specified path") do |path|
+        options[:junitxml] = path
+      end
+    end
+
+    def plugin_junitxml_init(options)
+      return if options[:junitxml].nil?
+
+      require 'minitest/distributed/reporters/junitxml_reporter'
+      reporter << Minitest::Distributed::Reporters::JUnitXMLReporter.new(options[:io], options)
+    end
+  end
+end

data/sorbet/rbi/minitest.rbi

@@ -1,23 +1,23 @@
-# This file is autogenerated. Do not edit it by hand. Regenerate it with:
-#   srb rbi sorbet-typed
-#
-# If you would like to make changes to this file, great! Please upstream any changes you make here:
-#
-#   https://github.com/sorbet/sorbet-typed/edit/master/lib/minitest/all/minitest.rbi
-#
-# typed: strong
+# typed: true
 
 module Minitest
   class Runnable
     def self.run_one_method(klass, method_name, reporter); end
     def self.runnables; end
 
+    def initialize(method_name); end
+
     def name; end
     def time; end
     def time=(duration); end
     def failures; end
-
-    def initialize(method_name); end
+    def failures=(failures); end
+    def assertions; end
+    def assertions=(assertions); end
+    def source_location; end
+    def source_location=(value); end
+    def klass; end
+    def klass=(value); end
   end
 
   class Test < Runnable
@@ -61,10 +61,15 @@ module Minitest
     def result_code; end
   end
 
-  class Skip < Exception
+  class Skip < Assertion
   end
 
   class UnexpectedError < Assertion
+    sig { params(error: Exception).void }
+    def initialize(error); end
+
+    sig { returns(Exception) }
+    def error; end
   end
 
   class AbstractReporter
@@ -126,6 +131,9 @@ module Minitest
 
   sig { returns(Float) }
   def self.clock_time; end
+
+  def self.backtrace_filter; end
+  def self.backtrace_filter=(filter); end
 end
 
 module Minitest::Assertions
@@ -190,6 +198,16 @@ module Minitest::Assertions
   end
   def assert_predicate(obj, predicate, msg = nil); end
 
+  sig do
+    params(
+      value: BasicObject,
+      operator: Symbol,
+      comparison: BasicObject,
+      msg: T.nilable(String)
+    ).returns(TrueClass)
+  end
+  def assert_operator(value, operator, comparison, msg = nil); end
+
   sig { params(test: T.untyped, msg: T.nilable(String)).returns(TrueClass) }
   def refute(test, msg = nil); end
 

data/sorbet/rbi/redis.rbi

@@ -7,9 +7,17 @@ class Redis
   class CommandError < Error
   end
 
+  class Future
+    sig { returns(T.untyped) }
+    def value; end
+  end
+
   sig { params(options: T::Hash[Symbol, T.untyped]).void }
   def initialize(options); end
 
+  sig { params(block: T.proc.params(arg0: String).void).void }
+  def monitor(&block); end
+
   sig { void }
   def flushdb; end
 
@@ -49,7 +57,7 @@ class Redis
   sig { params(key_value_pairs: T.untyped).returns(T::Boolean) }
   def msetnx(*key_value_pairs); end
 
-  sig { params(key: String).void }
+  sig { params(key: String).returns(Integer) }
   def incr(key); end
 
   sig { params(key: String, value: T.untyped).void }
@@ -58,13 +66,20 @@ class Redis
   sig { params(key: String, start: Integer, stop: Integer).void }
   def lrange(key, start, stop); end
 
-  sig { params(key: String, amount: Integer).void }
+  sig { params(key: String, value: T.untyped).returns(T.untyped) }
+  def sadd(key, value); end
+
+  sig { params(key: String, value: T.untyped).returns(T::Boolean) }
+  def srem(key, value); end
+
+  sig { params(key: String, amount: Integer).returns(Integer) }
   def incrby(key, amount); end
 
-  def xadd(*); end
-  def xack(*); end
+  def xadd(key, value); end
+  def xack(stream_key, group_name, *entry_ids); end
   def xgroup(*); end
   def xpending(*); end
   def xreadgroup(*); end
   def xclaim(*); end
+  def xinfo(*); end
 end