minitest-distributed 0.1.0

data/bin/rake

@@ -0,0 +1,29 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+#
+# This file was generated by Bundler.
+#
+# The application 'rake' is installed as part of a gem, and
+# this file is here to facilitate running it.
+#
+
+require "pathname"
+ENV["BUNDLE_GEMFILE"] ||= File.expand_path("../../Gemfile",
+  Pathname.new(__FILE__).realpath)
+
+bundle_binstub = File.expand_path("../bundle", __FILE__)
+
+if File.file?(bundle_binstub)
+  if File.read(bundle_binstub, 300) =~ /This file was generated by Bundler/
+    load(bundle_binstub)
+  else
+    abort("Your `bin/bundle` was not generated by Bundler, so this binstub cannot run.
+Replace `bin/bundle` by running `bundle binstubs bundler --force`, then run this command again.")
+  end
+end
+
+require "rubygems"
+require "bundler/setup"
+
+load(Gem.bin_path("rake", "rake"))

data/bin/rubocop

@@ -0,0 +1,29 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+#
+# This file was generated by Bundler.
+#
+# The application 'rubocop' is installed as part of a gem, and
+# this file is here to facilitate running it.
+#
+
+require "pathname"
+ENV["BUNDLE_GEMFILE"] ||= File.expand_path("../../Gemfile",
+  Pathname.new(__FILE__).realpath)
+
+bundle_binstub = File.expand_path("../bundle", __FILE__)
+
+if File.file?(bundle_binstub)
+  if File.read(bundle_binstub, 300) =~ /This file was generated by Bundler/
+    load(bundle_binstub)
+  else
+    abort("Your `bin/bundle` was not generated by Bundler, so this binstub cannot run.
+Replace `bin/bundle` by running `bundle binstubs bundler --force`, then run this command again.")
+  end
+end
+
+require "rubygems"
+require "bundler/setup"
+
+load(Gem.bin_path("rubocop", "rubocop"))

data/bin/setup

@@ -0,0 +1,8 @@
+#!/usr/bin/env bash
+set -euo pipefail
+IFS=$'\n\t'
+set -vx
+
+bundle install
+
+# Do any other automated setup that you need to do here

data/bin/srb

@@ -0,0 +1,29 @@
+#!/usr/bin/env ruby
+# frozen_string_literal: true
+
+#
+# This file was generated by Bundler.
+#
+# The application 'srb' is installed as part of a gem, and
+# this file is here to facilitate running it.
+#
+
+require "pathname"
+ENV["BUNDLE_GEMFILE"] ||= File.expand_path("../../Gemfile",
+  Pathname.new(__FILE__).realpath)
+
+bundle_binstub = File.expand_path("../bundle", __FILE__)
+
+if File.file?(bundle_binstub)
+  if File.read(bundle_binstub, 300) =~ /This file was generated by Bundler/
+    load(bundle_binstub)
+  else
+    abort("Your `bin/bundle` was not generated by Bundler, so this binstub cannot run.
+Replace `bin/bundle` by running `bundle binstubs bundler --force`, then run this command again.")
+  end
+end
+
+require "rubygems"
+require "bundler/setup"
+
+load(Gem.bin_path("sorbet", "srb"))

data/lib/minitest/distributed.rb

@@ -0,0 +1,36 @@
+# typed: strict
+# frozen_string_literal: true
+
+require 'minitest'
+require 'sorbet-runtime'
+
+require "minitest/distributed/configuration"
+require "minitest/distributed/test_runner"
+require "minitest/distributed/test_selector"
+require "minitest/distributed/enqueued_runnable"
+require "minitest/distributed/result_type"
+require "minitest/distributed/result_aggregate"
+require "minitest/distributed/filters/filter_interface"
+require "minitest/distributed/filters/include_filter"
+require "minitest/distributed/filters/exclude_filter"
+require "minitest/distributed/coordinators/coordinator_interface"
+require "minitest/distributed/coordinators/memory_coordinator"
+require "minitest/distributed/coordinators/redis_coordinator"
+require "minitest/distributed/reporters/redis_coordinator_warnings_reporter"
+require "minitest/distributed/reporters/distributed_progress_reporter"
+require "minitest/distributed/reporters/distributed_summary_reporter"
+
+module Minitest
+  module Distributed
+    class Error < StandardError; end
+
+    module TestRunnerPatch
+      extend T::Sig
+
+      sig { params(reporter: Minitest::AbstractReporter, options: T::Hash[Symbol, T.untyped]).void }
+      def __run(reporter, options)
+        TestRunner.new(options).run(reporter)
+      end
+    end
+  end
+end

data/lib/minitest/distributed/configuration.rb

@@ -0,0 +1,53 @@
+# typed: strict
+# frozen_string_literal: true
+
+require 'uri'
+require 'securerandom'
+
+module Minitest
+  module Distributed
+    class Configuration < T::Struct
+      DEFAULT_BATCH_SIZE = 10
+      DEFAULT_MAX_ATTEMPTS = 3
+      DEFAULT_TEST_TIMEOUT = 30_000 # milliseconds
+
+      class << self
+        extend T::Sig
+
+        sig { params(env: T::Hash[String, T.nilable(String)]).returns(T.attached_class) }
+        def from_env(env = ENV.to_h)
+          new(
+            coordinator_uri: URI(env['MINITEST_COORDINATOR'] || 'memory:'),
+            run_id: env['MINITEST_RUN_ID'] || SecureRandom.uuid,
+            worker_id: env['MINITEST_WORKER_ID'] || SecureRandom.uuid,
+            test_timeout: Integer(env['MINITEST_TEST_TIMEOUT'] || DEFAULT_TEST_TIMEOUT),
+            test_batch_size: Integer(env['MINITEST_TEST_BATCH_SIZE'] || DEFAULT_BATCH_SIZE),
+            max_attempts: Integer(env['MINITEST_MAX_ATTEMPTS'] || DEFAULT_MAX_ATTEMPTS),
+          )
+        end
+      end
+
+      extend T::Sig
+
+      prop :coordinator_uri, URI::Generic, default: URI('memory:')
+      prop :run_id, String, factory: -> { SecureRandom.uuid }
+      prop :worker_id, String, factory: -> { SecureRandom.uuid }
+      prop :test_timeout, Integer, default: DEFAULT_TEST_TIMEOUT
+      prop :test_batch_size, Integer, default: DEFAULT_BATCH_SIZE
+      prop :max_attempts, Integer, default: DEFAULT_MAX_ATTEMPTS
+
+      sig { returns(Coordinators::CoordinatorInterface) }
+      def coordinator
+        @coordinator = T.let(@coordinator, T.nilable(Coordinators::CoordinatorInterface))
+        @coordinator ||= case coordinator_uri.scheme
+        when 'redis'
+          Coordinators::RedisCoordinator.new(configuration: self)
+        when 'memory'
+          Coordinators::MemoryCoordinator.new(configuration: self)
+        else
+          raise NotImplementedError, "Unknown coordinator implementation: #{coordinator_uri.scheme}"
+        end
+      end
+    end
+  end
+end

data/lib/minitest/distributed/coordinators/coordinator_interface.rb

@@ -0,0 +1,29 @@
+# typed: strict
+# frozen_string_literal: true
+
+module Minitest
+  module Distributed
+    module Coordinators
+      module CoordinatorInterface
+        extend T::Sig
+        extend T::Helpers
+        interface!
+
+        sig { abstract.params(reporter: Minitest::CompositeReporter, options: T::Hash[Symbol, T.untyped]).void }
+        def register_reporters(reporter:, options:); end
+
+        sig { abstract.returns(ResultAggregate) }
+        def local_results; end
+
+        sig { abstract.returns(ResultAggregate) }
+        def combined_results; end
+
+        sig { abstract.params(test_selector: TestSelector).void }
+        def produce(test_selector:); end
+
+        sig { abstract.params(reporter: Minitest::AbstractReporter).void }
+        def consume(reporter:); end
+      end
+    end
+  end
+end

data/lib/minitest/distributed/coordinators/memory_coordinator.rb

@@ -0,0 +1,67 @@
+# typed: strict
+# frozen_string_literal: true
+
+module Minitest
+  module Distributed
+    module Coordinators
+      class MemoryCoordinator
+        extend T::Sig
+        include CoordinatorInterface
+
+        sig { returns(Configuration) }
+        attr_reader :configuration
+
+        sig { returns(Queue) }
+        attr_reader :queue
+
+        sig { override.returns(ResultAggregate) }
+        attr_reader :local_results
+
+        alias_method :combined_results, :local_results
+
+        sig { params(configuration: Configuration).void }
+        def initialize(configuration:)
+          @configuration = configuration
+
+          @leader = T.let(Mutex.new, Mutex)
+          @queue = T.let(Queue.new, Queue)
+          @local_results = T.let(ResultAggregate.new, ResultAggregate)
+        end
+
+        sig { override.params(reporter: Minitest::CompositeReporter, options: T::Hash[Symbol, T.untyped]).void }
+        def register_reporters(reporter:, options:)
+          # No need for any additional reporters
+        end
+
+        sig { override.params(test_selector: TestSelector).void }
+        def produce(test_selector:)
+          if @leader.try_lock
+            tests = test_selector.tests
+            @local_results.size = tests.size
+            if tests.empty?
+              queue.close
+            else
+              tests.each { |test| queue << test }
+            end
+          end
+        end
+
+        sig { override.params(reporter: AbstractReporter).void }
+        def consume(reporter:)
+          until queue.empty? && queue.closed?
+            enqueued_runnable = queue.pop
+            reporter.prerecord(enqueued_runnable.runnable_class, enqueued_runnable.method_name)
+            result = enqueued_runnable.run
+
+            local_results.update_with_result(result)
+            local_results.acks += 1
+
+            reporter.record(result)
+
+            queue.close if local_results.completed?
+          end
+        end
+      end
+    end
+  end
+end

data/lib/minitest/distributed/coordinators/redis_coordinator.rb

@@ -0,0 +1,387 @@
+# typed: strict
+# frozen_string_literal: true
+
+require 'redis'
+
+module Minitest
+  module Distributed
+    module Coordinators
+      # The RedisCoordinator is an implementation of the test coordinator interface
+      # using a Redis stream + consumergroup for coordination.
+      #
+      # We assume a bunch of workers will be started at the same time. Every worker
+      # will try to become the leader by trying to create the consumergroup. Only one
+      # will succeed, which will then continue to populate the list of tests to run
+      # to the stream.
+      #
+      # AFter that, all workers will start consuming from the stream. They will first
+      # try to claim stale entries from other workers (determined by the `test_timeout`
+      # option), and process them tp to a maxumim of `max_attempts` attempts. Then,
+      # they will consume tests from the stream, run them, and ack them. This is done
+      # in batches to reduce load on Redis.
+      #
+      # Finally, when we have acked the same number of tests as we populated into the
+      # queue, the run is considered finished. The first worker to detect this will
+      # remove the consumergroup and the associated stream from Redis.
+      #
+      # If a worker starts for the same run_id while it is already considered completed,
+      # it will start a "retry run". It will find all the tests that failed/errored on
+      # the previous attempt, and schedule only those tests to be run, rather than the
+      # full test suite returned by the test selector. This can be useful to retry flaky
+      # tests. Subsequent workers coming online will join this worker to form a consumer
+      # group exactly as described above.
+      class RedisCoordinator
+        extend T::Sig
+        include CoordinatorInterface
+
+        sig { returns(Configuration) }
+        attr_reader :configuration
+
+        sig { returns(String) }
+        attr_reader :stream_key
+
+        sig { returns(String) }
+        attr_reader :group_name
+
+        sig { override.returns(ResultAggregate) }
+        attr_reader :local_results
+
+        sig { returns(T::Set[EnqueuedRunnable]) }
+        attr_reader :reclaimed_tests
+
+        sig { params(configuration: Configuration).void }
+        def initialize(configuration:)
+          @configuration = configuration
+
+          @redis = T.let(nil, T.nilable(Redis))
+          @stream_key = T.let(key('queue'), String)
+          @group_name = T.let('minitest-distributed', String)
+          @local_results = T.let(ResultAggregate.new, ResultAggregate)
+          @combined_results = T.let(nil, T.nilable(ResultAggregate))
+          @reclaimed_tests = T.let(Set.new, T::Set[EnqueuedRunnable])
+        end
+
+        sig { override.params(reporter: Minitest::CompositeReporter, options: T::Hash[Symbol, T.untyped]).void }
+        def register_reporters(reporter:, options:)
+          reporter << Reporters::RedisCoordinatorWarningsReporter.new(options[:io], options)
+        end
+
+        sig { override.returns(ResultAggregate) }
+        def combined_results
+          @combined_results ||= begin
+            stats_as_string = redis.mget(key('runs'), key('assertions'), key('passes'),
+              key('failures'), key('errors'), key('skips'), key('reruns'), key('acks'), key('size'))
+
+            ResultAggregate.new(
+              runs: Integer(stats_as_string.fetch(0) || 0),
+              assertions: Integer(stats_as_string.fetch(1) || 0),
+              passes: Integer(stats_as_string.fetch(2) || 0),
+              failures: Integer(stats_as_string.fetch(3) || 0),
+              errors: Integer(stats_as_string.fetch(4) || 0),
+              skips: Integer(stats_as_string.fetch(5) || 0),
+              reruns: Integer(stats_as_string.fetch(6) || 0),
+              acks: Integer(stats_as_string.fetch(7) || 0),
+
+              # In the case where we have no build szie number published yet, we initialize
+              # thesize of the test suite to be arbitrarity large, to make sure it is
+              # higher than the number of acks, so the run is not consider completed yet.
+              size: Integer(stats_as_string.fetch(8) || 2_147_483_647),
+            )
+          end
+        end
+
+        sig { override.params(test_selector: TestSelector).void }
+        def produce(test_selector:)
+          # Whoever ends up creating the consumer group will act as leader,
+          # and publish the list of tests to the stream.
+
+          begin
+            # When using `redis.multi`, the second DEL command gets executed even if the initial GROUP
+            # fails. This is bad, because only the leader should be issuing the DEL command.
+            # When using EVAL and a Lua script, the script aborts after the first XGROUP command
+            # fails, and the DEL never gets executed for followers.
+            redis.evalsha(
+              register_consumergroup_script,
+              keys: [stream_key, key('size'), key('acks')],
+              argv: [group_name],
+            )
+
+          rescue Redis::CommandError => ce
+            if ce.message.include?('BUSYGROUP')
+              # If Redis returns a BUSYGROUP error, it means that the consumer group already
+              # exists. In our case, it means that another worker managed to successfully
+              # run the XGROUP command, and will act as leader and publish the tests.
+              # This worker can simply move on the consumer mode.
+              return
+            else
+              raise
+            end
+          end
+
+          run_attempt, previous_failures, previous_errors, _deleted = redis.multi do
+            redis.incr(key('attempt'))
+            redis.lrange(key('failure_list'), 0, -1)
+            redis.lrange(key('error_list'), 0, -1)
+            redis.del(key('failure_list'), key('error_list'))
+          end
+
+          tests = if run_attempt == 1
+            # If this is the first attempt for this run ID, we will schedule the full
+            # test suite as returned by the test selector to run.
+            test_selector.tests
+          else
+            # For subsequent attempts, we check the list of previous failures and
+            # errors, and only schedule to re-run those tests. This allows for faster
+            # retries of potentially flaky tests.
+            (previous_failures + previous_errors).map do |test_to_retry|
+              EnqueuedRunnable.from_hash!(Marshal.load(test_to_retry))
+            end
+          end
+
+          # We set the `size` key to the number of tests we are planning to schedule.
+          # This will allow workers to tell when the run is done. We also adjust the
+          # number of failures and errors in case of a retry run.
+          adjust_combined_results(ResultAggregate.new(
+            size: tests.size,
+            failures: -previous_failures.length,
+            errors: -previous_errors.length,
+            reruns: previous_failures.length + previous_errors.length,
+          ))
+
+          # TODO: break this up in batches.
+          tests.each { |test| redis.xadd(stream_key, test.serialize) }
+        end
+
+        sig { override.params(reporter: AbstractReporter).void }
+        def consume(reporter:)
+          exponential_backoff = INITIAL_BACKOFF
+          loop do
+            # First, see if there are any pending tests from other workers to claim.
+            stale_runnables = claim_stale_runnables
+            stale_processed = process_batch(stale_runnables, reporter)
+
+            # Finally, try to process a regular batch of messages
+            fresh_runnables = claim_fresh_runnables(block: exponential_backoff)
+            fresh_processed = process_batch(fresh_runnables, reporter)
+
+            # If we have acked the same amount of tests as we were supposed to, the run
+            # is complete and we can exit our loop. Generally, only one worker will detect
+            # this condition. The pther workers will quit their consumer loop because the
+            # consumergroup will be deleted by the first worker, and their Redis commands
+            # will start to fail - see the rescue block below.
+            break if combined_results.completed?
+
+            # To make sure we don't end up in a busy loop overwhelming Redis with commands
+            # when there is no work to do, we increase the blocking time exponentially,
+            # and reset it to the initial value if we processed any messages
+            if stale_processed > 0 || fresh_processed > 0
+              exponential_backoff = INITIAL_BACKOFF
+            else
+              exponential_backoff <<= 1
+            end
+          end
+
+          cleanup
+        rescue Redis::CommandError => ce
+          if ce.message.start_with?('NOGROUP')
+            # When a redis conumer group commands fails with a NOGROUP error, we assume the
+            # consumer group was deleted by the first worker that detected the run is complete.
+            # So this worker can exit its loop as well.
+
+            # We have to invalidate the local combined_results cache so we get fresh
+            # final values from Redis when we try to report results in our summarizer.
+            @combined_results = nil
+          else
+            raise
+          end
+        end
+
+        private
+
+        sig { returns(Redis) }
+        def redis
+          @redis ||= Redis.new(url: configuration.coordinator_uri)
+        end
+
+        sig { returns(String) }
+        def ack_batch_script
+          @ack_batch_script = T.let(@ack_batch_script, T.nilable(String))
+          @ack_batch_script ||= redis.script(:load, <<~LUA)
+            local acked_ids, acked, i = {}, 0, 2
+            while ARGV[i] do
+              if redis.call('XACK', KEYS[1], ARGV[1], ARGV[i]) > 0 then
+                acked = acked + 1
+                acked_ids[acked] = ARGV[i]
+              end
+              i = i + 1
+            end
+            return acked_ids
+          LUA
+        end
+
+        sig { returns(String) }
+        def register_consumergroup_script
+          @register_consumergroup_script = T.let(@register_consumergroup_script, T.nilable(String))
+          @register_consumergroup_script ||= redis.script(:load, <<~LUA)
+            redis.call('XGROUP', 'CREATE', KEYS[1], ARGV[1], '0', 'MKSTREAM')
+            redis.call('DEL', KEYS[2], KEYS[3])
+          LUA
+        end
+
+        sig { params(block: Integer).returns(T::Array[EnqueuedRunnable]) }
+        def claim_fresh_runnables(block:)
+          result = redis.xreadgroup(group_name, configuration.worker_id, stream_key, '>',
+            block: block, count: configuration.test_batch_size)
+          EnqueuedRunnable.from_redis_stream_claim(result.fetch(stream_key, []))
+        end
+
+        sig { returns(T::Array[EnqueuedRunnable]) }
+        def claim_stale_runnables
+          # When we have to reclaim stale tests, those test are potentially too slow
+          # to run inside the test timeout. We only claim one test at a time in order
+          # to prevent the exact same batch from being too slow on repeated attempts,
+          # which would cause us to mark all the tests in that batch as failed.
+          #
+          # This has the side effect that for a retried test, the test timeout
+          # will be TEST_TIMEOUT * BATCH_SIZE in practice. This gives us a higher
+          # likelihood that the test will pass if the batch size > 1.
+          pending = redis.xpending(stream_key, group_name, '-', '+', 1)
+
+          # Every test is allowed to take test_timeout milliseconds. Because we process tests in
+          # batches, they should never be pending for TEST_TIMEOUT * BATCH_SIZE milliseconds.
+          # So, only try to claim messages older than that, with a bit of jitter.
+          max_idle_time = configuration.test_timeout * configuration.test_batch_size
+          max_idle_time_with_jitter = max_idle_time * rand(1.0...1.2)
+          to_claim = pending.each_with_object({}) do |message, hash|
+            if message['elapsed'] > max_idle_time_with_jitter
+              hash[message.fetch('entry_id')] = message
+            end
+          end
+
+          if to_claim.empty?
+            []
+          else
+            claimed = redis.xclaim(stream_key, group_name, configuration.worker_id, max_idle_time, to_claim.keys)
+            enqueued_runnables = EnqueuedRunnable.from_redis_stream_claim(claimed)
+            enqueued_runnables.each do |er|
+              # `count` will be set to the current attempt of a different worker that has timed out.
+              # The attempt we are going to try will be the next one, so add one.
+              attempt = to_claim.fetch(er.execution_id).fetch('count') + 1
+              if attempt > configuration.max_attempts
+                # If we exhaust our attempts, we will mark the test to immediately fail when it will be run next.
+                mark_runnable_to_fail_immediately(er)
+              else
+                reclaimed_tests << er
+              end
+            end
+
+            enqueued_runnables
+          end
+        end
+
+        sig { void }
+        def cleanup
+          redis.xgroup(:destroy, stream_key, group_name)
+          redis.del(stream_key)
+        rescue Redis::CommandError
+          # Apparently another consumer already removed the consumer group,
+          # so we can assume that all the Redis cleanup was completed.
+        end
+
+        sig { params(er: EnqueuedRunnable).void }
+        def mark_runnable_to_fail_immediately(er)
+          assertion = Minitest::Assertion.new(<<~EOM.chomp)
+            This test takes too long to run (> #{configuration.test_timeout}ms).
+
+            We have tried running this test #{configuration.max_attempts} on different workers, but every time the worker has not reported back a result within #{configuration.test_timeout}ms.
+            Try to make the test faster, or increase the test timeout.
+          EOM
+          assertion.set_backtrace(caller)
+          er.canned_failure = assertion
+        end
+
+        sig { params(results: ResultAggregate).void }
+        def adjust_combined_results(results)
+          updated = redis.multi do
+            redis.incrby(key('runs'), results.runs)
+            redis.incrby(key('assertions'), results.assertions)
+            redis.incrby(key('passes'), results.passes)
+            redis.incrby(key('failures'), results.failures)
+            redis.incrby(key('errors'), results.errors)
+            redis.incrby(key('skips'), results.skips)
+            redis.incrby(key('reruns'), results.reruns)
+            redis.incrby(key('acks'), results.acks)
+            redis.incrby(key('size'), results.size)
+          end
+
+          @combined_results = ResultAggregate.new(runs: updated[0], assertions: updated[1], passes: updated[2],
+            failures: updated[3], errors: updated[4], skips: updated[5], reruns: updated[6],
+            acks: updated[7], size: updated[8])
+        end
+
+        sig { params(name: String).returns(String) }
+        def key(name)
+          "minitest/#{configuration.run_id}/#{name}"
+        end
+
+        sig { params(batch: T::Array[EnqueuedRunnable], reporter: AbstractReporter).returns(Integer) }
+        def process_batch(batch, reporter)
+          to_be_acked = {}
+
+          batch.each do |enqueued_runnable|
+            local_results.size += 1
+            reporter.prerecord(enqueued_runnable.runnable_class, enqueued_runnable.method_name)
+            result = enqueued_runnable.run
+
+            case (result_type = ResultType.of(result))
+            when ResultType::Passed
+              # noop
+            when ResultType::Skipped
+              redis.lpush(key('skip_list'), Marshal.dump(enqueued_runnable.serialize))
+            when ResultType::Failed
+              redis.lpush(key('failure_list'), Marshal.dump(enqueued_runnable.serialize))
+            when ResultType::Error
+              redis.lpush(key('error_list'), Marshal.dump(enqueued_runnable.serialize))
+            else
+              T.absurd(result_type)
+            end
+
+            local_results.update_with_result(result)
+            to_be_acked[enqueued_runnable.execution_id] = result
+          end
+
+          return 0 if to_be_acked.empty?
+
+          acked = redis.evalsha(
+            ack_batch_script,
+            keys: [stream_key],
+            argv: [group_name] + to_be_acked.keys
+          )
+
+          batch_results = ResultAggregate.new(acks: acked.length)
+          acked.each do |execution_id|
+            acked_result = to_be_acked.delete(execution_id)
+            reporter.record(acked_result)
+            batch_results.update_with_result(acked_result)
+          end
+
+          to_be_acked.each do |_execution_id, unacked_result|
+            # TODO: use custom assertion class.
+            discard_assertion = Minitest::Skip.new("The test result was discarded, " \
+              "because the test has been claimed another worker.")
+            discard_assertion.set_backtrace(caller)
+            unacked_result.failures = [discard_assertion]
+            reporter.record(unacked_result)
+          end
+
+          adjust_combined_results(batch_results)
+          local_results.acks += acked.length
+          acked.length
+        end
+
+        INITIAL_BACKOFF = 10 # milliseconds
+        private_constant :INITIAL_BACKOFF
+      end
+    end
+  end
+end