right_support 2.10.1 → 2.11.2

data/spec/net/{balancing → lb}/health_check_spec.rb

@@ -11,7 +11,8 @@ describe RightSupport::Net::LB::HealthCheck do
     @reset_time = 60
     @policy = RightSupport::Net::LB::HealthCheck.new(
                 :yellow_states => @yellow_states,
-                :reset_time => @reset_time)
+                :reset_time => @reset_time,
+                :health_check => lambda { |ep| true })
     @policy.set_endpoints(@endpoints)
     @trials = 2500
   end
@@ -67,9 +68,11 @@ describe RightSupport::Net::LB::HealthCheck do
       before(:each) do
         @yellow_states = 3
         @health_updates = []
-        @policy = RightSupport::Net::LB::HealthCheck.new({
-                    :yellow_states => @yellow_states, :reset_time => @reset_time,
-                    :on_health_change => lambda { |health| @health_updates << health }})
+        @policy = RightSupport::Net::LB::HealthCheck.new(
+          :yellow_states => @yellow_states,
+          :reset_time => @reset_time,
+          :on_health_change => lambda { |health| @health_updates << health },
+          :health_check => lambda { |ep| true })
         @policy.set_endpoints(@endpoints)
 
         # put everyone into green state, then forget all health updates. this helps us write an
@@ -157,9 +160,10 @@ describe RightSupport::Net::LB::HealthCheck do
       before(:each) do
         @yellow_states = 3
         @health_updates = []
-        @policy = RightSupport::Net::LB::HealthCheck.new({
-                    :yellow_states => @yellow_states, :reset_time => @reset_time,
-                    :on_health_change => lambda { |health| @health_updates << health }})
+        @policy = RightSupport::Net::LB::HealthCheck.new(
+          :yellow_states => @yellow_states, :reset_time => @reset_time,
+          :on_health_change => lambda { |health| @health_updates << health },
+          :health_check => lambda { |ep| true })
         @policy.set_endpoints(@endpoints)
 
         # put everyone into green state, then forget all health updates. this helps us write an
@@ -212,7 +216,7 @@ describe RightSupport::Net::LB::HealthCheck do
         @yellow_states.times { @policy.bad(@red, 0, Time.now) }
         @policy.should have_red_endpoint(@red)
 
-        seen = find_empirical_distribution(@trials,@endpoints) do 
+        seen = find_empirical_distribution(@trials,@endpoints) do
           @policy.next
         end
 
@@ -232,24 +236,53 @@ describe RightSupport::Net::LB::HealthCheck do
         should_be_chosen_fairly(seen, @trials, @endpoints.size - 1)
       end
 
-      it 'demands a health check for yellow servers' do
-        pending
-      end
+      it "eliminates bad endpoints during round robin selection" do
+        endpoints = [1, 2, 3, 4, 5]
+        policy = RightSupport::Net::LB::HealthCheck.new(
+          :yellow_states => 2,
+          :health_check => lambda { |ep| true })
+        policy.set_endpoints(endpoints)
+        policy.instance_variable_set(:@counter, 0)
+        breaking_bad = lambda { |ep| ep < 4 }
 
-      it "maintains the same order of green and yellow servers" do
+        # first pass; initially all green, mark some yellow.
+        expected = [1, 2, 3, 4, 5]
         actual = []
-        expected = [3,4,1,2]
+        endpoints.size.times do
+          endpoint, need_health_check = policy.next
+          need_health_check.should be_false
+          if breaking_bad.call(endpoint)
+            policy.bad(endpoint, 0, Time.now)  # go yellow
+          end
+          actual << endpoint
+        end
+        actual.should == expected
 
-        endpoints = [1,2,3,4]
-        policy = RightSupport::Net::LB::HealthCheck.new(:yellow_states => 1)
-        policy.set_endpoints(endpoints)
-        policy.instance_variable_set(:@counter, 1)
+        # second pass; some yellow, others green. yellows go red after being
+        # selected and are eliminated from subsequent round-robin selection.
+        expected = [1, 4, 5, 2, 3]
+        actual = []
         endpoints.size.times do
-          endpoint, yellow = policy.next
-          policy.bad(endpoint, 0, Time.now) unless endpoint == 4
+          endpoint, need_health_check = policy.next
+          if breaking_bad.call(endpoint)
+            # yellow is still viable but a health-check is requested.
+            need_health_check.should be_true
+            policy.bad(endpoint, 0, Time.now)  # go red
+          else
+            need_health_check.should be_false
+          end
           actual << endpoint
         end
+        actual.should == expected
 
+        # third pass; green or red, no more yellow.
+        expected = [4, 5, 4, 5, 4]
+        actual = []
+        endpoints.size.times do
+          endpoint, need_health_check = policy.next
+          need_health_check.should be_false
+          actual << endpoint
+        end
         actual.should == expected
       end
     end
@@ -350,8 +383,10 @@ describe RightSupport::Net::LB::HealthCheck do
   context :set_endpoints do
     context 'given endpoints stack does not exist' do
       before(:each) do
-        @policy = RightSupport::Net::LB::HealthCheck.new({
-                    :yellow_states => @yellow_states, :reset_time => @reset_time})
+        @policy = RightSupport::Net::LB::HealthCheck.new(
+          :yellow_states => @yellow_states,
+          :reset_time => @reset_time,
+          :health_check => lambda { |ep| true })
       end
 
       it 'acts as initializer' do

data/spec/net/{balancing/sticky_policy_spec.rb → lb/sticky_spec.rb}

@@ -26,7 +26,7 @@ describe RightSupport::Net::LB::Sticky do
     context 'given all servers are healthy' do
       it 'sticks to chosen one' do
         chance = 1.0
-        seen = find_empirical_distribution(@trials,@endpoints) do 
+        seen = find_empirical_distribution(@trials,@endpoints) do
           @policy.next
         end
         seen.each_pair do |_, count|

data/spec/net/request_balancer_spec.rb

@@ -36,7 +36,7 @@ describe RightSupport::Net::RequestBalancer do
 
     exception = expect.first
     count = expect.last
-    rb = RightSupport::Net::RequestBalancer.new([1,2,3], :fatal=>fatal)
+    rb = described_class.new([1,2,3], :fatal=>fatal)
     @tries = 0
 
     code = lambda do
@@ -72,10 +72,11 @@ describe RightSupport::Net::RequestBalancer do
 
     expect = number_of_endpoints
     yellow_states = 4
-    rb = RightSupport::Net::RequestBalancer.new((1..expect).to_a,
-                                                :policy => RightSupport::Net::LB::HealthCheck,
-                                                :health_check => test,
-                                                :yellow_states => yellow_states)
+    rb = described_class.new(
+      (1..expect).to_a,
+      :policy => RightSupport::Net::LB::HealthCheck,
+      :health_check => test,
+      :yellow_states => yellow_states)
     @health_checks = 0
     tries = 0
     l = lambda do
@@ -98,20 +99,25 @@ describe RightSupport::Net::RequestBalancer do
   context :initialize do
     it 'requires a list of endpoint URLs' do
       lambda do
-        RightSupport::Net::RequestBalancer.new(nil)
+        described_class.new(nil)
       end.should raise_exception(ArgumentError)
     end
 
     context 'with Integer :retry option' do
-      it 'stops after N total tries' do
-        lambda do
-          @tries = 0
-          RightSupport::Net::RequestBalancer.new([1, 2, 3], :retry=>1).request do |u|
-            @tries += 1
-            raise NoBigDeal
+      [1, 2].each do |retry_value|
+        context "and :retry=#{retry_value}" do
+          it "stops after #{retry_value} total tries" do
+            lambda do
+              # note the legacy :retry integer value has no backoff.
+              @tries = 0
+              described_class.new([1, 2, 3], :retry => retry_value).request do |u|
+                @tries += 1
+                raise NoBigDeal
+              end
+            end.should raise_error
+            @tries.should == retry_value
           end
-        end.should raise_error
-        @tries.should == 1
+        end
       end
     end
 
@@ -123,7 +129,7 @@ describe RightSupport::Net::RequestBalancer do
           @tries < 1
         end
 
-        balancer = RightSupport::Net::RequestBalancer.new([1, 2, 3], :retry => proc)
+        balancer = described_class.new([1, 2, 3], :retry => proc)
         lambda do
           balancer.request do |u|
             @tries += 1
@@ -133,12 +139,33 @@ describe RightSupport::Net::RequestBalancer do
 
         @tries.should == 1
       end
+
+      [1, 2].each do |max_attempts|
+        context "with backoff_retry_callback(#{max_attempts})" do
+          it "stops after #{max_attempts} total tries" do
+            start_time = ::Time.now
+            retry_proc = described_class.backoff_retry_callback(max_attempts)
+            lambda do
+              @tries = 0
+              described_class.new([1, 2, 3], :retry => retry_proc).request do |u|
+                @tries += 1
+                raise NoBigDeal
+              end
+            end.should raise_error
+            end_time = ::Time.now
+            @tries.should == max_attempts
+            if max_attempts == 2
+              ((end_time - start_time) >= 2).should be_true  # assert backoff
+            end
+          end
+        end
+      end
     end
 
     context ':fatal option' do
       it 'has reasonable defaults' do
-        exceptions = RightSupport::Net::RequestBalancer::DEFAULT_FATAL_EXCEPTIONS - [SignalException]
-        balancer = RightSupport::Net::RequestBalancer.new([1])
+        exceptions = described_class::DEFAULT_FATAL_EXCEPTIONS - [SignalException]
+        balancer = described_class.new([1])
         exceptions.each do |klass|
           lambda do
             balancer.request { |ep| raise klass }
@@ -150,13 +177,13 @@ describe RightSupport::Net::RequestBalancer do
         it 'validates the arity' do
           bad_lambda = lambda { |too, many, arguments| }
           lambda do
-            RightSupport::Net::RequestBalancer.new([1,2], :fatal=>bad_lambda)
+            described_class.new([1,2], :fatal=>bad_lambda)
           end.should raise_error(ArgumentError)
         end
 
         it 'delegates to the Proc' do
           always_retry = lambda { |e| false }
-          balancer = RightSupport::Net::RequestBalancer.new([1,2], :fatal=>always_retry)
+          balancer = described_class.new([1,2], :fatal=>always_retry)
 
           lambda do
             balancer.request do |ep|
@@ -174,7 +201,7 @@ describe RightSupport::Net::RequestBalancer do
 
       context 'with an Exception' do
         it 'considers that class of Exception to be fatal' do
-          balancer = RightSupport::Net::RequestBalancer.new([1], :fatal=>BigDeal)
+          balancer = described_class.new([1], :fatal=>BigDeal)
           lambda do
             balancer.request { |ep| raise BigDeal }
           end.should raise_error(BigDeal)
@@ -184,7 +211,7 @@ describe RightSupport::Net::RequestBalancer do
       context 'with an Array' do
         it 'considers any class in the array to be fatal' do
           exceptions = [ArgumentError, BigDeal]
-          balancer = RightSupport::Net::RequestBalancer.new([1], :fatal=>exceptions)
+          balancer = described_class.new([1], :fatal=>exceptions)
           exceptions.each do |klass|
             lambda do
               balancer.request { |ep| raise klass }
@@ -198,7 +225,7 @@ describe RightSupport::Net::RequestBalancer do
       it 'validates the arity' do
         bad_lambda = lambda { |way, too, many, arguments| }
         lambda do
-          RightSupport::Net::RequestBalancer.new([1,2], :on_exception=>bad_lambda)
+          described_class.new([1,2], :on_exception=>bad_lambda)
         end.should raise_error(ArgumentError)
       end
     end
@@ -207,23 +234,23 @@ describe RightSupport::Net::RequestBalancer do
       it 'accepts a Class' do
         policy = RightSupport::Net::LB::RoundRobin
         lambda {
-          RightSupport::Net::RequestBalancer.new([1,2], :policy=>policy)
+          described_class.new([1,2], :policy=>policy)
         }.should_not raise_error
       end
 
       it 'accepts an object' do
         policy = RightSupport::Net::LB::RoundRobin.new([1,2])
         lambda {
-          RightSupport::Net::RequestBalancer.new([1,2], :policy=>policy)
+          described_class.new([1,2], :policy=>policy)
         }.should_not raise_error
       end
 
       it 'checks for duck-type compatibility' do
         lambda {
-          RightSupport::Net::RequestBalancer.new([1,2], :policy=>String)
+          described_class.new([1,2], :policy=>String)
         }.should raise_error
         lambda {
-          RightSupport::Net::RequestBalancer.new([1,2], :policy=>'I like cheese')
+          described_class.new([1,2], :policy=>'I like cheese')
         }.should raise_error
       end
     end
@@ -235,12 +262,12 @@ describe RightSupport::Net::RequestBalancer do
 
       it 'accepts a block' do
         lambda {
-          RightSupport::Net::RequestBalancer.new([1,2], :health_check => @health_check)
+          described_class.new([1,2], :health_check => @health_check)
         }.should_not raise_error
       end
 
       it 'calls specified block' do
-        @balancer = RightSupport::Net::RequestBalancer.new([1,2], :health_check => @health_check)
+        @balancer = described_class.new([1,2], :health_check => @health_check)
         @options = @balancer.instance_variable_get("@options")
         @options[:health_check].call(1).should be_eql("HealthCheck passed for 1!")
       end
@@ -249,7 +276,7 @@ describe RightSupport::Net::RequestBalancer do
 
     context 'with default :health_check option' do
       it 'calls default block' do
-        @balancer = RightSupport::Net::RequestBalancer.new([1,2])
+        @balancer = described_class.new([1,2])
         @options = @balancer.instance_variable_get("@options")
         @options[:health_check].call(1).should be_true
       end
@@ -263,7 +290,7 @@ describe RightSupport::Net::RequestBalancer do
 
       it 'accepts a block' do
         lambda {
-          RightSupport::Net::RequestBalancer.new([1,2], :on_health_change => @on_health_change)
+          described_class.new([1,2], :on_health_change => @on_health_change)
         }.should_not raise_error
       end
     end
@@ -272,7 +299,7 @@ describe RightSupport::Net::RequestBalancer do
       before(:each) do
         flexmock(RightSupport::Net::DNS).should_receive(:resolve_with_hostnames).
             with(['host1', 'host2']).and_return({'host1' => make_endpoint(['1.1.1.1', '2.2.2.2']), 'host2' => make_endpoint(['3.3.3.3'])})
-        @balancer = RightSupport::Net::RequestBalancer.new(['host1', 'host2'], :resolve => 15)
+        @balancer = described_class.new(['host1', 'host2'], :resolve => 15)
       end
 
       it 'performs an initial resolution' do
@@ -291,26 +318,102 @@ describe RightSupport::Net::RequestBalancer do
   context :request do
     it 'requires a block' do
       lambda do
-        RightSupport::Net::RequestBalancer.new([1]).request
+        described_class.new([1]).request
       end.should raise_exception(ArgumentError)
     end
 
-    it 'retries until a request completes' do
-      list = [1,2,3,4,5,6,7,8,9,10]
+    context 'with a single thread' do
+      let(:list) { [1,2,3,4,5,6,7,8,9,10] }
+
+      subject { described_class.new(list) }
+
+      it 'retries until a request completes' do
+        queue = Queue.new
+        10.times do
+          x = subject.request do |l|
+            raise NoBigDeal, "Fall down go boom!" unless l == 5
+            l
+          end
+          x.should == 5
+        end
+      end
+    end
+
+    context 'with multiple threads and thread-safety enabled' do
+      let(:list) { [1,2,3,4,5,6,7,8,9,10] }
+
+      def boom(ep)
+        raise NoBigDeal, "Fall down go boom!" unless ep == 5
+        true
+      end
 
-      10.times do
-        x = RightSupport::Net::RequestBalancer.new(list).request do |l|
-          raise NoBigDeal, "Fall down go boom!" unless l == 5
-          l
+      subject do
+        described_class.new(
+          list,
+          thread_safe: true,
+          policy: ::RightSupport::Net::LB::HealthCheck,
+          health_check: method(:boom))
+      end
+
+      it 'retries until a request completes' do
+        count = list.size * 3
+        expected_responses = nil
+        expected_stats = list.inject({}) do |h, ep|
+          begin
+            boom(ep)
+            h[ep] = 'green'
+            expected_responses = [ep] * count
+          rescue NoBigDeal
+            h[ep] = 'red'
+          end
+          h
         end
 
-        x.should == 5
+        queue = Queue.new
+        count.times do
+          ::Thread.new do
+            stop_time = ::Time.now + 10
+            loop do
+              begin
+                x = subject.request do |ep|
+                  boom(ep)
+                  ep
+                end
+                queue.push(x)
+                break
+              rescue ::RightSupport::Net::NoResult => e
+                # this can happen with too much thread contention for failing
+                # endpoints. the issue is that other threads keep snaking the
+                # only good endpoint away from this thread (i.e. keep
+                # incrementing the counter in health-check policy) until the
+                # allowed number of retries is exhausted. only when the other
+                # threads die can we be sure of getting the only good endpoint.
+                #
+                # keep trying for the purpose of this test but do not allow
+                # infinite retries.
+                if ::Time.now >= stop_time
+                  queue.push(e)
+                  break
+                end
+              rescue ::Exception => e
+                queue.push(e)
+                break
+              end
+            end
+          end
+        end
+        actual_responses = []
+        count.times do
+          actual_responses << queue.pop
+        end
+        actual_responses.should == expected_responses
+        subject.get_stats.should == expected_stats
       end
     end
 
     it 'raises if no request completes' do
       lambda do
-        RightSupport::Net::RequestBalancer.request([1,2,3]) do |l|
+        described_class.request([1,2,3]) do |l|
           raise NoBigDeal, "Fall down go boom!"
         end
       end.should raise_exception(RightSupport::Net::NoResult, /NoBigDeal/)
@@ -349,7 +452,7 @@ describe RightSupport::Net::RequestBalancer do
     context 'with default :fatal option' do
       it 'retries most Ruby builtin errors' do
         list = [1,2,3,4,5,6,7,8,9,10]
-        rb = RightSupport::Net::RequestBalancer.new(list)
+        rb = described_class.new(list)
 
         [IOError, SystemCallError, SocketError].each do |klass|
           test_raise(nil, klass, [RightSupport::Net::NoResult, 3])
@@ -358,7 +461,7 @@ describe RightSupport::Net::RequestBalancer do
 
       it 'does not retry program errors' do
         list = [1,2,3,4,5,6,7,8,9,10]
-        rb = RightSupport::Net::RequestBalancer.new(list)
+        rb = described_class.new(list)
 
         [ArgumentError, LoadError, NameError].each do |klass|
           test_raise(nil, klass, [klass, 1])
@@ -372,7 +475,7 @@ describe RightSupport::Net::RequestBalancer do
 
       it 'does not retry HTTP 4xx other than timeout' do
         list = [1,2,3,4,5,6,7,8,9,10]
-        rb = RightSupport::Net::RequestBalancer.new(list)
+        rb = described_class.new(list)
 
         codes = [401, 402, 403, 404, 405, 406, 407, 409]
         codes.each do |code|
@@ -384,7 +487,7 @@ describe RightSupport::Net::RequestBalancer do
 
       context 'with default :retry option' do
         it 'marks endpoints as bad if they encounter retryable errors' do
-          rb = RightSupport::Net::RequestBalancer.new([1,2,3], :policy => RightSupport::Net::LB::HealthCheck, :health_check => Proc.new {|endpoint| false})
+          rb = described_class.new([1,2,3], :policy => RightSupport::Net::LB::HealthCheck, :health_check => Proc.new {|endpoint| false})
           expect = rb.get_stats
           codes = [401, 402, 403, 404, 405, 406, 407, 408, 409]
           codes.each do |code|
@@ -397,7 +500,7 @@ describe RightSupport::Net::RequestBalancer do
         end
 
         it 'does not mark endpoints as bad if they raise fatal errors' do
-          rb = RightSupport::Net::RequestBalancer.new([1,2,3], :policy => RightSupport::Net::LB::HealthCheck, :health_check => Proc.new {|endpoint| false})
+          rb = described_class.new([1,2,3], :policy => RightSupport::Net::LB::HealthCheck, :health_check => Proc.new {|endpoint| false})
           codes = [401, 402, 403, 404, 405, 406, 407, 409]
           codes.each do |code|
             lambda do
@@ -420,7 +523,7 @@ describe RightSupport::Net::RequestBalancer do
         @callback.should_receive(:respond_to?).with(:call).and_return(true)
         @callback.should_receive(:respond_to?).with(:arity).and_return(true)
         @callback.should_receive(:arity).and_return(3)
-        @rb = RightSupport::Net::RequestBalancer.new(@list, :fatal=>BigDeal, :on_exception=>@callback)
+        @rb = described_class.new(@list, :fatal=>BigDeal, :on_exception=>@callback)
       end
 
       it 'calls me back with fatal exceptions' do
@@ -442,12 +545,12 @@ describe RightSupport::Net::RequestBalancer do
     context 'given a class-level logger' do
       before(:all) do
         @logger = Logger.new(StringIO.new)
-        RightSupport::Net::RequestBalancer.logger = @logger
+        described_class.logger = @logger
         RightSupport::Net::LB::HealthCheck.logger = @logger
       end
 
       after(:all) do
-        RightSupport::Net::RequestBalancer.logger = nil
+        described_class.logger = nil
       end
 
       context 'when a retryable exception is raised' do
@@ -455,7 +558,7 @@ describe RightSupport::Net::RequestBalancer do
           flexmock(@logger).should_receive(:error).times(4)
 
           lambda {
-            balancer = RightSupport::Net::RequestBalancer.new([1,2,3])
+            balancer = described_class.new([1,2,3])
             balancer.request do |ep|
               raise NoBigDeal, "Too many cows on the moon"
             end
@@ -471,7 +574,7 @@ describe RightSupport::Net::RequestBalancer do
           flexmock(@logger).should_receive(:info).times(8)
 
           lambda {
-            balancer = RightSupport::Net::RequestBalancer.new([1,2,3,4], :policy => RightSupport::Net::LB::HealthCheck, :health_check => health_check)
+            balancer = described_class.new([1,2,3,4], :policy => RightSupport::Net::LB::HealthCheck, :health_check => health_check)
             balancer.request do |ep|
               raise "Bad Endpoint"
             end
@@ -498,7 +601,7 @@ describe RightSupport::Net::RequestBalancer do
 
       it 'resolves ip addresses for specified list of endpoints' do
         @dns.should_receive(:resolve_with_hostnames).with(@endpoints).and_return(@resolved_set_1)
-        @rb = RightSupport::Net::RequestBalancer.new(@endpoints, :resolve => 15)
+        @rb = described_class.new(@endpoints, :resolve => 15)
 
         @rb.request { true }
         @policy = @rb.instance_variable_get("@policy")
@@ -507,7 +610,7 @@ describe RightSupport::Net::RequestBalancer do
 
       it 're-resolves list of ip addresses if TTL is expired' do
         @dns.should_receive(:resolve_with_hostnames).with(@endpoints).twice.and_return(@resolved_set_1, @resolved_set_2)
-        @rb = RightSupport::Net::RequestBalancer.new(@endpoints, :resolve => 15)
+        @rb = described_class.new(@endpoints, :resolve => 15)
 
         @rb.request { true }
         @policy = @rb.instance_variable_get("@policy")
@@ -524,7 +627,7 @@ describe RightSupport::Net::RequestBalancer do
       before(:each) do
         @endpoints = [1,2,3,4,5,6]
         @exceptions = [BigDeal, NoBigDeal, OtherTestException]
-        @rb = RightSupport::Net::RequestBalancer.new(@endpoints)
+        @rb = described_class.new(@endpoints)
         @tries = 0
         begin
           @rb.request do |ep|
@@ -564,7 +667,7 @@ describe RightSupport::Net::RequestBalancer do
         expected_hash = {}
         list = [1,2,3,4]
         list.each { |k| expected_hash[k] = 'n/a' }
-        rb = RightSupport::Net::RequestBalancer.new(list)
+        rb = described_class.new(list)
 
         rb.get_stats.should_not be_nil
         rb.get_stats.should == expected_hash
@@ -575,9 +678,10 @@ describe RightSupport::Net::RequestBalancer do
       it 'returns stats in an endpoint-keyed hash' do
         expected_hash = {}
         list = [1,2,3,4]
-        rb = RightSupport::Net::RequestBalancer.new(list,
-                                                :policy => RightSupport::Net::LB::HealthCheck,
-                                                :health_check => Proc.new {|endpoint| "HealthCheck passed for #{endpoint}!"})
+        rb = described_class.new(
+          list,
+          :policy => RightSupport::Net::LB::HealthCheck,
+          :health_check => Proc.new {|endpoint| "HealthCheck passed for #{endpoint}!"})
         rb.get_stats.should_not be_nil
         rb.get_stats.should_not == expected_hash
       end