right_cloud_api_base 0.2.2 → 0.2.3

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: a15b1f9a44a3084eb718cb87639fb3c96c015157
-  data.tar.gz: 99545b9605f6d1b63c2ad1a138fd3ac614a9a811
+  metadata.gz: 2fc2a22737e482014f01a5978e0250a4daac0b19
+  data.tar.gz: e334a9f77d2425bc111c33fa344532c521a62c26
 SHA512:
-  metadata.gz: 710087ad760c2ec6b5e553e10848c4e54916f9568b957a28a579d1b1d7ca5613015a7604c58b3b07a43c3bbcd0bfbf5c89b7c08f9bf44761d38f576183a560fb
-  data.tar.gz: 24018fc8b936dc861528452f14e55adc4341297443cd1078c36f9bd03ea1ba359668135ae026e32325dd04633ad1c911ef595b3ce659ffa528f824cfa101f84e
+  metadata.gz: a149f1eebecce88b4b3f96bcf33d59ab5d5d79adf9265a8ca0081f21e7cfdbd7465d6eaf94afe43a86fe066d1caa068dfe7a37367fd0e8edf063f75814c20103
+  data.tar.gz: b07ded92b3b379c9f7bdcb84b766bc51ac1b5d009ad0efde50753c578bc9017c9e81165a11ab3d7b4b39a49481cf6f128e762a655d88146cb3b98afe60567db6

data/HISTORY

@@ -1,3 +1,6 @@
+== 2015-11-11, v0.2.3
+  - Add jitter style retry strategies to retry manager
+
 == 2015-03-11, v0.2.2
   - Fixed "Cannot do block-based chunk gets on S3" issue
     https://github.com/rightscale/right_aws_api/issues/24

data/lib/base/api_manager.rb

@@ -191,6 +191,14 @@ module RightScale
       #   Current logger. If is not provided then it logs to STDOUT. When if nil is given it
       #   logs to '/dev/nul'.
       #
+      # @option options [Hash] :retry
+      #   A set of options for how retry behavior works.
+      #   available retry options
+      #   :count => Integer # number of retry attempts
+      #   :strategy => Symbol # retry strategy[ :exponentional (default), :full_jitter, :equal_jitter, :decorrelated_jitter]
+      #   :reiteration_time => Integer # maximum amount of time to allow for retries
+      #   :sleep_time => Integer # base sleep time in seconds, actual sleep time depends on strategy and count
+      #
       # @option options [Symbol]  :log_filter_patterns
       #   A set of log filters that define what to log (see {RightScale::CloudApi::CloudApiLogger}).
       #

data/lib/base/routines/retry_manager.rb

@@ -47,26 +47,24 @@ module RightScale
       #  1. There was a redirect request (HTTP 3xx code)
       #  2. There was an error (HTTP 5xx, 4xx) and
       #
+      # Adding strategies [ :full_jitter, :equal_jitter, :decorrelated_jitter]
+      #   see http://www.awsarchitectureblog.com/2015/03/backoff.html for details
       def process
-        retry_options    = data[:options][:retry]           || {}
+        retry_options    = @data[:options][:retry]           || {}
+        retry_strategy   = retry_options[:strategy] # nil or garbage is acceptable
         max_retry_count  = retry_options[:count]            || DEFAULT_RETRY_COUNT
         reiteration_time = retry_options[:reiteration_time] || DEFAULT_REITERATION_TIME
-        sleep_time       = retry_options[:sleep_time]       || DEFAULT_SLEEP_TIME
-        
+        base_sleep_time  = retry_options[:sleep_time]       || DEFAULT_SLEEP_TIME
+
         # Initialize things on the first run for the current request.
-        unless data[:vars][:retry]
-          data[:vars][:retry]              = {}
-          data[:vars][:retry][:count]      = 0
-          data[:vars][:retry][:sleep_time] = 0
-          # if body is a IO object - remember its initial position in a file
-          data[:vars][:retry][:orig_body_stream_pos] = data[:request][:body].is_a?(IO) && data[:request][:body].pos
-        else
-          # Increment retry attempts count
-          data[:vars][:retry][:count] += 1
-        end
-        
+        @data[:vars][:retry] ||= {}
+        @data[:vars][:retry][:count] ||= -1        
+        @data[:vars][:retry][:count] += 1  # Increment retry attempts count
+        @data[:vars][:retry][:orig_body_stream_pos] ||= @data[:request][:body].is_a?(IO) && @data[:request][:body].pos
+
+        attempt = @data[:vars][:retry][:count]        
         # Complain on any issue
-        if max_retry_count < @data[:vars][:retry][:count]
+        if max_retry_count < attempt
           error_message = "RetryManager: No more retries left."
         elsif Time.now > @data[:vars][:system][:started_at] + reiteration_time
           error_message = "RetryManager: Retry timeout of #{reiteration_time} seconds has been reached."
@@ -82,22 +80,35 @@ module RightScale
         end
 
         # Continue (with a delay when needed)
-        if data[:vars][:retry][:sleep_time] > 0
-          cloud_api_logger.log("Sleeping for #{data[:vars][:retry][:sleep_time]} seconds before retry attempt ##{data[:vars][:retry][:count]}", :retry_manager)
-          sleep data[:vars][:retry][:sleep_time]
-          data[:vars][:retry][:sleep_time] *= 2
-        else
-          data[:vars][:retry][:sleep_time] = sleep_time
+        if attempt > 0 #only sleep on a retry
+          previous_sleep = @data[:vars][:retry][:previous_sleep_time] || base_sleep_time
+          sleep_time = case retry_strategy
+                       when :full_jitter
+                         #sleep = random_between(0, base * 2 ** attempt)
+                         rand * (base_sleep_time * 2**(attempt-1))
+                       when :equal_jitter
+                         # sleep = temp / 2 + random_between(0, temp / 2)
+                         temp = base_sleep_time * 2 **(attempt-1)
+                         temp / 2 + rand * (temp / 2)
+                       when :decorrelated_jitter
+                         # sleep = random_between(base, previous_sleep * 3)
+                         rand * (3*previous_sleep - base_sleep_time) + base_sleep_time
+                       else # default behavior, exponential
+                         base_sleep_time * 2**(attempt-1)
+                       end
+          @data[:vars][:retry][:previous_sleep_time] = sleep_time
+          cloud_api_logger.log("Sleeping for #{sleep_time} seconds before retry attempt ##{attempt}", :retry_manager)
+          sleep(sleep_time)
         end
         
         # Restore file pointer in IO body case.
-        if data[:request][:instance]                          &&
-           data[:request][:instance].is_io?                   &&
-           data[:request][:instance].body.respond_to?('pos')  &&
-           data[:request][:instance].body.respond_to?('pos=') &&
-           data[:request][:instance].body.pos != data[:vars][:retry][:orig_body_stream_pos]
-          cloud_api_logger.log("Restoring file position to #{data[:vars][:retry][:orig_body_stream_pos]}", :retry_manager)
-          data[:request][:instance].body.pos = data[:vars][:retry][:orig_body_stream_pos]
+        if @data[:request][:instance]                          &&
+           @data[:request][:instance].is_io?                   &&
+           @data[:request][:instance].body.respond_to?('pos')  &&
+           @data[:request][:instance].body.respond_to?('pos=') &&
+           @data[:request][:instance].body.pos != @data[:vars][:retry][:orig_body_stream_pos]
+          cloud_api_logger.log("Restoring file position to #{@data[:vars][:retry][:orig_body_stream_pos]}", :retry_manager)
+          @data[:request][:instance].body.pos = @data[:vars][:retry][:orig_body_stream_pos]
         end
       end
     end

data/lib/right_cloud_api_base_version.rb

@@ -31,7 +31,7 @@ module RightScale
     # CloudApi gem version namespace
     module VERSION
       # The gem version
-      STRING = '0.2.2'
+      STRING = '0.2.3'
     end
   end
 end

data/spec/routines/test_retry_manager_spec.rb

@@ -50,21 +50,129 @@ describe "" do
   end
 
   context "RightScale::CloudApi::RetryManager" do
-    it "works" do
+    it "works - default exponential" do
       # 1st run
       @retrymanager.execute(@test_data)
       expect(@test_data[:vars][:retry][:count]).to eq 0
-      expect(@test_data[:vars][:retry][:sleep_time]).to eq 0.2
+      expect(@test_data[:vars][:retry][:previous_sleep_time]).to eq nil
 
       # 2nd run, +1 count *2 sleep
       @retrymanager.execute(@test_data)
       expect(@test_data[:vars][:retry][:count]).to eq 1
-      expect(@test_data[:vars][:retry][:sleep_time]).to eq 0.4
+      expect(@test_data[:vars][:retry][:previous_sleep_time]).to eq 0.2
 
       # 3rd run, +1 count, *2 sleep
       @retrymanager.execute(@test_data)
       expect(@test_data[:vars][:retry][:count]).to eq 2
-      expect(@test_data[:vars][:retry][:sleep_time]).to eq 0.8
+      expect(@test_data[:vars][:retry][:previous_sleep_time]).to eq 0.4
+
+      #4th run, case 1: default error
+      default_rm_error = "RetryManager: No more retries left."
+      expect do
+        @retrymanager.execute(@test_data)
+      end.to raise_error(RightScale::CloudApi::RetryManager::Error, default_rm_error)
+
+      #4th run, case 2: cloud_error + default error
+      http_error  = 'Banana.'
+      expectation = "#{http_error}\n#{default_rm_error}"
+      @test_data[:vars][:retry][:http] = { :code => 777, :message => http_error }
+      expect do
+        @retrymanager.execute(@test_data)
+      end.to raise_error(RightScale::CloudApi::RetryManager::Error, expectation)
+    end
+
+    it "works full jitter" do
+      @test_data[:options][:retry][:strategy] = :full_jitter
+      @test_data[:options][:retry][:count] = 3
+      
+      @retrymanager.execute(@test_data)
+      expect(@test_data[:vars][:retry][:count]).to eq 0
+      expect(@test_data[:vars][:retry][:previous_sleep_time]).to eq nil
+
+      @retrymanager.execute(@test_data)
+      expect(@test_data[:vars][:retry][:count]).to eq 1
+      expect(@test_data[:vars][:retry][:previous_sleep_time]).to be_between(0,0.2)
+
+      @retrymanager.execute(@test_data)
+      expect(@test_data[:vars][:retry][:count]).to eq 2
+      expect(@test_data[:vars][:retry][:previous_sleep_time]).to be_between(0,0.4)
+
+      @retrymanager.execute(@test_data)
+      expect(@test_data[:vars][:retry][:count]).to eq 3
+      expect(@test_data[:vars][:retry][:previous_sleep_time]).to be_between(0,0.8)
+
+      #4th run, case 1: default error
+      default_rm_error = "RetryManager: No more retries left."
+      expect do
+        @retrymanager.execute(@test_data)
+      end.to raise_error(RightScale::CloudApi::RetryManager::Error, default_rm_error)
+
+      #4th run, case 2: cloud_error + default error
+      http_error  = 'Banana.'
+      expectation = "#{http_error}\n#{default_rm_error}"
+      @test_data[:vars][:retry][:http] = { :code => 777, :message => http_error }
+      expect do
+        @retrymanager.execute(@test_data)
+      end.to raise_error(RightScale::CloudApi::RetryManager::Error, expectation)
+    end
+
+    it "works equal jitter" do
+      @test_data[:options][:retry][:strategy] = :equal_jitter
+      @test_data[:options][:retry][:count] = 3
+      
+      @retrymanager.execute(@test_data)
+      expect(@test_data[:vars][:retry][:count]).to eq 0
+      expect(@test_data[:vars][:retry][:previous_sleep_time]).to eq nil
+
+      @retrymanager.execute(@test_data)
+      expect(@test_data[:vars][:retry][:count]).to eq 1
+      expect(@test_data[:vars][:retry][:previous_sleep_time]).to be_between(0.1,0.2)
+
+      @retrymanager.execute(@test_data)
+      expect(@test_data[:vars][:retry][:count]).to eq 2
+      expect(@test_data[:vars][:retry][:previous_sleep_time]).to be_between(0.2,0.4)
+
+      @retrymanager.execute(@test_data)
+      expect(@test_data[:vars][:retry][:count]).to eq 3
+      expect(@test_data[:vars][:retry][:previous_sleep_time]).to be_between(0.4,0.8)
+
+      #4th run, case 1: default error
+      default_rm_error = "RetryManager: No more retries left."
+      expect do
+        @retrymanager.execute(@test_data)
+      end.to raise_error(RightScale::CloudApi::RetryManager::Error, default_rm_error)
+
+      #4th run, case 2: cloud_error + default error
+      http_error  = 'Banana.'
+      expectation = "#{http_error}\n#{default_rm_error}"
+      @test_data[:vars][:retry][:http] = { :code => 777, :message => http_error }
+      expect do
+        @retrymanager.execute(@test_data)
+      end.to raise_error(RightScale::CloudApi::RetryManager::Error, expectation)
+    end
+
+    it "works decorrelated jitter" do
+      @test_data[:options][:retry][:strategy] = :decorrelated_jitter
+      @test_data[:options][:retry][:count] = 3
+      
+      @retrymanager.execute(@test_data)
+      expect(@test_data[:vars][:retry][:count]).to eq 0
+      expect(@test_data[:vars][:retry][:previous_sleep_time]).to eq nil
+
+      @retrymanager.execute(@test_data)
+      expect(@test_data[:vars][:retry][:count]).to eq 1
+      expect(@test_data[:vars][:retry][:previous_sleep_time]).to be_between(0.2,0.6)
+
+      previous_sleep_time = @test_data[:vars][:retry][:previous_sleep_time]
+      @retrymanager.execute(@test_data)
+      expect(@test_data[:vars][:retry][:count]).to eq 2
+      expect(@test_data[:vars][:retry][:previous_sleep_time]).to be_between(0.2,3*previous_sleep_time)
+      previous_sleep_time = @test_data[:vars][:retry][:previous_sleep_time]
+
+      previous_sleep_time = @test_data[:vars][:retry][:previous_sleep_time]
+      @retrymanager.execute(@test_data)
+      expect(@test_data[:vars][:retry][:count]).to eq 3
+      expect(@test_data[:vars][:retry][:previous_sleep_time]).to be_between(0.2,3*previous_sleep_time)
 
       #4th run, case 1: default error
       default_rm_error = "RetryManager: No more retries left."

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: right_cloud_api_base
 version: !ruby/object:Gem::Version
-  version: 0.2.2
+  version: 0.2.3
 platform: ruby
 authors:
 - RightScale, Inc.
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-03-12 00:00:00.000000000 Z
+date: 2015-11-17 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: json