right_cloud_api_base 0.2.2 → 0.2.3
Sign up to get free protection for your applications and to get access to all the features.
- checksums.yaml +4 -4
- data/HISTORY +3 -0
- data/lib/base/api_manager.rb +8 -0
- data/lib/base/routines/retry_manager.rb +39 -28
- data/lib/right_cloud_api_base_version.rb +1 -1
- data/spec/routines/test_retry_manager_spec.rb +112 -4
- metadata +2 -2
checksums.yaml
CHANGED
@@ -1,7 +1,7 @@
|
|
1
1
|
---
|
2
2
|
SHA1:
|
3
|
-
metadata.gz:
|
4
|
-
data.tar.gz:
|
3
|
+
metadata.gz: 2fc2a22737e482014f01a5978e0250a4daac0b19
|
4
|
+
data.tar.gz: e334a9f77d2425bc111c33fa344532c521a62c26
|
5
5
|
SHA512:
|
6
|
-
metadata.gz:
|
7
|
-
data.tar.gz:
|
6
|
+
metadata.gz: a149f1eebecce88b4b3f96bcf33d59ab5d5d79adf9265a8ca0081f21e7cfdbd7465d6eaf94afe43a86fe066d1caa068dfe7a37367fd0e8edf063f75814c20103
|
7
|
+
data.tar.gz: b07ded92b3b379c9f7bdcb84b766bc51ac1b5d009ad0efde50753c578bc9017c9e81165a11ab3d7b4b39a49481cf6f128e762a655d88146cb3b98afe60567db6
|
data/HISTORY
CHANGED
data/lib/base/api_manager.rb
CHANGED
@@ -191,6 +191,14 @@ module RightScale
|
|
191
191
|
# Current logger. If is not provided then it logs to STDOUT. When if nil is given it
|
192
192
|
# logs to '/dev/nul'.
|
193
193
|
#
|
194
|
+
# @option options [Hash] :retry
|
195
|
+
# A set of options for how retry behavior works.
|
196
|
+
# available retry options
|
197
|
+
# :count => Integer # number of retry attempts
|
198
|
+
# :strategy => Symbol # retry strategy[ :exponentional (default), :full_jitter, :equal_jitter, :decorrelated_jitter]
|
199
|
+
# :reiteration_time => Integer # maximum amount of time to allow for retries
|
200
|
+
# :sleep_time => Integer # base sleep time in seconds, actual sleep time depends on strategy and count
|
201
|
+
#
|
194
202
|
# @option options [Symbol] :log_filter_patterns
|
195
203
|
# A set of log filters that define what to log (see {RightScale::CloudApi::CloudApiLogger}).
|
196
204
|
#
|
@@ -47,26 +47,24 @@ module RightScale
|
|
47
47
|
# 1. There was a redirect request (HTTP 3xx code)
|
48
48
|
# 2. There was an error (HTTP 5xx, 4xx) and
|
49
49
|
#
|
50
|
+
# Adding strategies [ :full_jitter, :equal_jitter, :decorrelated_jitter]
|
51
|
+
# see http://www.awsarchitectureblog.com/2015/03/backoff.html for details
|
50
52
|
def process
|
51
|
-
retry_options = data[:options][:retry] || {}
|
53
|
+
retry_options = @data[:options][:retry] || {}
|
54
|
+
retry_strategy = retry_options[:strategy] # nil or garbage is acceptable
|
52
55
|
max_retry_count = retry_options[:count] || DEFAULT_RETRY_COUNT
|
53
56
|
reiteration_time = retry_options[:reiteration_time] || DEFAULT_REITERATION_TIME
|
54
|
-
|
55
|
-
|
57
|
+
base_sleep_time = retry_options[:sleep_time] || DEFAULT_SLEEP_TIME
|
58
|
+
|
56
59
|
# Initialize things on the first run for the current request.
|
57
|
-
|
58
|
-
|
59
|
-
|
60
|
-
|
61
|
-
|
62
|
-
|
63
|
-
else
|
64
|
-
# Increment retry attempts count
|
65
|
-
data[:vars][:retry][:count] += 1
|
66
|
-
end
|
67
|
-
|
60
|
+
@data[:vars][:retry] ||= {}
|
61
|
+
@data[:vars][:retry][:count] ||= -1
|
62
|
+
@data[:vars][:retry][:count] += 1 # Increment retry attempts count
|
63
|
+
@data[:vars][:retry][:orig_body_stream_pos] ||= @data[:request][:body].is_a?(IO) && @data[:request][:body].pos
|
64
|
+
|
65
|
+
attempt = @data[:vars][:retry][:count]
|
68
66
|
# Complain on any issue
|
69
|
-
if max_retry_count <
|
67
|
+
if max_retry_count < attempt
|
70
68
|
error_message = "RetryManager: No more retries left."
|
71
69
|
elsif Time.now > @data[:vars][:system][:started_at] + reiteration_time
|
72
70
|
error_message = "RetryManager: Retry timeout of #{reiteration_time} seconds has been reached."
|
@@ -82,22 +80,35 @@ module RightScale
|
|
82
80
|
end
|
83
81
|
|
84
82
|
# Continue (with a delay when needed)
|
85
|
-
if
|
86
|
-
|
87
|
-
|
88
|
-
|
89
|
-
|
90
|
-
|
83
|
+
if attempt > 0 #only sleep on a retry
|
84
|
+
previous_sleep = @data[:vars][:retry][:previous_sleep_time] || base_sleep_time
|
85
|
+
sleep_time = case retry_strategy
|
86
|
+
when :full_jitter
|
87
|
+
#sleep = random_between(0, base * 2 ** attempt)
|
88
|
+
rand * (base_sleep_time * 2**(attempt-1))
|
89
|
+
when :equal_jitter
|
90
|
+
# sleep = temp / 2 + random_between(0, temp / 2)
|
91
|
+
temp = base_sleep_time * 2 **(attempt-1)
|
92
|
+
temp / 2 + rand * (temp / 2)
|
93
|
+
when :decorrelated_jitter
|
94
|
+
# sleep = random_between(base, previous_sleep * 3)
|
95
|
+
rand * (3*previous_sleep - base_sleep_time) + base_sleep_time
|
96
|
+
else # default behavior, exponential
|
97
|
+
base_sleep_time * 2**(attempt-1)
|
98
|
+
end
|
99
|
+
@data[:vars][:retry][:previous_sleep_time] = sleep_time
|
100
|
+
cloud_api_logger.log("Sleeping for #{sleep_time} seconds before retry attempt ##{attempt}", :retry_manager)
|
101
|
+
sleep(sleep_time)
|
91
102
|
end
|
92
103
|
|
93
104
|
# Restore file pointer in IO body case.
|
94
|
-
if data[:request][:instance] &&
|
95
|
-
data[:request][:instance].is_io? &&
|
96
|
-
data[:request][:instance].body.respond_to?('pos') &&
|
97
|
-
data[:request][:instance].body.respond_to?('pos=') &&
|
98
|
-
data[:request][:instance].body.pos != data[:vars][:retry][:orig_body_stream_pos]
|
99
|
-
cloud_api_logger.log("Restoring file position to #{data[:vars][:retry][:orig_body_stream_pos]}", :retry_manager)
|
100
|
-
data[:request][:instance].body.pos = data[:vars][:retry][:orig_body_stream_pos]
|
105
|
+
if @data[:request][:instance] &&
|
106
|
+
@data[:request][:instance].is_io? &&
|
107
|
+
@data[:request][:instance].body.respond_to?('pos') &&
|
108
|
+
@data[:request][:instance].body.respond_to?('pos=') &&
|
109
|
+
@data[:request][:instance].body.pos != @data[:vars][:retry][:orig_body_stream_pos]
|
110
|
+
cloud_api_logger.log("Restoring file position to #{@data[:vars][:retry][:orig_body_stream_pos]}", :retry_manager)
|
111
|
+
@data[:request][:instance].body.pos = @data[:vars][:retry][:orig_body_stream_pos]
|
101
112
|
end
|
102
113
|
end
|
103
114
|
end
|
@@ -50,21 +50,129 @@ describe "" do
|
|
50
50
|
end
|
51
51
|
|
52
52
|
context "RightScale::CloudApi::RetryManager" do
|
53
|
-
it "works" do
|
53
|
+
it "works - default exponential" do
|
54
54
|
# 1st run
|
55
55
|
@retrymanager.execute(@test_data)
|
56
56
|
expect(@test_data[:vars][:retry][:count]).to eq 0
|
57
|
-
expect(@test_data[:vars][:retry][:
|
57
|
+
expect(@test_data[:vars][:retry][:previous_sleep_time]).to eq nil
|
58
58
|
|
59
59
|
# 2nd run, +1 count *2 sleep
|
60
60
|
@retrymanager.execute(@test_data)
|
61
61
|
expect(@test_data[:vars][:retry][:count]).to eq 1
|
62
|
-
expect(@test_data[:vars][:retry][:
|
62
|
+
expect(@test_data[:vars][:retry][:previous_sleep_time]).to eq 0.2
|
63
63
|
|
64
64
|
# 3rd run, +1 count, *2 sleep
|
65
65
|
@retrymanager.execute(@test_data)
|
66
66
|
expect(@test_data[:vars][:retry][:count]).to eq 2
|
67
|
-
expect(@test_data[:vars][:retry][:
|
67
|
+
expect(@test_data[:vars][:retry][:previous_sleep_time]).to eq 0.4
|
68
|
+
|
69
|
+
#4th run, case 1: default error
|
70
|
+
default_rm_error = "RetryManager: No more retries left."
|
71
|
+
expect do
|
72
|
+
@retrymanager.execute(@test_data)
|
73
|
+
end.to raise_error(RightScale::CloudApi::RetryManager::Error, default_rm_error)
|
74
|
+
|
75
|
+
#4th run, case 2: cloud_error + default error
|
76
|
+
http_error = 'Banana.'
|
77
|
+
expectation = "#{http_error}\n#{default_rm_error}"
|
78
|
+
@test_data[:vars][:retry][:http] = { :code => 777, :message => http_error }
|
79
|
+
expect do
|
80
|
+
@retrymanager.execute(@test_data)
|
81
|
+
end.to raise_error(RightScale::CloudApi::RetryManager::Error, expectation)
|
82
|
+
end
|
83
|
+
|
84
|
+
it "works full jitter" do
|
85
|
+
@test_data[:options][:retry][:strategy] = :full_jitter
|
86
|
+
@test_data[:options][:retry][:count] = 3
|
87
|
+
|
88
|
+
@retrymanager.execute(@test_data)
|
89
|
+
expect(@test_data[:vars][:retry][:count]).to eq 0
|
90
|
+
expect(@test_data[:vars][:retry][:previous_sleep_time]).to eq nil
|
91
|
+
|
92
|
+
@retrymanager.execute(@test_data)
|
93
|
+
expect(@test_data[:vars][:retry][:count]).to eq 1
|
94
|
+
expect(@test_data[:vars][:retry][:previous_sleep_time]).to be_between(0,0.2)
|
95
|
+
|
96
|
+
@retrymanager.execute(@test_data)
|
97
|
+
expect(@test_data[:vars][:retry][:count]).to eq 2
|
98
|
+
expect(@test_data[:vars][:retry][:previous_sleep_time]).to be_between(0,0.4)
|
99
|
+
|
100
|
+
@retrymanager.execute(@test_data)
|
101
|
+
expect(@test_data[:vars][:retry][:count]).to eq 3
|
102
|
+
expect(@test_data[:vars][:retry][:previous_sleep_time]).to be_between(0,0.8)
|
103
|
+
|
104
|
+
#4th run, case 1: default error
|
105
|
+
default_rm_error = "RetryManager: No more retries left."
|
106
|
+
expect do
|
107
|
+
@retrymanager.execute(@test_data)
|
108
|
+
end.to raise_error(RightScale::CloudApi::RetryManager::Error, default_rm_error)
|
109
|
+
|
110
|
+
#4th run, case 2: cloud_error + default error
|
111
|
+
http_error = 'Banana.'
|
112
|
+
expectation = "#{http_error}\n#{default_rm_error}"
|
113
|
+
@test_data[:vars][:retry][:http] = { :code => 777, :message => http_error }
|
114
|
+
expect do
|
115
|
+
@retrymanager.execute(@test_data)
|
116
|
+
end.to raise_error(RightScale::CloudApi::RetryManager::Error, expectation)
|
117
|
+
end
|
118
|
+
|
119
|
+
it "works equal jitter" do
|
120
|
+
@test_data[:options][:retry][:strategy] = :equal_jitter
|
121
|
+
@test_data[:options][:retry][:count] = 3
|
122
|
+
|
123
|
+
@retrymanager.execute(@test_data)
|
124
|
+
expect(@test_data[:vars][:retry][:count]).to eq 0
|
125
|
+
expect(@test_data[:vars][:retry][:previous_sleep_time]).to eq nil
|
126
|
+
|
127
|
+
@retrymanager.execute(@test_data)
|
128
|
+
expect(@test_data[:vars][:retry][:count]).to eq 1
|
129
|
+
expect(@test_data[:vars][:retry][:previous_sleep_time]).to be_between(0.1,0.2)
|
130
|
+
|
131
|
+
@retrymanager.execute(@test_data)
|
132
|
+
expect(@test_data[:vars][:retry][:count]).to eq 2
|
133
|
+
expect(@test_data[:vars][:retry][:previous_sleep_time]).to be_between(0.2,0.4)
|
134
|
+
|
135
|
+
@retrymanager.execute(@test_data)
|
136
|
+
expect(@test_data[:vars][:retry][:count]).to eq 3
|
137
|
+
expect(@test_data[:vars][:retry][:previous_sleep_time]).to be_between(0.4,0.8)
|
138
|
+
|
139
|
+
#4th run, case 1: default error
|
140
|
+
default_rm_error = "RetryManager: No more retries left."
|
141
|
+
expect do
|
142
|
+
@retrymanager.execute(@test_data)
|
143
|
+
end.to raise_error(RightScale::CloudApi::RetryManager::Error, default_rm_error)
|
144
|
+
|
145
|
+
#4th run, case 2: cloud_error + default error
|
146
|
+
http_error = 'Banana.'
|
147
|
+
expectation = "#{http_error}\n#{default_rm_error}"
|
148
|
+
@test_data[:vars][:retry][:http] = { :code => 777, :message => http_error }
|
149
|
+
expect do
|
150
|
+
@retrymanager.execute(@test_data)
|
151
|
+
end.to raise_error(RightScale::CloudApi::RetryManager::Error, expectation)
|
152
|
+
end
|
153
|
+
|
154
|
+
it "works decorrelated jitter" do
|
155
|
+
@test_data[:options][:retry][:strategy] = :decorrelated_jitter
|
156
|
+
@test_data[:options][:retry][:count] = 3
|
157
|
+
|
158
|
+
@retrymanager.execute(@test_data)
|
159
|
+
expect(@test_data[:vars][:retry][:count]).to eq 0
|
160
|
+
expect(@test_data[:vars][:retry][:previous_sleep_time]).to eq nil
|
161
|
+
|
162
|
+
@retrymanager.execute(@test_data)
|
163
|
+
expect(@test_data[:vars][:retry][:count]).to eq 1
|
164
|
+
expect(@test_data[:vars][:retry][:previous_sleep_time]).to be_between(0.2,0.6)
|
165
|
+
|
166
|
+
previous_sleep_time = @test_data[:vars][:retry][:previous_sleep_time]
|
167
|
+
@retrymanager.execute(@test_data)
|
168
|
+
expect(@test_data[:vars][:retry][:count]).to eq 2
|
169
|
+
expect(@test_data[:vars][:retry][:previous_sleep_time]).to be_between(0.2,3*previous_sleep_time)
|
170
|
+
previous_sleep_time = @test_data[:vars][:retry][:previous_sleep_time]
|
171
|
+
|
172
|
+
previous_sleep_time = @test_data[:vars][:retry][:previous_sleep_time]
|
173
|
+
@retrymanager.execute(@test_data)
|
174
|
+
expect(@test_data[:vars][:retry][:count]).to eq 3
|
175
|
+
expect(@test_data[:vars][:retry][:previous_sleep_time]).to be_between(0.2,3*previous_sleep_time)
|
68
176
|
|
69
177
|
#4th run, case 1: default error
|
70
178
|
default_rm_error = "RetryManager: No more retries left."
|
metadata
CHANGED
@@ -1,14 +1,14 @@
|
|
1
1
|
--- !ruby/object:Gem::Specification
|
2
2
|
name: right_cloud_api_base
|
3
3
|
version: !ruby/object:Gem::Version
|
4
|
-
version: 0.2.
|
4
|
+
version: 0.2.3
|
5
5
|
platform: ruby
|
6
6
|
authors:
|
7
7
|
- RightScale, Inc.
|
8
8
|
autorequire:
|
9
9
|
bindir: bin
|
10
10
|
cert_chain: []
|
11
|
-
date: 2015-
|
11
|
+
date: 2015-11-17 00:00:00.000000000 Z
|
12
12
|
dependencies:
|
13
13
|
- !ruby/object:Gem::Dependency
|
14
14
|
name: json
|