right_cloud_api_base 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- checksums.yaml +7 -0
- data/HISTORY +2 -0
- data/LICENSE +19 -0
- data/README.md +14 -0
- data/Rakefile +37 -0
- data/lib/base/api_manager.rb +707 -0
- data/lib/base/helpers/cloud_api_logger.rb +214 -0
- data/lib/base/helpers/http_headers.rb +239 -0
- data/lib/base/helpers/http_parent.rb +103 -0
- data/lib/base/helpers/http_request.rb +173 -0
- data/lib/base/helpers/http_response.rb +122 -0
- data/lib/base/helpers/net_http_patch.rb +31 -0
- data/lib/base/helpers/query_api_patterns.rb +862 -0
- data/lib/base/helpers/support.rb +270 -0
- data/lib/base/helpers/support.xml.rb +306 -0
- data/lib/base/helpers/utils.rb +380 -0
- data/lib/base/manager.rb +122 -0
- data/lib/base/parsers/json.rb +38 -0
- data/lib/base/parsers/plain.rb +36 -0
- data/lib/base/parsers/rexml.rb +83 -0
- data/lib/base/parsers/sax.rb +200 -0
- data/lib/base/routines/cache_validator.rb +184 -0
- data/lib/base/routines/connection_proxies/net_http_persistent_proxy.rb +194 -0
- data/lib/base/routines/connection_proxies/right_http_connection_proxy.rb +224 -0
- data/lib/base/routines/connection_proxy.rb +66 -0
- data/lib/base/routines/request_analyzer.rb +122 -0
- data/lib/base/routines/request_generator.rb +48 -0
- data/lib/base/routines/request_initializer.rb +52 -0
- data/lib/base/routines/response_analyzer.rb +152 -0
- data/lib/base/routines/response_parser.rb +79 -0
- data/lib/base/routines/result_wrapper.rb +75 -0
- data/lib/base/routines/retry_manager.rb +106 -0
- data/lib/base/routines/routine.rb +98 -0
- data/lib/right_cloud_api_base.rb +72 -0
- data/lib/right_cloud_api_base_version.rb +37 -0
- data/right_cloud_api_base.gemspec +63 -0
- data/spec/helpers/query_api_pattern_spec.rb +312 -0
- data/spec/helpers/support_spec.rb +211 -0
- data/spec/helpers/support_xml_spec.rb +207 -0
- data/spec/helpers/utils_spec.rb +179 -0
- data/spec/routines/connection_proxies/test_net_http_persistent_proxy_spec.rb +143 -0
- data/spec/routines/test_cache_validator_spec.rb +152 -0
- data/spec/routines/test_connection_proxy_spec.rb +44 -0
- data/spec/routines/test_request_analyzer_spec.rb +106 -0
- data/spec/routines/test_response_analyzer_spec.rb +132 -0
- data/spec/routines/test_response_parser_spec.rb +228 -0
- data/spec/routines/test_result_wrapper_spec.rb +63 -0
- data/spec/routines/test_retry_manager_spec.rb +84 -0
- data/spec/spec_helper.rb +15 -0
- metadata +215 -0
@@ -0,0 +1,200 @@
|
|
1
|
+
#--
|
2
|
+
# Copyright (c) 2013 RightScale, Inc.
|
3
|
+
#
|
4
|
+
# Permission is hereby granted, free of charge, to any person obtaining
|
5
|
+
# a copy of this software and associated documentation files (the
|
6
|
+
# 'Software'), to deal in the Software without restriction, including
|
7
|
+
# without limitation the rights to use, copy, modify, merge, publish,
|
8
|
+
# distribute, sublicense, and/or sell copies of the Software, and to
|
9
|
+
# permit persons to whom the Software is furnished to do so, subject to
|
10
|
+
# the following conditions:
|
11
|
+
#
|
12
|
+
# The above copyright notice and this permission notice shall be
|
13
|
+
# included in all copies or substantial portions of the Software.
|
14
|
+
#
|
15
|
+
# THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND,
|
16
|
+
# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
17
|
+
# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
|
18
|
+
# IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
|
19
|
+
# CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
|
20
|
+
# TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
|
21
|
+
# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
22
|
+
#++
|
23
|
+
|
24
|
+
require 'xml/libxml'
|
25
|
+
|
26
|
+
module RightScale
|
27
|
+
|
28
|
+
module CloudApi
|
29
|
+
module Parser
|
30
|
+
|
31
|
+
class Sax
|
32
|
+
|
33
|
+
UTF_8_STR = "UTF-8"
|
34
|
+
TEXT_MARK = "@@text"
|
35
|
+
|
36
|
+
def self.parse(input, options = {})
|
37
|
+
# Parse the xml text
|
38
|
+
# http://libxml.rubyforge.org/rdoc/
|
39
|
+
xml_context = ::XML::Parser::Context.string(input)
|
40
|
+
xml_context.encoding = ::XML::Encoding::UTF_8 if options[:encoding] == UTF_8_STR
|
41
|
+
sax_parser = ::XML::SaxParser.new(xml_context)
|
42
|
+
sax_parser.callbacks = new(options)
|
43
|
+
sax_parser.parse
|
44
|
+
sax_parser.callbacks.result
|
45
|
+
end
|
46
|
+
|
47
|
+
|
48
|
+
def initialize(options = {})
|
49
|
+
@tag = {}
|
50
|
+
@path = []
|
51
|
+
@str_path = []
|
52
|
+
@options = options
|
53
|
+
@cached_strings = {}
|
54
|
+
end
|
55
|
+
|
56
|
+
|
57
|
+
def result
|
58
|
+
@cached_strings.clear
|
59
|
+
@tag
|
60
|
+
end
|
61
|
+
|
62
|
+
|
63
|
+
def cache_string(name)
|
64
|
+
unless @cached_strings[name]
|
65
|
+
name = name.freeze
|
66
|
+
@cached_strings[name] = name
|
67
|
+
end
|
68
|
+
@cached_strings[name]
|
69
|
+
end
|
70
|
+
|
71
|
+
|
72
|
+
# Callbacks
|
73
|
+
|
74
|
+
def on_error(msg)
|
75
|
+
fail msg
|
76
|
+
end
|
77
|
+
|
78
|
+
|
79
|
+
def on_start_element_ns(name, attr_hash, prefix, uri, namespaces)
|
80
|
+
name = cache_string(name)
|
81
|
+
# Push parent tag
|
82
|
+
@path << @tag
|
83
|
+
# Create a new tag
|
84
|
+
if @tag[name]
|
85
|
+
@tag[name] = [ @tag[name] ] unless @tag[name].is_a?(Array)
|
86
|
+
@tag[name] << {}
|
87
|
+
@tag = @tag[name].last
|
88
|
+
else
|
89
|
+
@tag[name] = {}
|
90
|
+
@tag = @tag[name]
|
91
|
+
end
|
92
|
+
# Put attributes
|
93
|
+
current_namespaces = Array(namespaces.keys)
|
94
|
+
current_namespaces << nil if current_namespaces._blank?
|
95
|
+
attr_hash.each do |key, value|
|
96
|
+
current_namespaces.each do |namespace|
|
97
|
+
namespace = namespace ? "#{namespace}:" : ''
|
98
|
+
namespace_and_key = cache_string("@#{namespace}#{key}")
|
99
|
+
@tag[namespace_and_key] = value
|
100
|
+
end
|
101
|
+
end
|
102
|
+
# Put namespaces
|
103
|
+
namespaces.each do |key, value|
|
104
|
+
namespace = cache_string(key ? "@xmlns:#{key}" : '@xmlns')
|
105
|
+
@tag[namespace] = value
|
106
|
+
end
|
107
|
+
end
|
108
|
+
|
109
|
+
|
110
|
+
def on_characters(chars)
|
111
|
+
# Ignore lines that contains white spaces only
|
112
|
+
return if chars[/\A\s*\z/m]
|
113
|
+
# Put Text
|
114
|
+
if @options[:encoding] == UTF_8_STR
|
115
|
+
# setting the encoding in context doesn't work(open issue with libxml-ruby).
|
116
|
+
# force encode as a work around.
|
117
|
+
# TODO remove the force encoding when issue in libxml is fixed
|
118
|
+
chars = chars.force_encoding(UTF_8_STR) if chars.respond_to?(:force_encoding)
|
119
|
+
end
|
120
|
+
name = cache_string(TEXT_MARK)
|
121
|
+
(@tag[name] ||= '') << chars
|
122
|
+
end
|
123
|
+
|
124
|
+
|
125
|
+
def on_comment(msg)
|
126
|
+
# Put Comments
|
127
|
+
name = cache_string('@@comment')
|
128
|
+
(@tag[name] ||= '') << msg
|
129
|
+
end
|
130
|
+
|
131
|
+
|
132
|
+
def on_end_element_ns(name, prefix, uri)
|
133
|
+
name = cache_string(name)
|
134
|
+
# Finalize tag's text
|
135
|
+
if @tag.key?(TEXT_MARK) && @tag[TEXT_MARK].empty?
|
136
|
+
# Delete text if it is blank
|
137
|
+
@tag.delete(TEXT_MARK)
|
138
|
+
elsif @tag.keys.count == 0
|
139
|
+
# Set tag value to nil then the tag is blank
|
140
|
+
@tag = nil
|
141
|
+
elsif @tag.keys == [TEXT_MARK]
|
142
|
+
# Set tag value to string if it has no any other data
|
143
|
+
@tag = @tag[TEXT_MARK]
|
144
|
+
end
|
145
|
+
# Make sure we saved the changes
|
146
|
+
if @path.last[name].is_a?(Array)
|
147
|
+
# If it is an Array then update the very last item
|
148
|
+
@path.last[name][-1] = @tag
|
149
|
+
else
|
150
|
+
# Otherwise just replace the tag
|
151
|
+
@path.last[name] = @tag
|
152
|
+
end
|
153
|
+
# Pop parent tag
|
154
|
+
@tag = @path.pop
|
155
|
+
end
|
156
|
+
|
157
|
+
|
158
|
+
def on_start_document
|
159
|
+
end
|
160
|
+
|
161
|
+
|
162
|
+
def on_reference (name)
|
163
|
+
end
|
164
|
+
|
165
|
+
|
166
|
+
def on_processing_instruction(target, data)
|
167
|
+
end
|
168
|
+
|
169
|
+
|
170
|
+
def on_cdata_block(cdata)
|
171
|
+
end
|
172
|
+
|
173
|
+
|
174
|
+
def on_has_internal_subset()
|
175
|
+
end
|
176
|
+
|
177
|
+
|
178
|
+
def on_internal_subset(name, external_id, system_id)
|
179
|
+
end
|
180
|
+
|
181
|
+
|
182
|
+
def on_is_standalone ()
|
183
|
+
end
|
184
|
+
|
185
|
+
|
186
|
+
def on_has_external_subset ()
|
187
|
+
end
|
188
|
+
|
189
|
+
|
190
|
+
def on_external_subset (name, external_id, system_id)
|
191
|
+
end
|
192
|
+
|
193
|
+
|
194
|
+
def on_end_document
|
195
|
+
end
|
196
|
+
end
|
197
|
+
|
198
|
+
end
|
199
|
+
end
|
200
|
+
end
|
@@ -0,0 +1,184 @@
|
|
1
|
+
#--
|
2
|
+
# Copyright (c) 2013 RightScale, Inc.
|
3
|
+
#
|
4
|
+
# Permission is hereby granted, free of charge, to any person obtaining
|
5
|
+
# a copy of this software and associated documentation files (the
|
6
|
+
# 'Software'), to deal in the Software without restriction, including
|
7
|
+
# without limitation the rights to use, copy, modify, merge, publish,
|
8
|
+
# distribute, sublicense, and/or sell copies of the Software, and to
|
9
|
+
# permit persons to whom the Software is furnished to do so, subject to
|
10
|
+
# the following conditions:
|
11
|
+
#
|
12
|
+
# The above copyright notice and this permission notice shall be
|
13
|
+
# included in all copies or substantial portions of the Software.
|
14
|
+
#
|
15
|
+
# THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND,
|
16
|
+
# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
17
|
+
# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
|
18
|
+
# IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
|
19
|
+
# CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
|
20
|
+
# TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
|
21
|
+
# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
22
|
+
#++
|
23
|
+
|
24
|
+
module RightScale
|
25
|
+
module CloudApi
|
26
|
+
|
27
|
+
# The routine processes cache validations (when caching is enabled).
|
28
|
+
#
|
29
|
+
# It takes a response from a cloud and tries to find a pre-defined caching pattern that would
|
30
|
+
# fit to this response and its request. If there is a pattern it extracts a previous response
|
31
|
+
# from the cache and compares it to the current one.
|
32
|
+
#
|
33
|
+
# If both the responses match it raises RightScale::CloudApi::CacheHit exception.
|
34
|
+
#
|
35
|
+
# The main point of the caching - it is performed before parsing a response. So if we get a 10M
|
36
|
+
# XML from Amazon it will take seconds to parse it but if the response did not change there is
|
37
|
+
# need to parse it.
|
38
|
+
#
|
39
|
+
# @example
|
40
|
+
# ec2 = RightScale::CloudApi::AWS::EC2.new(key, secret_key, :cache => true)
|
41
|
+
# ec2.DescribeInstances #=> a list of instances
|
42
|
+
# ec2.DescribeInstances(:options => {:cache => false}) #=> the same list of instances
|
43
|
+
# ec2.DescribeInstances #=> exception if the response did not change
|
44
|
+
#
|
45
|
+
# The caching setting is per cloud specific ApiManager. For some of them it is on by default so
|
46
|
+
# you need to look at the ApiManager definition.
|
47
|
+
#
|
48
|
+
class CacheValidator < Routine
|
49
|
+
|
50
|
+
class Error < CloudApi::Error
|
51
|
+
end
|
52
|
+
|
53
|
+
# Logs a message.
|
54
|
+
#
|
55
|
+
# @param [String] message Some text.
|
56
|
+
#
|
57
|
+
def log(message)
|
58
|
+
cloud_api_logger.log( "#{message}", :cache_validator)
|
59
|
+
end
|
60
|
+
|
61
|
+
module ClassMethods
|
62
|
+
CACHE_PATTERN_KEYS = [ :verb, :verb!, :path, :path!, :request, :request!, :code, :code!, :response, :response!, :key, :if, :sign ]
|
63
|
+
|
64
|
+
def self.extended(base)
|
65
|
+
unless base.respond_to?(:options) && base.options.is_a?(Hash)
|
66
|
+
raise Error::new("CacheValidator routine assumes class being extended responds to :options and returns a hash")
|
67
|
+
end
|
68
|
+
end
|
69
|
+
|
70
|
+
# Adds new cache patters.
|
71
|
+
# Patterns are analyzed in order of their definnition. If one pattern hits
|
72
|
+
# the rest are not analyzed.
|
73
|
+
#
|
74
|
+
# @param [Hash] cache_pattern A hash of pattern keys.
|
75
|
+
# @option cache_pattern [Proc] :key A method that calculates a kache key name.
|
76
|
+
# @option cache_pattern [Proc] :sign A method that modifies the response before calculating md5.
|
77
|
+
#
|
78
|
+
# @see file:lib/base/helper/utils.rb self.pattern_matches? for the other options.
|
79
|
+
#
|
80
|
+
# @example:
|
81
|
+
# cache_pattern :verb => /get|post/,
|
82
|
+
# :path => /Action=Describe/,
|
83
|
+
# :if => Proc::new{ |o| (o[:params].keys - %w{Action Version AWSAccessKeyId})._blank? },
|
84
|
+
# :key => Proc::new{ |o| o[:params]['Action'] },
|
85
|
+
# :sign => Proc::new{ |o| o[:response].body.to_s.sub(%r{<requestId>.+?</requestId>}i,'') }
|
86
|
+
#
|
87
|
+
def cache_pattern(cache_pattern)
|
88
|
+
fail Error::new("Pattern should be a Hash and should not be blank") if !cache_pattern.is_a?(Hash) || cache_pattern._blank?
|
89
|
+
fail Error::new("Key field not found in cache pattern definition #{cache_pattern.inspect}") unless cache_pattern.keys.include?(:key)
|
90
|
+
unsupported_keys = cache_pattern.keys - CACHE_PATTERN_KEYS
|
91
|
+
fail Error::new("Unsupported keys #{unsupported_keys.inspect} in cache pattern definition #{cache_pattern.inspect}") unless unsupported_keys._blank?
|
92
|
+
(options[:cache_patterns] ||= []) << cache_pattern
|
93
|
+
end
|
94
|
+
end
|
95
|
+
|
96
|
+
# The main entry point.
|
97
|
+
#
|
98
|
+
def process
|
99
|
+
# Do nothing if caching is off
|
100
|
+
return nil unless data[:options][:cache]
|
101
|
+
# There is nothing to cache if we stream things
|
102
|
+
return nil if data[:response][:instance].is_io?
|
103
|
+
|
104
|
+
cache_patterns = data[:options][:cache_patterns] || []
|
105
|
+
opts = { :relative_path => data[:request][:relative_path],
|
106
|
+
:request => data[:request][:instance],
|
107
|
+
:response => data[:response][:instance],
|
108
|
+
:verb => data[:request][:verb],
|
109
|
+
:params => data[:request][:orig_params].dup }
|
110
|
+
|
111
|
+
# Walk through all the cache patterns and find the first that matches
|
112
|
+
cache_patterns.each do |pattern|
|
113
|
+
# Try on the next pattern unless the current one matches.
|
114
|
+
next unless Utils::pattern_matches?(pattern, opts)
|
115
|
+
# Process the matching pattern.
|
116
|
+
log("Request matches to cache pattern: #{pattern.inspect}")
|
117
|
+
# Build a cache key and get a text to be signed
|
118
|
+
cache_key, text_to_sign = build_cache_key(pattern, opts)
|
119
|
+
cache_record = {
|
120
|
+
:timestamp => Time::now.utc,
|
121
|
+
:md5 => Digest::MD5::hexdigest(text_to_sign).to_s,
|
122
|
+
:hits => 0
|
123
|
+
}
|
124
|
+
log("Processing cache record: #{cache_key} => #{cache_record.inspect}")
|
125
|
+
# Save current cache key for later use (by other Routines)
|
126
|
+
data[:vars][:cache] ||= {}
|
127
|
+
data[:vars][:cache][:key] = cache_key
|
128
|
+
data[:vars][:cache][:record] = cache_record
|
129
|
+
# Get the cache storage
|
130
|
+
storage = (data[:vars][:system][:storage][:cache] ||= {} )
|
131
|
+
unless storage[cache_key]
|
132
|
+
# Create a new record unless exists.
|
133
|
+
storage[cache_key] = cache_record
|
134
|
+
log("New cache record created")
|
135
|
+
else
|
136
|
+
# If the record is already there but the response changed the replace the old record.
|
137
|
+
unless storage[cache_key][:md5] == cache_record[:md5]
|
138
|
+
storage[cache_key] = cache_record
|
139
|
+
log("Missed. Record is replaced")
|
140
|
+
else
|
141
|
+
# Raise if cache hits.
|
142
|
+
storage[cache_key][:hits] += 1
|
143
|
+
message = "Cache hit: #{cache_key.inspect} has not changed since " +
|
144
|
+
"#{storage[cache_key][:timestamp].strftime('%Y-%m-%d %H:%M:%S')}, "+
|
145
|
+
"hits: #{storage[cache_key][:hits]}."
|
146
|
+
log(message)
|
147
|
+
fail CacheHit::new("CacheValidator: #{message}")
|
148
|
+
end
|
149
|
+
end
|
150
|
+
break
|
151
|
+
end
|
152
|
+
true
|
153
|
+
end
|
154
|
+
|
155
|
+
private
|
156
|
+
|
157
|
+
# Builds the cached record key and body.
|
158
|
+
#
|
159
|
+
# @param [Hash] pattern The pattern that matched to the current response.
|
160
|
+
# @option options [Hash] opts A set of options that will be passed to :key and :sign procs.
|
161
|
+
#
|
162
|
+
# @return [Array] An array: [key, body]
|
163
|
+
#
|
164
|
+
# @raise [RightScale::CloudApi::CacheValidator::Error] Unless :key proc id set.
|
165
|
+
# @raise [RightScale::CloudApi::CacheValidator::Error] Unless :sign proc returns a valid body.
|
166
|
+
#
|
167
|
+
# @example
|
168
|
+
# build_cache_key(pattern) #=>
|
169
|
+
# ["DescribeVolumes", "<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n<DescribeVolumesResponse ... </DescribeVolumesResponse>"]
|
170
|
+
#
|
171
|
+
def build_cache_key(pattern, opts)
|
172
|
+
key = pattern[:key].is_a?(Proc) ? pattern[:key].call(opts) : pattern[:key]
|
173
|
+
fail Error::new("Cannot build cache key using pattern #{pattern.inspect}") unless key
|
174
|
+
|
175
|
+
body_to_sign = opts[:response].body.to_s if opts[:response].body
|
176
|
+
body_to_sign = pattern[:sign].call(opts) if pattern[:sign]
|
177
|
+
fail Error::new("Could not create body to sign using pattern #{pattern.inspect}") unless body_to_sign
|
178
|
+
|
179
|
+
[key, body_to_sign]
|
180
|
+
end
|
181
|
+
|
182
|
+
end
|
183
|
+
end
|
184
|
+
end
|
@@ -0,0 +1,194 @@
|
|
1
|
+
#--
|
2
|
+
# Copyright (c) 2013 RightScale, Inc.
|
3
|
+
#
|
4
|
+
# Permission is hereby granted, free of charge, to any person obtaining
|
5
|
+
# a copy of this software and associated documentation files (the
|
6
|
+
# 'Software'), to deal in the Software without restriction, including
|
7
|
+
# without limitation the rights to use, copy, modify, merge, publish,
|
8
|
+
# distribute, sublicense, and/or sell copies of the Software, and to
|
9
|
+
# permit persons to whom the Software is furnished to do so, subject to
|
10
|
+
# the following conditions:
|
11
|
+
#
|
12
|
+
# The above copyright notice and this permission notice shall be
|
13
|
+
# included in all copies or substantial portions of the Software.
|
14
|
+
#
|
15
|
+
# THE SOFTWARE IS PROVIDED 'AS IS', WITHOUT WARRANTY OF ANY KIND,
|
16
|
+
# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
17
|
+
# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
|
18
|
+
# IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
|
19
|
+
# CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
|
20
|
+
# TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
|
21
|
+
# SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
22
|
+
#++
|
23
|
+
|
24
|
+
module RightScale
|
25
|
+
module CloudApi
|
26
|
+
class ConnectionProxy
|
27
|
+
|
28
|
+
class NetHttpPersistentProxy
|
29
|
+
class Error < CloudApi::Error
|
30
|
+
end
|
31
|
+
|
32
|
+
|
33
|
+
# Known timeout errors
|
34
|
+
TIMEOUT_ERRORS = /Timeout|ETIMEDOUT/
|
35
|
+
|
36
|
+
# Other re-triable errors
|
37
|
+
OTHER_ERRORS = /SocketError|EOFError/
|
38
|
+
|
39
|
+
|
40
|
+
def log(message)
|
41
|
+
@data[:options][:cloud_api_logger].log(message, :connection_proxy, :warn)
|
42
|
+
end
|
43
|
+
|
44
|
+
# Performs an HTTP request.
|
45
|
+
#
|
46
|
+
# @param [Hash] data The API request +data+ storage.
|
47
|
+
# See {RightScale::CloudApi::ApiManager.initialize_api_request_options} code for its explanation.
|
48
|
+
#
|
49
|
+
# P.S. Options not supported by Net::HTTP::Persistent:
|
50
|
+
# :connection_retry_count, :connection_retry_delay, :cloud_api_logger
|
51
|
+
#
|
52
|
+
def request(data)
|
53
|
+
require "net/http/persistent"
|
54
|
+
|
55
|
+
@data = data
|
56
|
+
@data[:response] = {}
|
57
|
+
uri = @data[:connection][:uri]
|
58
|
+
|
59
|
+
# Create a connection
|
60
|
+
connection = Net::HTTP::Persistent.new('right_cloud_api_gem')
|
61
|
+
|
62
|
+
# Create a fake HTTP request
|
63
|
+
fake = @data[:request][:instance]
|
64
|
+
http_request = "Net::HTTP::#{fake.verb._camelize}"._constantize::new(fake.path)
|
65
|
+
if fake.is_io?
|
66
|
+
http_request.body_stream = fake.body
|
67
|
+
else
|
68
|
+
http_request.body = fake.body
|
69
|
+
end
|
70
|
+
fake.headers.each{|header, value| http_request[header] = value }
|
71
|
+
fake.raw = http_request
|
72
|
+
|
73
|
+
# Register a callback to close current connection
|
74
|
+
@data[:callbacks][:close_current_connection] = Proc::new do |reason|
|
75
|
+
connection.shutdown
|
76
|
+
log "Current connection closed: #{reason}"
|
77
|
+
end
|
78
|
+
|
79
|
+
# Set all required options
|
80
|
+
# P.S. :connection_retry_count, :http_connection_retry_delay are not supported by this proxy
|
81
|
+
#
|
82
|
+
http_request['user-agent'] ||= @data[:options][:connection_user_agent] if @data[:options].has_key?(:connection_user_agent)
|
83
|
+
connection.ca_file = @data[:options][:connection_ca_file] if @data[:options].has_key?(:connection_ca_file)
|
84
|
+
connection.read_timeout = @data[:options][:connection_read_timeout] if @data[:options].has_key?(:connection_read_timeout)
|
85
|
+
connection.open_timeout = @data[:options][:connection_open_timeout] if @data[:options].has_key?(:connection_open_timeout)
|
86
|
+
connection.cert = OpenSSL::X509::Certificate.new(@data[:credentials][:cert]) if @data[:credentials].has_key?(:cert)
|
87
|
+
connection.key = OpenSSL::PKey::RSA.new(@data[:credentials][:key]) if @data[:credentials].has_key?(:key)
|
88
|
+
|
89
|
+
# Make a request
|
90
|
+
begin
|
91
|
+
make_request_with_retries(connection, uri, http_request)
|
92
|
+
rescue => e
|
93
|
+
connection.shutdown
|
94
|
+
fail(ConnectionError, e.message)
|
95
|
+
end
|
96
|
+
end
|
97
|
+
|
98
|
+
|
99
|
+
# Makes request with low level retries.
|
100
|
+
#
|
101
|
+
# Net::HTTP::Persistent does not fully support retries logic that we used to have.
|
102
|
+
# To deal with this we disable Net::HTTP::Persistent's retries and handle them in our code.
|
103
|
+
#
|
104
|
+
# @param [Net::HTTP::Persistent] connection
|
105
|
+
# @param [URI] uri
|
106
|
+
# @param [Net::HTTPRequest] http_request
|
107
|
+
#
|
108
|
+
# @return [void]
|
109
|
+
#
|
110
|
+
def make_request_with_retries(connection, uri, http_request)
|
111
|
+
disable_net_http_persistent_retries(connection)
|
112
|
+
# Initialize retry vars:
|
113
|
+
connection_retry_count = @data[:options][:connection_retry_count] || 3
|
114
|
+
connection_retry_delay = @data[:options][:connection_retry_delay] || 0.5
|
115
|
+
retries_performed = 0
|
116
|
+
# If block is given - pass there all the chunks of a response and then stop
|
117
|
+
# (don't do any parsing, analysis, etc)
|
118
|
+
block = @data[:vars][:system][:block]
|
119
|
+
begin
|
120
|
+
if block
|
121
|
+
# Response.body is a Net::ReadAdapter instance - it can't be read as a string.
|
122
|
+
# WEB: On its own, Net::HTTP causes response.body to be a Net::ReadAdapter when you make a request with a block
|
123
|
+
# that calls read_body on the response.
|
124
|
+
connection.request(uri, http_request) do |response|
|
125
|
+
# If we are at the point when we have started reading from the remote end
|
126
|
+
# then there is no low level retry is allowed. Otherwise we would need to reset the
|
127
|
+
# IO pointer, etc.
|
128
|
+
connection_retry_count = 0
|
129
|
+
# Set IO response
|
130
|
+
set_http_response(response)
|
131
|
+
response.read_body(&block)
|
132
|
+
end
|
133
|
+
else
|
134
|
+
# Set text response
|
135
|
+
response = connection.request(uri, http_request)
|
136
|
+
set_http_response(response)
|
137
|
+
end
|
138
|
+
nil
|
139
|
+
rescue => e
|
140
|
+
# Fail if it is an unknown error
|
141
|
+
fail(e) if !(e.message[TIMEOUT_ERRORS] || e.message[OTHER_ERRORS])
|
142
|
+
# Fail if it is a Timeout and timeouts are banned
|
143
|
+
fail(e) if e.message[TIMEOUT_ERRORS] && !!@data[:options][:abort_on_timeout]
|
144
|
+
# Fail if there are no retries left...
|
145
|
+
fail(e) if (connection_retry_count -= 1) < 0
|
146
|
+
# ... otherwise sleep a bit and retry.
|
147
|
+
retries_performed += 1
|
148
|
+
log("#{self.class.name}: Performing retry ##{retries_performed} caused by: #{e.class.name}: #{e.message}")
|
149
|
+
sleep(connection_retry_delay) unless connection_retry_delay._blank?
|
150
|
+
connection_retry_delay *= 2
|
151
|
+
retry
|
152
|
+
end
|
153
|
+
end
|
154
|
+
|
155
|
+
|
156
|
+
# Saves HTTP Response into data hash.
|
157
|
+
#
|
158
|
+
# @param [Net::HTTPResponse] response
|
159
|
+
#
|
160
|
+
# @return [void]
|
161
|
+
#
|
162
|
+
def set_http_response(response)
|
163
|
+
@data[:response][:instance] = HTTPResponse.new(
|
164
|
+
response.code,
|
165
|
+
response.body.is_a?(IO) ? nil : response.body,
|
166
|
+
response.to_hash,
|
167
|
+
response
|
168
|
+
)
|
169
|
+
nil
|
170
|
+
end
|
171
|
+
|
172
|
+
|
173
|
+
# Net::HTTP::Persistent believes that it can retry on any GET call what is not true for
|
174
|
+
# Query like API clouds (Amazon, CloudStack, Euca, etc).
|
175
|
+
# The solutions is to monkeypatch Net::HTTP::Persistent#can_retry? so that is returns
|
176
|
+
# Net::HTTP::Persistent#retry_change_requests.
|
177
|
+
#
|
178
|
+
# @param [Net::HTTP::Persistent] connection
|
179
|
+
#
|
180
|
+
# @return [void]
|
181
|
+
#
|
182
|
+
def disable_net_http_persistent_retries(connection)
|
183
|
+
connection.retry_change_requests = false
|
184
|
+
# Monkey patch this connection instance only.
|
185
|
+
def connection.can_retry?(*args)
|
186
|
+
false
|
187
|
+
end
|
188
|
+
nil
|
189
|
+
end
|
190
|
+
|
191
|
+
end
|
192
|
+
end
|
193
|
+
end
|
194
|
+
end
|