talaris-right_aws 2.1.0
Sign up to get free protection for your applications and to get access to all the features.
- data/History.txt +305 -0
- data/Manifest.txt +60 -0
- data/README.txt +163 -0
- data/Rakefile +104 -0
- data/lib/acf/right_acf_interface.rb +549 -0
- data/lib/acf/right_acf_invalidations.rb +144 -0
- data/lib/acf/right_acf_origin_access_identities.rb +230 -0
- data/lib/acf/right_acf_streaming_interface.rb +229 -0
- data/lib/acw/right_acw_interface.rb +248 -0
- data/lib/as/right_as_interface.rb +698 -0
- data/lib/awsbase/benchmark_fix.rb +39 -0
- data/lib/awsbase/right_awsbase.rb +1174 -0
- data/lib/awsbase/support.rb +35 -0
- data/lib/awsbase/version.rb +9 -0
- data/lib/ec2/right_ec2.rb +458 -0
- data/lib/ec2/right_ec2_ebs.rb +465 -0
- data/lib/ec2/right_ec2_images.rb +413 -0
- data/lib/ec2/right_ec2_instances.rb +785 -0
- data/lib/ec2/right_ec2_monitoring.rb +70 -0
- data/lib/ec2/right_ec2_placement_groups.rb +108 -0
- data/lib/ec2/right_ec2_reserved_instances.rb +174 -0
- data/lib/ec2/right_ec2_security_groups.rb +396 -0
- data/lib/ec2/right_ec2_spot_instances.rb +425 -0
- data/lib/ec2/right_ec2_tags.rb +139 -0
- data/lib/ec2/right_ec2_vpc.rb +583 -0
- data/lib/ec2/right_ec2_windows_mobility.rb +84 -0
- data/lib/elb/right_elb_interface.rb +571 -0
- data/lib/iam/right_iam_access_keys.rb +71 -0
- data/lib/iam/right_iam_groups.rb +195 -0
- data/lib/iam/right_iam_interface.rb +341 -0
- data/lib/iam/right_iam_mfa_devices.rb +67 -0
- data/lib/iam/right_iam_users.rb +251 -0
- data/lib/rds/right_rds_interface.rb +1309 -0
- data/lib/right_aws.rb +83 -0
- data/lib/route_53/right_route_53_interface.rb +630 -0
- data/lib/s3/right_s3.rb +1123 -0
- data/lib/s3/right_s3_interface.rb +1198 -0
- data/lib/sdb/active_sdb.rb +1107 -0
- data/lib/sdb/right_sdb_interface.rb +753 -0
- data/lib/sqs/right_sqs.rb +387 -0
- data/lib/sqs/right_sqs_gen2.rb +342 -0
- data/lib/sqs/right_sqs_gen2_interface.rb +523 -0
- data/lib/sqs/right_sqs_interface.rb +593 -0
- data/right_aws.gemspec +91 -0
- data/test/acf/test_helper.rb +2 -0
- data/test/acf/test_right_acf.rb +138 -0
- data/test/awsbase/test_helper.rb +2 -0
- data/test/awsbase/test_right_awsbase.rb +12 -0
- data/test/ec2/test_helper.rb +2 -0
- data/test/ec2/test_right_ec2.rb +108 -0
- data/test/http_connection.rb +87 -0
- data/test/rds/test_helper.rb +2 -0
- data/test/rds/test_right_rds.rb +120 -0
- data/test/s3/test_helper.rb +2 -0
- data/test/s3/test_right_s3.rb +421 -0
- data/test/s3/test_right_s3_stubbed.rb +97 -0
- data/test/sdb/test_active_sdb.rb +357 -0
- data/test/sdb/test_batch_put_attributes.rb +54 -0
- data/test/sdb/test_helper.rb +3 -0
- data/test/sdb/test_right_sdb.rb +253 -0
- data/test/sqs/test_helper.rb +2 -0
- data/test/sqs/test_right_sqs.rb +285 -0
- data/test/sqs/test_right_sqs_gen2.rb +264 -0
- data/test/test_credentials.rb +37 -0
- data/test/ts_right_aws.rb +14 -0
- metadata +214 -0
@@ -0,0 +1,39 @@
|
|
1
|
+
#
|
2
|
+
# Copyright (c) 2007-2008 RightScale Inc
|
3
|
+
#
|
4
|
+
# Permission is hereby granted, free of charge, to any person obtaining
|
5
|
+
# a copy of this software and associated documentation files (the
|
6
|
+
# "Software"), to deal in the Software without restriction, including
|
7
|
+
# without limitation the rights to use, copy, modify, merge, publish,
|
8
|
+
# distribute, sublicense, and/or sell copies of the Software, and to
|
9
|
+
# permit persons to whom the Software is furnished to do so, subject to
|
10
|
+
# the following conditions:
|
11
|
+
#
|
12
|
+
# The above copyright notice and this permission notice shall be
|
13
|
+
# included in all copies or substantial portions of the Software.
|
14
|
+
#
|
15
|
+
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
16
|
+
# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
17
|
+
# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
18
|
+
# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
|
19
|
+
# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
|
20
|
+
# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
|
21
|
+
# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
22
|
+
#
|
23
|
+
#
|
24
|
+
|
25
|
+
|
26
|
+
# A hack because there's a bug in add! in Benchmark::Tms
|
27
|
+
module Benchmark #:nodoc:
|
28
|
+
class Tms #:nodoc:
|
29
|
+
def add!(&blk)
|
30
|
+
t = Benchmark::measure(&blk)
|
31
|
+
@utime = utime + t.utime
|
32
|
+
@stime = stime + t.stime
|
33
|
+
@cutime = cutime + t.cutime
|
34
|
+
@cstime = cstime + t.cstime
|
35
|
+
@real = real + t.real
|
36
|
+
self
|
37
|
+
end
|
38
|
+
end
|
39
|
+
end
|
@@ -0,0 +1,1174 @@
|
|
1
|
+
#
|
2
|
+
# Copyright (c) 2007-2008 RightScale Inc
|
3
|
+
#
|
4
|
+
# Permission is hereby granted, free of charge, to any person obtaining
|
5
|
+
# a copy of this software and associated documentation files (the
|
6
|
+
# "Software"), to deal in the Software without restriction, including
|
7
|
+
# without limitation the rights to use, copy, modify, merge, publish,
|
8
|
+
# distribute, sublicense, and/or sell copies of the Software, and to
|
9
|
+
# permit persons to whom the Software is furnished to do so, subject to
|
10
|
+
# the following conditions:
|
11
|
+
#
|
12
|
+
# The above copyright notice and this permission notice shall be
|
13
|
+
# included in all copies or substantial portions of the Software.
|
14
|
+
#
|
15
|
+
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
16
|
+
# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
17
|
+
# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
18
|
+
# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
|
19
|
+
# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
|
20
|
+
# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
|
21
|
+
# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
22
|
+
#
|
23
|
+
|
24
|
+
# Test
|
25
|
+
module RightAws
|
26
|
+
require 'digest/md5'
|
27
|
+
require 'pp'
|
28
|
+
|
29
|
+
class AwsUtils #:nodoc:
|
30
|
+
@@digest1 = OpenSSL::Digest::Digest.new("sha1")
|
31
|
+
@@digest256 = nil
|
32
|
+
if OpenSSL::OPENSSL_VERSION_NUMBER > 0x00908000
|
33
|
+
@@digest256 = OpenSSL::Digest::Digest.new("sha256") rescue nil # Some installation may not support sha256
|
34
|
+
end
|
35
|
+
|
36
|
+
def self.utc_iso8601(time)
|
37
|
+
if time.is_a?(Fixnum) then time = Time::at(time)
|
38
|
+
elsif time.is_a?(String) then time = Time::parse(time)
|
39
|
+
end
|
40
|
+
time.utc.strftime("%Y-%m-%dT%H:%M:%S.000Z")
|
41
|
+
end
|
42
|
+
|
43
|
+
def self.sign(aws_secret_access_key, auth_string)
|
44
|
+
Base64.encode64(OpenSSL::HMAC.digest(@@digest1, aws_secret_access_key, auth_string)).strip
|
45
|
+
end
|
46
|
+
|
47
|
+
# Escape a string accordingly Amazon rulles
|
48
|
+
# http://docs.amazonwebservices.com/AmazonSimpleDB/2007-11-07/DeveloperGuide/index.html?REST_RESTAuth.html
|
49
|
+
def self.amz_escape(param)
|
50
|
+
param.to_s.gsub(/([^a-zA-Z0-9._~-]+)/n) do
|
51
|
+
'%' + $1.unpack('H2' * $1.size).join('%').upcase
|
52
|
+
end
|
53
|
+
end
|
54
|
+
|
55
|
+
def self.xml_escape(text) # :nodoc:
|
56
|
+
REXML::Text::normalize(text)
|
57
|
+
end
|
58
|
+
|
59
|
+
def self.xml_unescape(text) # :nodoc:
|
60
|
+
REXML::Text::unnormalize(text)
|
61
|
+
end
|
62
|
+
|
63
|
+
# Set a timestamp and a signature version
|
64
|
+
def self.fix_service_params(service_hash, signature)
|
65
|
+
service_hash["Timestamp"] ||= utc_iso8601(Time.now) unless service_hash["Expires"]
|
66
|
+
service_hash["SignatureVersion"] = signature
|
67
|
+
service_hash
|
68
|
+
end
|
69
|
+
|
70
|
+
# Signature Version 0
|
71
|
+
# A deprecated guy (should work till septemper 2009)
|
72
|
+
def self.sign_request_v0(aws_secret_access_key, service_hash)
|
73
|
+
fix_service_params(service_hash, '0')
|
74
|
+
string_to_sign = "#{service_hash['Action']}#{service_hash['Timestamp'] || service_hash['Expires']}"
|
75
|
+
service_hash['Signature'] = AwsUtils::sign(aws_secret_access_key, string_to_sign)
|
76
|
+
service_hash.to_a.collect{|key,val| "#{amz_escape(key)}=#{amz_escape(val.to_s)}" }.join("&")
|
77
|
+
end
|
78
|
+
|
79
|
+
# Signature Version 1
|
80
|
+
# Another deprecated guy (should work till septemper 2009)
|
81
|
+
def self.sign_request_v1(aws_secret_access_key, service_hash)
|
82
|
+
fix_service_params(service_hash, '1')
|
83
|
+
string_to_sign = service_hash.sort{|a,b| (a[0].to_s.downcase)<=>(b[0].to_s.downcase)}.to_s
|
84
|
+
service_hash['Signature'] = AwsUtils::sign(aws_secret_access_key, string_to_sign)
|
85
|
+
service_hash.to_a.collect{|key,val| "#{amz_escape(key)}=#{amz_escape(val.to_s)}" }.join("&")
|
86
|
+
end
|
87
|
+
|
88
|
+
# Signature Version 2
|
89
|
+
# EC2, SQS and SDB requests must be signed by this guy.
|
90
|
+
# See: http://docs.amazonwebservices.com/AmazonSimpleDB/2007-11-07/DeveloperGuide/index.html?REST_RESTAuth.html
|
91
|
+
# http://developer.amazonwebservices.com/connect/entry.jspa?externalID=1928
|
92
|
+
def self.sign_request_v2(aws_secret_access_key, service_hash, http_verb, host, uri)
|
93
|
+
fix_service_params(service_hash, '2')
|
94
|
+
# select a signing method (make an old openssl working with sha1)
|
95
|
+
# make 'HmacSHA256' to be a default one
|
96
|
+
service_hash['SignatureMethod'] = 'HmacSHA256' unless ['HmacSHA256', 'HmacSHA1'].include?(service_hash['SignatureMethod'])
|
97
|
+
service_hash['SignatureMethod'] = 'HmacSHA1' unless @@digest256
|
98
|
+
# select a digest
|
99
|
+
digest = (service_hash['SignatureMethod'] == 'HmacSHA256' ? @@digest256 : @@digest1)
|
100
|
+
# form string to sign
|
101
|
+
canonical_string = service_hash.keys.sort.map do |key|
|
102
|
+
"#{amz_escape(key)}=#{amz_escape(service_hash[key])}"
|
103
|
+
end.join('&')
|
104
|
+
string_to_sign = "#{http_verb.to_s.upcase}\n#{host.downcase}\n#{uri}\n#{canonical_string}"
|
105
|
+
# sign the string
|
106
|
+
signature = amz_escape(Base64.encode64(OpenSSL::HMAC.digest(digest, aws_secret_access_key, string_to_sign)).strip)
|
107
|
+
"#{canonical_string}&Signature=#{signature}"
|
108
|
+
end
|
109
|
+
|
110
|
+
# From Amazon's SQS Dev Guide, a brief description of how to escape:
|
111
|
+
# "URL encode the computed signature and other query parameters as specified in
|
112
|
+
# RFC1738, section 2.2. In addition, because the + character is interpreted as a blank space
|
113
|
+
# by Sun Java classes that perform URL decoding, make sure to encode the + character
|
114
|
+
# although it is not required by RFC1738."
|
115
|
+
# Avoid using CGI::escape to escape URIs.
|
116
|
+
# CGI::escape will escape characters in the protocol, host, and port
|
117
|
+
# sections of the URI. Only target chars in the query
|
118
|
+
# string should be escaped.
|
119
|
+
def self.URLencode(raw)
|
120
|
+
e = URI.escape(raw)
|
121
|
+
e.gsub(/\+/, "%2b")
|
122
|
+
end
|
123
|
+
|
124
|
+
def self.allow_only(allowed_keys, params)
|
125
|
+
bogus_args = []
|
126
|
+
params.keys.each {|p| bogus_args.push(p) unless allowed_keys.include?(p) }
|
127
|
+
raise AwsError.new("The following arguments were given but are not legal for the function call #{caller_method}: #{bogus_args.inspect}") if bogus_args.length > 0
|
128
|
+
end
|
129
|
+
|
130
|
+
def self.mandatory_arguments(required_args, params)
|
131
|
+
rargs = required_args.dup
|
132
|
+
params.keys.each {|p| rargs.delete(p)}
|
133
|
+
raise AwsError.new("The following mandatory arguments were not provided to #{caller_method}: #{rargs.inspect}") if rargs.length > 0
|
134
|
+
end
|
135
|
+
|
136
|
+
def self.caller_method
|
137
|
+
caller[1]=~/`(.*?)'/
|
138
|
+
$1
|
139
|
+
end
|
140
|
+
|
141
|
+
def self.split_items_and_params(array)
|
142
|
+
items = Array(array).flatten.compact
|
143
|
+
params = items.last.kind_of?(Hash) ? items.pop : {}
|
144
|
+
[items, params]
|
145
|
+
end
|
146
|
+
|
147
|
+
# Generates a token in format of:
|
148
|
+
# 1. "1dd8d4e4-db6b-11df-b31d-0025b37efad0 (if UUID gem is loaded)
|
149
|
+
# 2. "1287483761-855215-zSv2z-bWGj2-31M5t-ags9m" (if UUID gem is not loaded)
|
150
|
+
TOKEN_GENERATOR_CHARSET = ('a'..'z').to_a + ('A'..'Z').to_a + ('0'..'9').to_a
|
151
|
+
def self.generate_unique_token
|
152
|
+
time = Time.now
|
153
|
+
token = "%d-%06d" % [time.to_i, time.usec]
|
154
|
+
4.times do
|
155
|
+
token << "-"
|
156
|
+
5.times { token << TOKEN_GENERATOR_CHARSET[rand(TOKEN_GENERATOR_CHARSET.size)] }
|
157
|
+
end
|
158
|
+
token
|
159
|
+
end
|
160
|
+
end
|
161
|
+
|
162
|
+
class AwsBenchmarkingBlock #:nodoc:
|
163
|
+
attr_accessor :xml, :service
|
164
|
+
def initialize
|
165
|
+
# Benchmark::Tms instance for service (Ec2, S3, or SQS) access benchmarking.
|
166
|
+
@service = Benchmark::Tms.new()
|
167
|
+
# Benchmark::Tms instance for XML parsing benchmarking.
|
168
|
+
@xml = Benchmark::Tms.new()
|
169
|
+
end
|
170
|
+
end
|
171
|
+
|
172
|
+
class AwsNoChange < RuntimeError
|
173
|
+
end
|
174
|
+
|
175
|
+
class RightAwsBase
|
176
|
+
|
177
|
+
# Amazon HTTP Error handling
|
178
|
+
|
179
|
+
# Text, if found in an error message returned by AWS, indicates that this may be a transient
|
180
|
+
# error. Transient errors are automatically retried with exponential back-off.
|
181
|
+
AMAZON_PROBLEMS = [ 'internal service error',
|
182
|
+
'is currently unavailable',
|
183
|
+
'no response from',
|
184
|
+
'Please try again',
|
185
|
+
'InternalError',
|
186
|
+
'Internal Server Error',
|
187
|
+
'ServiceUnavailable', #from SQS docs
|
188
|
+
'Unavailable',
|
189
|
+
'This application is not currently available',
|
190
|
+
'InsufficientInstanceCapacity'
|
191
|
+
]
|
192
|
+
@@amazon_problems = AMAZON_PROBLEMS
|
193
|
+
# Returns a list of Amazon service responses which are known to be transient problems.
|
194
|
+
# We have to re-request if we get any of them, because the problem will probably disappear.
|
195
|
+
# By default this method returns the same value as the AMAZON_PROBLEMS const.
|
196
|
+
def self.amazon_problems
|
197
|
+
@@amazon_problems
|
198
|
+
end
|
199
|
+
|
200
|
+
# Sets the list of Amazon side problems. Use in conjunction with the
|
201
|
+
# getter to append problems.
|
202
|
+
def self.amazon_problems=(problems_list)
|
203
|
+
@@amazon_problems = problems_list
|
204
|
+
end
|
205
|
+
|
206
|
+
# Raise an exception if a timeout occures while an API call is in progress.
|
207
|
+
# This helps to avoid a duplicate resources creation when Amazon hangs for some time and
|
208
|
+
# RightHttpConnection is forced to use retries to get a response from it.
|
209
|
+
#
|
210
|
+
# If an API call action is in the list then no attempts to retry are performed.
|
211
|
+
#
|
212
|
+
RAISE_ON_TIMEOUT_ON_ACTIONS = %w{
|
213
|
+
AllocateAddress
|
214
|
+
CreateSnapshot
|
215
|
+
CreateVolume
|
216
|
+
PurchaseReservedInstancesOffering
|
217
|
+
RequestSpotInstances
|
218
|
+
RunInstances
|
219
|
+
}
|
220
|
+
@@raise_on_timeout_on_actions = RAISE_ON_TIMEOUT_ON_ACTIONS.dup
|
221
|
+
|
222
|
+
def self.raise_on_timeout_on_actions
|
223
|
+
@@raise_on_timeout_on_actions
|
224
|
+
end
|
225
|
+
|
226
|
+
def self.raise_on_timeout_on_actions=(actions_list)
|
227
|
+
@@raise_on_timeout_on_actions = actions_list
|
228
|
+
end
|
229
|
+
|
230
|
+
end
|
231
|
+
|
232
|
+
module RightAwsBaseInterface
|
233
|
+
DEFAULT_SIGNATURE_VERSION = '2'
|
234
|
+
|
235
|
+
@@caching = false
|
236
|
+
def self.caching
|
237
|
+
@@caching
|
238
|
+
end
|
239
|
+
def self.caching=(caching)
|
240
|
+
@@caching = caching
|
241
|
+
end
|
242
|
+
|
243
|
+
# Current aws_access_key_id
|
244
|
+
attr_reader :aws_access_key_id
|
245
|
+
# Current aws_secret_access_key
|
246
|
+
attr_reader :aws_secret_access_key
|
247
|
+
# Last HTTP request object
|
248
|
+
attr_reader :last_request
|
249
|
+
# Last HTTP response object
|
250
|
+
attr_reader :last_response
|
251
|
+
# Last AWS errors list (used by AWSErrorHandler)
|
252
|
+
attr_accessor :last_errors
|
253
|
+
# Last AWS request id (used by AWSErrorHandler)
|
254
|
+
attr_accessor :last_request_id
|
255
|
+
# Logger object
|
256
|
+
attr_accessor :logger
|
257
|
+
# Initial params hash
|
258
|
+
attr_accessor :params
|
259
|
+
# RightHttpConnection instance
|
260
|
+
attr_reader :connection
|
261
|
+
# Cache
|
262
|
+
attr_reader :cache
|
263
|
+
# Signature version (all services except s3)
|
264
|
+
attr_reader :signature_version
|
265
|
+
|
266
|
+
def init(service_info, aws_access_key_id, aws_secret_access_key, params={}) #:nodoc:
|
267
|
+
@params = params
|
268
|
+
# If one defines EC2_URL he may forget to use a single slash as an "empty service" path.
|
269
|
+
# Amazon does not like this therefore add this bad boy if he is missing...
|
270
|
+
service_info[:default_service] = '/' if service_info[:default_service].right_blank?
|
271
|
+
raise AwsError.new("AWS access keys are required to operate on #{service_info[:name]}") \
|
272
|
+
if aws_access_key_id.right_blank? || aws_secret_access_key.right_blank?
|
273
|
+
@aws_access_key_id = aws_access_key_id
|
274
|
+
@aws_secret_access_key = aws_secret_access_key
|
275
|
+
# if the endpoint was explicitly defined - then use it
|
276
|
+
if @params[:endpoint_url]
|
277
|
+
@params[:server] = URI.parse(@params[:endpoint_url]).host
|
278
|
+
@params[:port] = URI.parse(@params[:endpoint_url]).port
|
279
|
+
@params[:service] = URI.parse(@params[:endpoint_url]).path
|
280
|
+
# make sure the 'service' path is not empty
|
281
|
+
@params[:service] = service_info[:default_service] if @params[:service].right_blank?
|
282
|
+
@params[:protocol] = URI.parse(@params[:endpoint_url]).scheme
|
283
|
+
@params[:region] = nil
|
284
|
+
else
|
285
|
+
@params[:server] ||= service_info[:default_host]
|
286
|
+
@params[:server] = "#{@params[:region]}.#{@params[:server]}" if @params[:region]
|
287
|
+
@params[:port] ||= service_info[:default_port]
|
288
|
+
@params[:service] ||= service_info[:default_service]
|
289
|
+
@params[:protocol] ||= service_info[:default_protocol]
|
290
|
+
end
|
291
|
+
# a set of options to be passed to RightHttpConnection object
|
292
|
+
@params[:connection_options] = {} unless @params[:connection_options].is_a?(Hash)
|
293
|
+
@with_connection_options = {}
|
294
|
+
@params[:connections] ||= :shared # || :dedicated
|
295
|
+
@params[:max_connections] ||= 10
|
296
|
+
@params[:connection_lifetime] ||= 20*60
|
297
|
+
@params[:api_version] ||= service_info[:default_api_version]
|
298
|
+
@logger = @params[:logger]
|
299
|
+
@logger = ::Rails.logger if !@logger && defined?(::Rails) && ::Rails.respond_to?(:logger)
|
300
|
+
@logger = RAILS_DEFAULT_LOGGER if !@logger && defined?(RAILS_DEFAULT_LOGGER)
|
301
|
+
@logger = Logger.new(STDOUT) if !@logger
|
302
|
+
@logger.info "New #{self.class.name} using #{@params[:connections]} connections mode"
|
303
|
+
@error_handler = nil
|
304
|
+
@cache = {}
|
305
|
+
@signature_version = (params[:signature_version] || DEFAULT_SIGNATURE_VERSION).to_s
|
306
|
+
end
|
307
|
+
|
308
|
+
def signed_service_params(aws_secret_access_key, service_hash, http_verb=nil, host=nil, service=nil )
|
309
|
+
case signature_version.to_s
|
310
|
+
when '0' then AwsUtils::sign_request_v0(aws_secret_access_key, service_hash)
|
311
|
+
when '1' then AwsUtils::sign_request_v1(aws_secret_access_key, service_hash)
|
312
|
+
when '2' then AwsUtils::sign_request_v2(aws_secret_access_key, service_hash, http_verb, host, service)
|
313
|
+
else raise AwsError.new("Unknown signature version (#{signature_version.to_s}) requested")
|
314
|
+
end
|
315
|
+
end
|
316
|
+
|
317
|
+
# Returns +true+ if the describe_xxx responses are being cached
|
318
|
+
def caching?
|
319
|
+
@params.key?(:cache) ? @params[:cache] : @@caching
|
320
|
+
end
|
321
|
+
|
322
|
+
# Check if the aws function response hits the cache or not.
|
323
|
+
# If the cache hits:
|
324
|
+
# - raises an +AwsNoChange+ exception if +do_raise+ == +:raise+.
|
325
|
+
# - returnes parsed response from the cache if it exists or +true+ otherwise.
|
326
|
+
# If the cache miss or the caching is off then returns +false+.
|
327
|
+
def cache_hits?(function, response, do_raise=:raise)
|
328
|
+
result = false
|
329
|
+
if caching?
|
330
|
+
function = function.to_sym
|
331
|
+
# get rid of requestId (this bad boy was added for API 2008-08-08+ and it is uniq for every response)
|
332
|
+
# feb 04, 2009 (load balancer uses 'RequestId' hence use 'i' modifier to hit it also)
|
333
|
+
response = response.sub(%r{<requestId>.+?</requestId>}i, '')
|
334
|
+
response_md5 = MD5.md5(response).to_s
|
335
|
+
# check for changes
|
336
|
+
unless @cache[function] && @cache[function][:response_md5] == response_md5
|
337
|
+
# well, the response is new, reset cache data
|
338
|
+
update_cache(function, {:response_md5 => response_md5,
|
339
|
+
:timestamp => Time.now,
|
340
|
+
:hits => 0,
|
341
|
+
:parsed => nil})
|
342
|
+
else
|
343
|
+
# aha, cache hits, update the data and throw an exception if needed
|
344
|
+
@cache[function][:hits] += 1
|
345
|
+
if do_raise == :raise
|
346
|
+
raise(AwsNoChange, "Cache hit: #{function} response has not changed since "+
|
347
|
+
"#{@cache[function][:timestamp].strftime('%Y-%m-%d %H:%M:%S')}, "+
|
348
|
+
"hits: #{@cache[function][:hits]}.")
|
349
|
+
else
|
350
|
+
result = @cache[function][:parsed] || true
|
351
|
+
end
|
352
|
+
end
|
353
|
+
end
|
354
|
+
result
|
355
|
+
end
|
356
|
+
|
357
|
+
def update_cache(function, hash)
|
358
|
+
(@cache[function.to_sym] ||= {}).merge!(hash) if caching?
|
359
|
+
end
|
360
|
+
|
361
|
+
def on_exception(options={:raise=>true, :log=>true}) # :nodoc:
|
362
|
+
raise if $!.is_a?(AwsNoChange)
|
363
|
+
AwsError::on_aws_exception(self, options)
|
364
|
+
end
|
365
|
+
|
366
|
+
#----------------------------
|
367
|
+
# HTTP Connections handling
|
368
|
+
#----------------------------
|
369
|
+
|
370
|
+
def get_server_url(request) # :nodoc:
|
371
|
+
"#{request[:protocol]}://#{request[:server]}:#{request[:port]}"
|
372
|
+
end
|
373
|
+
|
374
|
+
def get_connections_storage(aws_service) # :nodoc:
|
375
|
+
case @params[:connections].to_s
|
376
|
+
when 'dedicated' then @connections_storage ||= {}
|
377
|
+
else Thread.current[aws_service] ||= {}
|
378
|
+
end
|
379
|
+
end
|
380
|
+
|
381
|
+
def destroy_connection(request, reason) # :nodoc:
|
382
|
+
connections = get_connections_storage(request[:aws_service])
|
383
|
+
server_url = get_server_url(request)
|
384
|
+
if connections[server_url]
|
385
|
+
connections[server_url][:connection].finish(reason)
|
386
|
+
connections.delete(server_url)
|
387
|
+
end
|
388
|
+
end
|
389
|
+
|
390
|
+
# Expire the connection if it has expired.
|
391
|
+
def get_connection(request) # :nodoc:
|
392
|
+
server_url = get_server_url(request)
|
393
|
+
connection_storage = get_connections_storage(request[:aws_service])
|
394
|
+
life_time_scratch = Time.now-@params[:connection_lifetime]
|
395
|
+
# Delete out-of-dated connections
|
396
|
+
connections_in_list = 0
|
397
|
+
connection_storage.to_a.sort{|conn1, conn2| conn2[1][:last_used_at] <=> conn1[1][:last_used_at]}.each do |serv_url, conn_opts|
|
398
|
+
if @params[:max_connections] <= connections_in_list
|
399
|
+
conn_opts[:connection].finish('out-of-limit')
|
400
|
+
connection_storage.delete(server_url)
|
401
|
+
elsif conn_opts[:last_used_at] < life_time_scratch
|
402
|
+
conn_opts[:connection].finish('out-of-date')
|
403
|
+
connection_storage.delete(server_url)
|
404
|
+
else
|
405
|
+
connections_in_list += 1
|
406
|
+
end
|
407
|
+
end
|
408
|
+
connection = (connection_storage[server_url] ||= {})
|
409
|
+
connection[:last_used_at] = Time.now
|
410
|
+
connection[:connection] ||= Rightscale::HttpConnection.new(:exception => RightAws::AwsError, :logger => @logger)
|
411
|
+
end
|
412
|
+
|
413
|
+
#----------------------------
|
414
|
+
# HTTP Requests handling
|
415
|
+
#----------------------------
|
416
|
+
|
417
|
+
# ACF, AMS, EC2, LBS and SDB uses this guy
|
418
|
+
# SQS and S3 use their own methods
|
419
|
+
def generate_request_impl(verb, action, options={}) #:nodoc:
|
420
|
+
# Form a valid http verb: 'GET' or 'POST' (all the other are not supported now)
|
421
|
+
http_verb = verb.to_s.upcase
|
422
|
+
# remove empty keys from request options
|
423
|
+
options.delete_if { |key, value| value.nil? }
|
424
|
+
# prepare service data
|
425
|
+
service_hash = {"Action" => action,
|
426
|
+
"AWSAccessKeyId" => @aws_access_key_id,
|
427
|
+
"Version" => @params[:api_version] }
|
428
|
+
service_hash.merge!(options)
|
429
|
+
# Sign request options
|
430
|
+
service_params = signed_service_params(@aws_secret_access_key, service_hash, http_verb, @params[:server], @params[:service])
|
431
|
+
# Use POST if the length of the query string is too large
|
432
|
+
# see http://docs.amazonwebservices.com/AmazonSimpleDB/2007-11-07/DeveloperGuide/MakingRESTRequests.html
|
433
|
+
if http_verb != 'POST' && service_params.size > 2000
|
434
|
+
http_verb = 'POST'
|
435
|
+
if signature_version == '2'
|
436
|
+
service_params = signed_service_params(@aws_secret_access_key, service_hash, http_verb, @params[:server], @params[:service])
|
437
|
+
end
|
438
|
+
end
|
439
|
+
# create a request
|
440
|
+
case http_verb
|
441
|
+
when 'GET'
|
442
|
+
request = Net::HTTP::Get.new("#{@params[:service]}?#{service_params}")
|
443
|
+
when 'POST'
|
444
|
+
request = Net::HTTP::Post.new(@params[:service])
|
445
|
+
request.body = service_params
|
446
|
+
request['Content-Type'] = 'application/x-www-form-urlencoded; charset=utf-8'
|
447
|
+
else
|
448
|
+
raise "Unsupported HTTP verb #{verb.inspect}!"
|
449
|
+
end
|
450
|
+
# prepare output hash
|
451
|
+
request_hash = { :request => request,
|
452
|
+
:server => @params[:server],
|
453
|
+
:port => @params[:port],
|
454
|
+
:protocol => @params[:protocol] }
|
455
|
+
request_hash.merge!(@params[:connection_options])
|
456
|
+
request_hash.merge!(@with_connection_options)
|
457
|
+
|
458
|
+
# If an action is marked as "non-retryable" and there was no :raise_on_timeout option set
|
459
|
+
# explicitly then do set that option
|
460
|
+
if Array(RightAwsBase::raise_on_timeout_on_actions).include?(action) && !request_hash.has_key?(:raise_on_timeout)
|
461
|
+
request_hash.merge!(:raise_on_timeout => true)
|
462
|
+
end
|
463
|
+
|
464
|
+
request_hash
|
465
|
+
end
|
466
|
+
|
467
|
+
# All services uses this guy.
|
468
|
+
def request_info_impl(aws_service, benchblock, request, parser, &block) #:nodoc:
|
469
|
+
request[:aws_service] = aws_service
|
470
|
+
@connection = get_connection(request)
|
471
|
+
@last_request = request[:request]
|
472
|
+
@last_response = nil
|
473
|
+
response = nil
|
474
|
+
blockexception = nil
|
475
|
+
|
476
|
+
if(block != nil)
|
477
|
+
# TRB 9/17/07 Careful - because we are passing in blocks, we get a situation where
|
478
|
+
# an exception may get thrown in the block body (which is high-level
|
479
|
+
# code either here or in the application) but gets caught in the
|
480
|
+
# low-level code of HttpConnection. The solution is not to let any
|
481
|
+
# exception escape the block that we pass to HttpConnection::request.
|
482
|
+
# Exceptions can originate from code directly in the block, or from user
|
483
|
+
# code called in the other block which is passed to response.read_body.
|
484
|
+
benchblock.service.add! do
|
485
|
+
begin
|
486
|
+
responsehdr = @connection.request(request) do |response|
|
487
|
+
#########
|
488
|
+
begin
|
489
|
+
@last_response = response
|
490
|
+
if response.is_a?(Net::HTTPSuccess)
|
491
|
+
@error_handler = nil
|
492
|
+
response.read_body(&block)
|
493
|
+
else
|
494
|
+
@error_handler = AWSErrorHandler.new(self, parser, :errors_list => self.class.amazon_problems) unless @error_handler
|
495
|
+
check_result = @error_handler.check(request)
|
496
|
+
if check_result
|
497
|
+
@error_handler = nil
|
498
|
+
return check_result
|
499
|
+
end
|
500
|
+
raise AwsError.new(@last_errors, @last_response.code, @last_request_id)
|
501
|
+
end
|
502
|
+
rescue Exception => e
|
503
|
+
blockexception = e
|
504
|
+
end
|
505
|
+
end
|
506
|
+
rescue Exception => e
|
507
|
+
# Kill a connection if we run into a low level connection error
|
508
|
+
destroy_connection(request, "error: #{e.message}")
|
509
|
+
raise e
|
510
|
+
end
|
511
|
+
#########
|
512
|
+
|
513
|
+
#OK, now we are out of the block passed to the lower level
|
514
|
+
if(blockexception)
|
515
|
+
raise blockexception
|
516
|
+
end
|
517
|
+
benchblock.xml.add! do
|
518
|
+
parser.parse(responsehdr)
|
519
|
+
end
|
520
|
+
return parser.result
|
521
|
+
end
|
522
|
+
else
|
523
|
+
benchblock.service.add! do
|
524
|
+
begin
|
525
|
+
response = @connection.request(request)
|
526
|
+
rescue Exception => e
|
527
|
+
# Kill a connection if we run into a low level connection error
|
528
|
+
destroy_connection(request, "error: #{e.message}")
|
529
|
+
raise e
|
530
|
+
end
|
531
|
+
end
|
532
|
+
# check response for errors...
|
533
|
+
@last_response = response
|
534
|
+
if response.is_a?(Net::HTTPSuccess)
|
535
|
+
@error_handler = nil
|
536
|
+
benchblock.xml.add! { parser.parse(response) }
|
537
|
+
return parser.result
|
538
|
+
else
|
539
|
+
@error_handler = AWSErrorHandler.new(self, parser, :errors_list => self.class.amazon_problems) unless @error_handler
|
540
|
+
check_result = @error_handler.check(request)
|
541
|
+
if check_result
|
542
|
+
@error_handler = nil
|
543
|
+
return check_result
|
544
|
+
end
|
545
|
+
raise AwsError.new(@last_errors, @last_response.code, @last_request_id)
|
546
|
+
end
|
547
|
+
end
|
548
|
+
rescue
|
549
|
+
@error_handler = nil
|
550
|
+
raise
|
551
|
+
end
|
552
|
+
|
553
|
+
def request_cache_or_info(method, link, parser_class, benchblock, use_cache=true, &block) #:nodoc:
|
554
|
+
# We do not want to break the logic of parsing hence will use a dummy parser to process all the standard
|
555
|
+
# steps (errors checking etc). The dummy parser does nothig - just returns back the params it received.
|
556
|
+
# If the caching is enabled and hit then throw AwsNoChange.
|
557
|
+
# P.S. caching works for the whole images list only! (when the list param is blank)
|
558
|
+
# check cache
|
559
|
+
response, params = request_info(link, RightDummyParser.new)
|
560
|
+
cache_hits?(method.to_sym, response.body) if use_cache
|
561
|
+
parser = parser_class.new(:logger => @logger)
|
562
|
+
benchblock.xml.add!{ parser.parse(response, params) }
|
563
|
+
result = block ? block.call(parser) : parser.result
|
564
|
+
# update parsed data
|
565
|
+
update_cache(method.to_sym, :parsed => result) if use_cache
|
566
|
+
result
|
567
|
+
end
|
568
|
+
|
569
|
+
# Returns Amazons request ID for the latest request
|
570
|
+
def last_request_id
|
571
|
+
@last_response && @last_response.body.to_s[%r{<requestId>(.+?)</requestId>}i] && $1
|
572
|
+
end
|
573
|
+
|
574
|
+
# Incrementally lists something.
|
575
|
+
def incrementally_list_items(action, parser_class, params={}, &block) # :nodoc:
|
576
|
+
params = params.dup
|
577
|
+
params['MaxItems'] = params.delete(:max_items) if params[:max_items]
|
578
|
+
params['Marker'] = params.delete(:marker) if params[:marker]
|
579
|
+
last_response = nil
|
580
|
+
loop do
|
581
|
+
last_response = request_info( generate_request(action, params), parser_class.new(:logger => @logger))
|
582
|
+
params['Marker'] = last_response[:marker]
|
583
|
+
break unless block && block.call(last_response) && !last_response[:marker].right_blank?
|
584
|
+
end
|
585
|
+
last_response
|
586
|
+
end
|
587
|
+
|
588
|
+
# Format array of items into Amazons handy hash ('?' is a place holder):
|
589
|
+
# Options:
|
590
|
+
# :default => "something" : Set a value to "something" when it is nil
|
591
|
+
# :default => :skip_nils : Skip nil values
|
592
|
+
#
|
593
|
+
# amazonize_list('Item', ['a', 'b', 'c']) =>
|
594
|
+
# { 'Item.1' => 'a', 'Item.2' => 'b', 'Item.3' => 'c' }
|
595
|
+
#
|
596
|
+
# amazonize_list('Item.?.instance', ['a', 'c']) #=>
|
597
|
+
# { 'Item.1.instance' => 'a', 'Item.2.instance' => 'c' }
|
598
|
+
#
|
599
|
+
# amazonize_list(['Item.?.Name', 'Item.?.Value'], {'A' => 'a', 'B' => 'b'}) #=>
|
600
|
+
# { 'Item.1.Name' => 'A', 'Item.1.Value' => 'a',
|
601
|
+
# 'Item.2.Name' => 'B', 'Item.2.Value' => 'b' }
|
602
|
+
#
|
603
|
+
# amazonize_list(['Item.?.Name', 'Item.?.Value'], [['A','a'], ['B','b']]) #=>
|
604
|
+
# { 'Item.1.Name' => 'A', 'Item.1.Value' => 'a',
|
605
|
+
# 'Item.2.Name' => 'B', 'Item.2.Value' => 'b' }
|
606
|
+
#
|
607
|
+
# amazonize_list(['Filter.?.Key', 'Filter.?.Value.?'], {'A' => ['aa','ab'], 'B' => ['ba','bb']}) #=>
|
608
|
+
# amazonize_list(['Filter.?.Key', 'Filter.?.Value.?'], [['A',['aa','ab']], ['B',['ba','bb']]]) #=>
|
609
|
+
# {"Filter.1.Key"=>"A",
|
610
|
+
# "Filter.1.Value.1"=>"aa",
|
611
|
+
# "Filter.1.Value.2"=>"ab",
|
612
|
+
# "Filter.2.Key"=>"B",
|
613
|
+
# "Filter.2.Value.1"=>"ba",
|
614
|
+
# "Filter.2.Value.2"=>"bb"}
|
615
|
+
def amazonize_list(masks, list, options={}) #:nodoc:
|
616
|
+
groups = {}
|
617
|
+
Array(list).each_with_index do |list_item, i|
|
618
|
+
Array(masks).each_with_index do |mask, mask_idx|
|
619
|
+
key = mask[/\?/] ? mask.dup : mask.dup + '.?'
|
620
|
+
key.sub!('?', (i+1).to_s)
|
621
|
+
value = Array(list_item)[mask_idx]
|
622
|
+
if value.is_a?(Array)
|
623
|
+
groups.merge!(amazonize_list(key, value, options))
|
624
|
+
else
|
625
|
+
if value.nil?
|
626
|
+
next if options[:default] == :skip_nils
|
627
|
+
value = options[:default]
|
628
|
+
end
|
629
|
+
# Hack to avoid having unhandled '?' in keys : do replace them all with '1':
|
630
|
+
# bad: ec2.amazonize_list(['Filter.?.Key', 'Filter.?.Value.?'], { a: => :b }) => {"Filter.1.Key"=>:a, "Filter.1.Value.?"=>1}
|
631
|
+
# good: ec2.amazonize_list(['Filter.?.Key', 'Filter.?.Value.?'], { a: => :b }) => {"Filter.1.Key"=>:a, "Filter.1.Value.1"=>1}
|
632
|
+
key.gsub!('?', '1')
|
633
|
+
groups[key] = value
|
634
|
+
end
|
635
|
+
end
|
636
|
+
end
|
637
|
+
groups
|
638
|
+
end
|
639
|
+
|
640
|
+
BLOCK_DEVICE_KEY_MAPPING = { # :nodoc:
|
641
|
+
:device_name => 'DeviceName',
|
642
|
+
:virtual_name => 'VirtualName',
|
643
|
+
:no_device => 'NoDevice',
|
644
|
+
:ebs_snapshot_id => 'Ebs.SnapshotId',
|
645
|
+
:ebs_volume_size => 'Ebs.VolumeSize',
|
646
|
+
:ebs_delete_on_termination => 'Ebs.DeleteOnTermination' }
|
647
|
+
|
648
|
+
def amazonize_block_device_mappings(block_device_mappings, key = 'BlockDeviceMapping') # :nodoc:
|
649
|
+
result = {}
|
650
|
+
unless block_device_mappings.right_blank?
|
651
|
+
block_device_mappings = [block_device_mappings] unless block_device_mappings.is_a?(Array)
|
652
|
+
block_device_mappings.each_with_index do |b, idx|
|
653
|
+
BLOCK_DEVICE_KEY_MAPPING.each do |local_name, remote_name|
|
654
|
+
value = b[local_name]
|
655
|
+
case local_name
|
656
|
+
when :no_device then value = value ? '' : nil # allow to pass :no_device as boolean
|
657
|
+
end
|
658
|
+
result["#{key}.#{idx+1}.#{remote_name}"] = value unless value.nil?
|
659
|
+
end
|
660
|
+
end
|
661
|
+
end
|
662
|
+
result
|
663
|
+
end
|
664
|
+
|
665
|
+
# Execute a block of code with custom set of settings for right_http_connection.
|
666
|
+
# Accepts next options (see Rightscale::HttpConnection for explanation):
|
667
|
+
# :raise_on_timeout
|
668
|
+
# :http_connection_retry_count
|
669
|
+
# :http_connection_open_timeout
|
670
|
+
# :http_connection_read_timeout
|
671
|
+
# :http_connection_retry_delay
|
672
|
+
# :user_agent
|
673
|
+
# :exception
|
674
|
+
#
|
675
|
+
# Example #1:
|
676
|
+
#
|
677
|
+
# # Try to create a snapshot but stop with exception if timeout is received
|
678
|
+
# # to avoid having a duplicate API calls that create duplicate snapshots.
|
679
|
+
# ec2 = Rightscale::Ec2::new(aws_access_key_id, aws_secret_access_key)
|
680
|
+
# ec2.with_connection_options(:raise_on_timeout => true) do
|
681
|
+
# ec2.create_snapshot('vol-898a6fe0', 'KD: WooHoo!!')
|
682
|
+
# end
|
683
|
+
#
|
684
|
+
# Example #2:
|
685
|
+
#
|
686
|
+
# # Opposite case when the setting is global:
|
687
|
+
# @ec2 = Rightscale::Ec2::new(aws_access_key_id, aws_secret_access_key,
|
688
|
+
# :connection_options => { :raise_on_timeout => true })
|
689
|
+
# # Create an SSHKey but do tries on timeout
|
690
|
+
# ec2.with_connection_options(:raise_on_timeout => false) do
|
691
|
+
# new_key = ec2.create_key_pair('my_test_key')
|
692
|
+
# end
|
693
|
+
#
|
694
|
+
# Example #3:
|
695
|
+
#
|
696
|
+
# # Global settings (HttpConnection level):
|
697
|
+
# Rightscale::HttpConnection::params[:http_connection_open_timeout] = 5
|
698
|
+
# Rightscale::HttpConnection::params[:http_connection_read_timeout] = 250
|
699
|
+
# Rightscale::HttpConnection::params[:http_connection_retry_count] = 2
|
700
|
+
#
|
701
|
+
# # Local setings (RightAws level)
|
702
|
+
# ec2 = Rightscale::Ec2::new(AWS_ID, AWS_KEY,
|
703
|
+
# :region => 'us-east-1',
|
704
|
+
# :connection_options => {
|
705
|
+
# :http_connection_read_timeout => 2,
|
706
|
+
# :http_connection_retry_count => 5,
|
707
|
+
# :user_agent => 'Mozilla 4.0'
|
708
|
+
# })
|
709
|
+
#
|
710
|
+
# # Custom settings (API call level)
|
711
|
+
# ec2.with_connection_options(:raise_on_timeout => true,
|
712
|
+
# :http_connection_read_timeout => 10,
|
713
|
+
# :user_agent => '') do
|
714
|
+
# pp ec2.describe_images
|
715
|
+
# end
|
716
|
+
#
|
717
|
+
def with_connection_options(options, &block)
|
718
|
+
@with_connection_options = options
|
719
|
+
block.call self
|
720
|
+
ensure
|
721
|
+
@with_connection_options = {}
|
722
|
+
end
|
723
|
+
end
|
724
|
+
|
725
|
+
|
726
|
+
# Exception class to signal any Amazon errors. All errors occuring during calls to Amazon's
|
727
|
+
# web services raise this type of error.
|
728
|
+
# Attribute inherited by RuntimeError:
|
729
|
+
# message - the text of the error, generally as returned by AWS in its XML response.
|
730
|
+
class AwsError < RuntimeError
|
731
|
+
|
732
|
+
# either an array of errors where each item is itself an array of [code, message]),
|
733
|
+
# or an error string if the error was raised manually, as in <tt>AwsError.new('err_text')</tt>
|
734
|
+
attr_reader :errors
|
735
|
+
|
736
|
+
# Request id (if exists)
|
737
|
+
attr_reader :request_id
|
738
|
+
|
739
|
+
# Response HTTP error code
|
740
|
+
attr_reader :http_code
|
741
|
+
|
742
|
+
def initialize(errors=nil, http_code=nil, request_id=nil)
|
743
|
+
@errors = errors
|
744
|
+
@request_id = request_id
|
745
|
+
@http_code = http_code
|
746
|
+
super(@errors.is_a?(Array) ? @errors.map{|code, msg| "#{code}: #{msg}"}.join("; ") : @errors.to_s)
|
747
|
+
end
|
748
|
+
|
749
|
+
# Does any of the error messages include the regexp +pattern+?
|
750
|
+
# Used to determine whether to retry request.
|
751
|
+
def include?(pattern)
|
752
|
+
if @errors.is_a?(Array)
|
753
|
+
@errors.each{ |code, msg| return true if code =~ pattern }
|
754
|
+
else
|
755
|
+
return true if @errors_str =~ pattern
|
756
|
+
end
|
757
|
+
false
|
758
|
+
end
|
759
|
+
|
760
|
+
# Generic handler for AwsErrors. +aws+ is the RightAws::S3, RightAws::EC2, or RightAws::SQS
|
761
|
+
# object that caused the exception (it must provide last_request and last_response). Supported
|
762
|
+
# boolean options are:
|
763
|
+
# * <tt>:log</tt> print a message into the log using aws.logger to access the Logger
|
764
|
+
# * <tt>:puts</tt> do a "puts" of the error
|
765
|
+
# * <tt>:raise</tt> re-raise the error after logging
|
766
|
+
def self.on_aws_exception(aws, options={:raise=>true, :log=>true})
|
767
|
+
# Only log & notify if not user error
|
768
|
+
if !options[:raise] || system_error?($!)
|
769
|
+
error_text = "#{$!.inspect}\n#{$@}.join('\n')}"
|
770
|
+
puts error_text if options[:puts]
|
771
|
+
# Log the error
|
772
|
+
if options[:log]
|
773
|
+
request = aws.last_request ? aws.last_request.path : '-none-'
|
774
|
+
response = aws.last_response ? "#{aws.last_response.code} -- #{aws.last_response.message} -- #{aws.last_response.body}" : '-none-'
|
775
|
+
aws.logger.error error_text
|
776
|
+
aws.logger.error "Request was: #{request}"
|
777
|
+
aws.logger.error "Response was: #{response}"
|
778
|
+
end
|
779
|
+
end
|
780
|
+
raise if options[:raise] # re-raise an exception
|
781
|
+
return nil
|
782
|
+
end
|
783
|
+
|
784
|
+
# True if e is an AWS system error, i.e. something that is for sure not the caller's fault.
|
785
|
+
# Used to force logging.
|
786
|
+
def self.system_error?(e)
|
787
|
+
!e.is_a?(self) || e.message =~ /InternalError|InsufficientInstanceCapacity|Unavailable/
|
788
|
+
end
|
789
|
+
|
790
|
+
end
|
791
|
+
|
792
|
+
|
793
|
+
class AWSErrorHandler
|
794
|
+
# 0-100 (%)
|
795
|
+
DEFAULT_CLOSE_ON_4XX_PROBABILITY = 10
|
796
|
+
|
797
|
+
@@reiteration_start_delay = 0.2
|
798
|
+
def self.reiteration_start_delay
|
799
|
+
@@reiteration_start_delay
|
800
|
+
end
|
801
|
+
def self.reiteration_start_delay=(reiteration_start_delay)
|
802
|
+
@@reiteration_start_delay = reiteration_start_delay
|
803
|
+
end
|
804
|
+
|
805
|
+
@@reiteration_time = 5
|
806
|
+
def self.reiteration_time
|
807
|
+
@@reiteration_time
|
808
|
+
end
|
809
|
+
def self.reiteration_time=(reiteration_time)
|
810
|
+
@@reiteration_time = reiteration_time
|
811
|
+
end
|
812
|
+
|
813
|
+
@@close_on_error = true
|
814
|
+
def self.close_on_error
|
815
|
+
@@close_on_error
|
816
|
+
end
|
817
|
+
def self.close_on_error=(close_on_error)
|
818
|
+
@@close_on_error = close_on_error
|
819
|
+
end
|
820
|
+
|
821
|
+
@@close_on_4xx_probability = DEFAULT_CLOSE_ON_4XX_PROBABILITY
|
822
|
+
def self.close_on_4xx_probability
|
823
|
+
@@close_on_4xx_probability
|
824
|
+
end
|
825
|
+
def self.close_on_4xx_probability=(close_on_4xx_probability)
|
826
|
+
@@close_on_4xx_probability = close_on_4xx_probability
|
827
|
+
end
|
828
|
+
|
829
|
+
# params:
|
830
|
+
# :reiteration_time
|
831
|
+
# :errors_list
|
832
|
+
# :close_on_error = true | false
|
833
|
+
# :close_on_4xx_probability = 1-100
|
834
|
+
def initialize(aws, parser, params={}) #:nodoc:
|
835
|
+
@aws = aws # Link to RightEc2 | RightSqs | RightS3 instance
|
836
|
+
@parser = parser # parser to parse Amazon response
|
837
|
+
@started_at = Time.now
|
838
|
+
@stop_at = @started_at + (params[:reiteration_time] || @@reiteration_time)
|
839
|
+
@errors_list = params[:errors_list] || []
|
840
|
+
@reiteration_delay = @@reiteration_start_delay
|
841
|
+
@retries = 0
|
842
|
+
# close current HTTP(S) connection on 5xx, errors from list and 4xx errors
|
843
|
+
@close_on_error = params[:close_on_error].nil? ? @@close_on_error : params[:close_on_error]
|
844
|
+
@close_on_4xx_probability = params[:close_on_4xx_probability] || @@close_on_4xx_probability
|
845
|
+
end
|
846
|
+
|
847
|
+
# Returns false if
|
848
|
+
def check(request) #:nodoc:
|
849
|
+
result = false
|
850
|
+
error_found = false
|
851
|
+
redirect_detected= false
|
852
|
+
error_match = nil
|
853
|
+
last_errors_text = ''
|
854
|
+
response = @aws.last_response
|
855
|
+
# log error
|
856
|
+
request_text_data = "#{request[:protocol]}://#{request[:server]}:#{request[:port]}#{request[:request].path}"
|
857
|
+
# is this a redirect?
|
858
|
+
# yes!
|
859
|
+
if response.is_a?(Net::HTTPRedirection)
|
860
|
+
redirect_detected = true
|
861
|
+
else
|
862
|
+
# no, it's an error ...
|
863
|
+
@aws.logger.warn("##### #{@aws.class.name} returned an error: #{response.code} #{response.message}\n#{response.body} #####")
|
864
|
+
@aws.logger.warn("##### #{@aws.class.name} request: #{request_text_data} ####")
|
865
|
+
end
|
866
|
+
|
867
|
+
# Extract error/redirection message from the response body
|
868
|
+
# Amazon claims that a redirection must have a body but somethimes it is nil....
|
869
|
+
if response.body && response.body[/^(<\?xml|<ErrorResponse)/]
|
870
|
+
error_parser = RightErrorResponseParser.new
|
871
|
+
@aws.class.bench_xml.add! do
|
872
|
+
error_parser.parse(response.body)
|
873
|
+
end
|
874
|
+
@aws.last_errors = error_parser.errors
|
875
|
+
@aws.last_request_id = error_parser.requestID
|
876
|
+
last_errors_text = @aws.last_errors.flatten.join("\n")
|
877
|
+
else
|
878
|
+
@aws.last_errors = [[response.code, "#{response.message} (#{request_text_data})"]]
|
879
|
+
@aws.last_request_id = '-undefined-'
|
880
|
+
last_errors_text = response.message
|
881
|
+
end
|
882
|
+
|
883
|
+
# Ok, it is a redirect, find the new destination location
|
884
|
+
if redirect_detected
|
885
|
+
location = response['location']
|
886
|
+
# As for 301 ( Moved Permanently) Amazon does not return a 'Location' header but
|
887
|
+
# it is possible to extract a new endpoint from the response body
|
888
|
+
if location.right_blank? && response.code=='301' && response.body
|
889
|
+
new_endpoint = response.body[/<Endpoint>(.*?)<\/Endpoint>/] && $1
|
890
|
+
location = "#{request[:protocol]}://#{new_endpoint}:#{request[:port]}#{request[:request].path}"
|
891
|
+
end
|
892
|
+
# ... log information and ...
|
893
|
+
@aws.logger.info("##### #{@aws.class.name} redirect requested: #{response.code} #{response.message} #####")
|
894
|
+
@aws.logger.info(" Old location: #{request_text_data}")
|
895
|
+
@aws.logger.info(" New location: #{location}")
|
896
|
+
@aws.logger.info(" Request Verb: #{request[:request].class.name}")
|
897
|
+
# ... fix the connection data
|
898
|
+
request[:server] = URI.parse(location).host
|
899
|
+
request[:protocol] = URI.parse(location).scheme
|
900
|
+
request[:port] = URI.parse(location).port
|
901
|
+
else
|
902
|
+
# Not a redirect but an error: try to find the error in our list
|
903
|
+
@errors_list.each do |error_to_find|
|
904
|
+
if last_errors_text[/#{error_to_find}/i]
|
905
|
+
error_found = true
|
906
|
+
error_match = error_to_find
|
907
|
+
@aws.logger.warn("##### Retry is needed, error pattern match: #{error_to_find} #####")
|
908
|
+
break
|
909
|
+
end
|
910
|
+
end
|
911
|
+
end
|
912
|
+
|
913
|
+
# check the time has gone from the first error come
|
914
|
+
if redirect_detected || error_found
|
915
|
+
# Close the connection to the server and recreate a new one.
|
916
|
+
# It may have a chance that one server is a semi-down and reconnection
|
917
|
+
# will help us to connect to the other server
|
918
|
+
if !redirect_detected && @close_on_error
|
919
|
+
@aws.destroy_connection(request, "#{self.class.name}: error match to pattern '#{error_match}'")
|
920
|
+
end
|
921
|
+
|
922
|
+
if (Time.now < @stop_at)
|
923
|
+
@retries += 1
|
924
|
+
unless redirect_detected
|
925
|
+
@aws.logger.warn("##### Retry ##{@retries} is being performed. Sleeping for #{@reiteration_delay} sec. Whole time: #{Time.now-@started_at} sec ####")
|
926
|
+
sleep @reiteration_delay
|
927
|
+
@reiteration_delay *= 2
|
928
|
+
|
929
|
+
# Always make sure that the fp is set to point to the beginning(?)
|
930
|
+
# of the File/IO. TODO: it assumes that offset is 0, which is bad.
|
931
|
+
if(request[:request].body_stream && request[:request].body_stream.respond_to?(:pos))
|
932
|
+
begin
|
933
|
+
request[:request].body_stream.pos = 0
|
934
|
+
rescue Exception => e
|
935
|
+
@logger.warn("Retry may fail due to unable to reset the file pointer" +
|
936
|
+
" -- #{self.class.name} : #{e.inspect}")
|
937
|
+
end
|
938
|
+
end
|
939
|
+
else
|
940
|
+
@aws.logger.info("##### Retry ##{@retries} is being performed due to a redirect. ####")
|
941
|
+
end
|
942
|
+
result = @aws.request_info(request, @parser)
|
943
|
+
else
|
944
|
+
@aws.logger.warn("##### Ooops, time is over... ####")
|
945
|
+
end
|
946
|
+
# aha, this is unhandled error:
|
947
|
+
elsif @close_on_error
|
948
|
+
# On 5xx(Server errors), 403(RequestTimeTooSkewed) and 408(Request Timeout) a conection has to be closed
|
949
|
+
if @aws.last_response.code.to_s[/^(5\d\d|403|408)$/]
|
950
|
+
@aws.destroy_connection(request, "#{self.class.name}: code: #{@aws.last_response.code}: '#{@aws.last_response.message}'")
|
951
|
+
# Is this a 4xx error ?
|
952
|
+
elsif @aws.last_response.code.to_s[/^4\d\d$/] && @close_on_4xx_probability > rand(100)
|
953
|
+
@aws.destroy_connection(request, "#{self.class.name}: code: #{@aws.last_response.code}: '#{@aws.last_response.message}', " +
|
954
|
+
"probability: #{@close_on_4xx_probability}%")
|
955
|
+
end
|
956
|
+
end
|
957
|
+
result
|
958
|
+
end
|
959
|
+
|
960
|
+
end
|
961
|
+
|
962
|
+
|
963
|
+
#-----------------------------------------------------------------
|
964
|
+
|
965
|
+
class RightSaxParserCallbackTemplate #:nodoc:
|
966
|
+
def initialize(right_aws_parser)
|
967
|
+
@right_aws_parser = right_aws_parser
|
968
|
+
end
|
969
|
+
def on_characters(chars)
|
970
|
+
@right_aws_parser.text(chars)
|
971
|
+
end
|
972
|
+
def on_start_document; end
|
973
|
+
def on_comment(msg); end
|
974
|
+
def on_processing_instruction(target, data); end
|
975
|
+
def on_cdata_block(cdata); end
|
976
|
+
def on_end_document; end
|
977
|
+
end
|
978
|
+
|
979
|
+
class RightSaxParserCallback < RightSaxParserCallbackTemplate
|
980
|
+
def self.include_callback
|
981
|
+
include XML::SaxParser::Callbacks
|
982
|
+
end
|
983
|
+
def on_start_element(name, attr_hash)
|
984
|
+
@right_aws_parser.tag_start(name, attr_hash)
|
985
|
+
end
|
986
|
+
def on_end_element(name)
|
987
|
+
@right_aws_parser.tag_end(name)
|
988
|
+
end
|
989
|
+
end
|
990
|
+
|
991
|
+
class RightSaxParserCallbackNs < RightSaxParserCallbackTemplate
|
992
|
+
def on_start_element_ns(name, attr_hash, prefix, uri, namespaces)
|
993
|
+
@right_aws_parser.tag_start(name, attr_hash)
|
994
|
+
end
|
995
|
+
def on_end_element_ns(name, prefix, uri)
|
996
|
+
@right_aws_parser.tag_end(name)
|
997
|
+
end
|
998
|
+
end
|
999
|
+
|
1000
|
+
class RightAWSParser #:nodoc:
|
1001
|
+
# default parsing library
|
1002
|
+
DEFAULT_XML_LIBRARY = 'rexml'
|
1003
|
+
# a list of supported parsers
|
1004
|
+
@@supported_xml_libs = [DEFAULT_XML_LIBRARY, 'libxml']
|
1005
|
+
|
1006
|
+
@@xml_lib = DEFAULT_XML_LIBRARY # xml library name: 'rexml' | 'libxml'
|
1007
|
+
def self.xml_lib
|
1008
|
+
@@xml_lib
|
1009
|
+
end
|
1010
|
+
def self.xml_lib=(new_lib_name)
|
1011
|
+
@@xml_lib = new_lib_name
|
1012
|
+
end
|
1013
|
+
|
1014
|
+
attr_accessor :result
|
1015
|
+
attr_reader :xmlpath
|
1016
|
+
attr_accessor :xml_lib
|
1017
|
+
attr_reader :full_tag_name
|
1018
|
+
attr_reader :tag
|
1019
|
+
|
1020
|
+
def initialize(params={})
|
1021
|
+
@xmlpath = ''
|
1022
|
+
@full_tag_name = ''
|
1023
|
+
@result = false
|
1024
|
+
@text = ''
|
1025
|
+
@tag = ''
|
1026
|
+
@xml_lib = params[:xml_lib] || @@xml_lib
|
1027
|
+
@logger = params[:logger]
|
1028
|
+
reset
|
1029
|
+
end
|
1030
|
+
def tag_start(name, attributes)
|
1031
|
+
@text = ''
|
1032
|
+
@tag = name
|
1033
|
+
@full_tag_name += @full_tag_name.empty? ? name : "/#{name}"
|
1034
|
+
tagstart(name, attributes)
|
1035
|
+
@xmlpath = @full_tag_name
|
1036
|
+
end
|
1037
|
+
def tag_end(name)
|
1038
|
+
@xmlpath = @full_tag_name[/^(.*?)\/?#{name}$/] && $1
|
1039
|
+
tagend(name)
|
1040
|
+
@full_tag_name = @xmlpath
|
1041
|
+
end
|
1042
|
+
def text(text)
|
1043
|
+
@text += text
|
1044
|
+
tagtext(text)
|
1045
|
+
end
|
1046
|
+
# Parser method.
|
1047
|
+
# Params:
|
1048
|
+
# xml_text - xml message text(String) or Net:HTTPxxx instance (response)
|
1049
|
+
# params[:xml_lib] - library name: 'rexml' | 'libxml'
|
1050
|
+
def parse(xml_text, params={})
|
1051
|
+
# Get response body
|
1052
|
+
xml_text = xml_text.body unless xml_text.is_a?(String)
|
1053
|
+
@xml_lib = params[:xml_lib] || @xml_lib
|
1054
|
+
# check that we had no problems with this library otherwise use default
|
1055
|
+
@xml_lib = DEFAULT_XML_LIBRARY unless @@supported_xml_libs.include?(@xml_lib)
|
1056
|
+
# load xml library
|
1057
|
+
if @xml_lib=='libxml' && !defined?(XML::SaxParser)
|
1058
|
+
begin
|
1059
|
+
require 'xml/libxml'
|
1060
|
+
# Setup SaxParserCallback
|
1061
|
+
if XML::Parser::VERSION >= '0.5.1' &&
|
1062
|
+
XML::Parser::VERSION < '0.9.7'
|
1063
|
+
RightSaxParserCallback.include_callback
|
1064
|
+
end
|
1065
|
+
rescue LoadError => e
|
1066
|
+
@@supported_xml_libs.delete(@xml_lib)
|
1067
|
+
@xml_lib = DEFAULT_XML_LIBRARY
|
1068
|
+
if @logger
|
1069
|
+
@logger.error e.inspect
|
1070
|
+
@logger.error e.backtrace
|
1071
|
+
@logger.info "Can not load 'libxml' library. '#{DEFAULT_XML_LIBRARY}' is used for parsing."
|
1072
|
+
end
|
1073
|
+
end
|
1074
|
+
end
|
1075
|
+
# Parse the xml text
|
1076
|
+
case @xml_lib
|
1077
|
+
when 'libxml'
|
1078
|
+
if XML::Parser::VERSION >= '0.9.9'
|
1079
|
+
# avoid warning on every usage
|
1080
|
+
xml = XML::SaxParser.string(xml_text)
|
1081
|
+
else
|
1082
|
+
xml = XML::SaxParser.new
|
1083
|
+
xml.string = xml_text
|
1084
|
+
end
|
1085
|
+
# check libxml-ruby version
|
1086
|
+
if XML::Parser::VERSION >= '0.9.7'
|
1087
|
+
xml.callbacks = RightSaxParserCallbackNs.new(self)
|
1088
|
+
elsif XML::Parser::VERSION >= '0.5.1'
|
1089
|
+
xml.callbacks = RightSaxParserCallback.new(self)
|
1090
|
+
else
|
1091
|
+
xml.on_start_element{|name, attr_hash| self.tag_start(name, attr_hash)}
|
1092
|
+
xml.on_characters{ |text| self.text(text)}
|
1093
|
+
xml.on_end_element{ |name| self.tag_end(name)}
|
1094
|
+
end
|
1095
|
+
xml.parse
|
1096
|
+
else
|
1097
|
+
REXML::Document.parse_stream(xml_text, self)
|
1098
|
+
end
|
1099
|
+
end
|
1100
|
+
# Parser must have a lots of methods
|
1101
|
+
# (see /usr/lib/ruby/1.8/rexml/parsers/streamparser.rb)
|
1102
|
+
# We dont need most of them in RightAWSParser and method_missing helps us
|
1103
|
+
# to skip their definition
|
1104
|
+
def method_missing(method, *params)
|
1105
|
+
# if the method is one of known - just skip it ...
|
1106
|
+
return if [:comment, :attlistdecl, :notationdecl, :elementdecl,
|
1107
|
+
:entitydecl, :cdata, :xmldecl, :attlistdecl, :instruction,
|
1108
|
+
:doctype].include?(method)
|
1109
|
+
# ... else - call super to raise an exception
|
1110
|
+
super(method, params)
|
1111
|
+
end
|
1112
|
+
# the functions to be overriden by children (if nessesery)
|
1113
|
+
def reset ; end
|
1114
|
+
def tagstart(name, attributes); end
|
1115
|
+
def tagend(name) ; end
|
1116
|
+
def tagtext(text) ; end
|
1117
|
+
end
|
1118
|
+
|
1119
|
+
#-----------------------------------------------------------------
|
1120
|
+
# PARSERS: Errors
|
1121
|
+
#-----------------------------------------------------------------
|
1122
|
+
|
1123
|
+
#<Error>
|
1124
|
+
# <Code>TemporaryRedirect</Code>
|
1125
|
+
# <Message>Please re-send this request to the specified temporary endpoint. Continue to use the original request endpoint for future requests.</Message>
|
1126
|
+
# <RequestId>FD8D5026D1C5ABA3</RequestId>
|
1127
|
+
# <Endpoint>bucket-for-k.s3-external-3.amazonaws.com</Endpoint>
|
1128
|
+
# <HostId>ItJy8xPFPli1fq/JR3DzQd3iDvFCRqi1LTRmunEdM1Uf6ZtW2r2kfGPWhRE1vtaU</HostId>
|
1129
|
+
# <Bucket>bucket-for-k</Bucket>
|
1130
|
+
#</Error>
|
1131
|
+
|
1132
|
+
class RightErrorResponseParser < RightAWSParser #:nodoc:
|
1133
|
+
attr_accessor :errors # array of hashes: error/message
|
1134
|
+
attr_accessor :requestID
|
1135
|
+
# attr_accessor :endpoint, :host_id, :bucket
|
1136
|
+
def tagend(name)
|
1137
|
+
case name
|
1138
|
+
when 'RequestID' ; @requestID = @text
|
1139
|
+
when 'Code' ; @code = @text
|
1140
|
+
when 'Message' ; @message = @text
|
1141
|
+
# when 'Endpoint' ; @endpoint = @text
|
1142
|
+
# when 'HostId' ; @host_id = @text
|
1143
|
+
# when 'Bucket' ; @bucket = @text
|
1144
|
+
when 'Error' ; @errors << [ @code, @message ]
|
1145
|
+
end
|
1146
|
+
end
|
1147
|
+
def reset
|
1148
|
+
@errors = []
|
1149
|
+
end
|
1150
|
+
end
|
1151
|
+
|
1152
|
+
# Dummy parser - does nothing
|
1153
|
+
# Returns the original params back
|
1154
|
+
class RightDummyParser # :nodoc:
|
1155
|
+
attr_accessor :result
|
1156
|
+
def parse(response, params={})
|
1157
|
+
@result = [response, params]
|
1158
|
+
end
|
1159
|
+
end
|
1160
|
+
|
1161
|
+
class RightHttp2xxParser < RightAWSParser # :nodoc:
|
1162
|
+
def parse(response)
|
1163
|
+
@result = response.is_a?(Net::HTTPSuccess)
|
1164
|
+
end
|
1165
|
+
end
|
1166
|
+
|
1167
|
+
class RightBoolResponseParser < RightAWSParser #:nodoc:
|
1168
|
+
def tagend(name)
|
1169
|
+
@result = (@text=='true') if name == 'return'
|
1170
|
+
end
|
1171
|
+
end
|
1172
|
+
|
1173
|
+
end
|
1174
|
+
|