dustMason-right_aws 2.1.0

Sign up to get free protection for your applications and to get access to all the features.
Files changed (70) hide show
  1. data/History.txt +305 -0
  2. data/Manifest.txt +60 -0
  3. data/README.txt +165 -0
  4. data/Rakefile +112 -0
  5. data/lib/acf/right_acf_interface.rb +549 -0
  6. data/lib/acf/right_acf_invalidations.rb +144 -0
  7. data/lib/acf/right_acf_origin_access_identities.rb +230 -0
  8. data/lib/acf/right_acf_streaming_interface.rb +229 -0
  9. data/lib/acw/right_acw_interface.rb +248 -0
  10. data/lib/as/right_as_interface.rb +698 -0
  11. data/lib/awsbase/benchmark_fix.rb +39 -0
  12. data/lib/awsbase/right_awsbase.rb +1174 -0
  13. data/lib/awsbase/support.rb +35 -0
  14. data/lib/awsbase/version.rb +9 -0
  15. data/lib/ec2/right_ec2.rb +458 -0
  16. data/lib/ec2/right_ec2_ebs.rb +465 -0
  17. data/lib/ec2/right_ec2_images.rb +413 -0
  18. data/lib/ec2/right_ec2_instances.rb +785 -0
  19. data/lib/ec2/right_ec2_monitoring.rb +70 -0
  20. data/lib/ec2/right_ec2_placement_groups.rb +108 -0
  21. data/lib/ec2/right_ec2_reserved_instances.rb +174 -0
  22. data/lib/ec2/right_ec2_security_groups.rb +396 -0
  23. data/lib/ec2/right_ec2_spot_instances.rb +425 -0
  24. data/lib/ec2/right_ec2_tags.rb +139 -0
  25. data/lib/ec2/right_ec2_vpc.rb +583 -0
  26. data/lib/ec2/right_ec2_windows_mobility.rb +84 -0
  27. data/lib/elb/right_elb_interface.rb +571 -0
  28. data/lib/iam/right_iam_access_keys.rb +71 -0
  29. data/lib/iam/right_iam_groups.rb +195 -0
  30. data/lib/iam/right_iam_interface.rb +341 -0
  31. data/lib/iam/right_iam_mfa_devices.rb +67 -0
  32. data/lib/iam/right_iam_users.rb +251 -0
  33. data/lib/rds/right_rds_interface.rb +1309 -0
  34. data/lib/right_aws.rb +88 -0
  35. data/lib/route_53/right_route_53_interface.rb +630 -0
  36. data/lib/s3/right_s3.rb +1123 -0
  37. data/lib/s3/right_s3_interface.rb +1198 -0
  38. data/lib/sdb/active_sdb.rb +1107 -0
  39. data/lib/sdb/right_sdb_interface.rb +753 -0
  40. data/lib/sns/right_sns.rb +205 -0
  41. data/lib/sns/right_sns_interface.rb +343 -0
  42. data/lib/sqs/right_sqs.rb +387 -0
  43. data/lib/sqs/right_sqs_gen2.rb +342 -0
  44. data/lib/sqs/right_sqs_gen2_interface.rb +523 -0
  45. data/lib/sqs/right_sqs_interface.rb +593 -0
  46. data/right_aws.gemspec +91 -0
  47. data/test/acf/test_helper.rb +2 -0
  48. data/test/acf/test_right_acf.rb +138 -0
  49. data/test/awsbase/test_helper.rb +2 -0
  50. data/test/awsbase/test_right_awsbase.rb +12 -0
  51. data/test/ec2/test_helper.rb +2 -0
  52. data/test/ec2/test_right_ec2.rb +108 -0
  53. data/test/http_connection.rb +87 -0
  54. data/test/rds/test_helper.rb +2 -0
  55. data/test/rds/test_right_rds.rb +120 -0
  56. data/test/s3/test_helper.rb +2 -0
  57. data/test/s3/test_right_s3.rb +421 -0
  58. data/test/s3/test_right_s3_stubbed.rb +97 -0
  59. data/test/sdb/test_active_sdb.rb +357 -0
  60. data/test/sdb/test_batch_put_attributes.rb +54 -0
  61. data/test/sdb/test_helper.rb +3 -0
  62. data/test/sdb/test_right_sdb.rb +253 -0
  63. data/test/sns/test_helper.rb +2 -0
  64. data/test/sns/test_right_sns.rb +73 -0
  65. data/test/sqs/test_helper.rb +2 -0
  66. data/test/sqs/test_right_sqs.rb +285 -0
  67. data/test/sqs/test_right_sqs_gen2.rb +264 -0
  68. data/test/test_credentials.rb +37 -0
  69. data/test/ts_right_aws.rb +14 -0
  70. metadata +244 -0
@@ -0,0 +1,39 @@
1
+ #
2
+ # Copyright (c) 2007-2008 RightScale Inc
3
+ #
4
+ # Permission is hereby granted, free of charge, to any person obtaining
5
+ # a copy of this software and associated documentation files (the
6
+ # "Software"), to deal in the Software without restriction, including
7
+ # without limitation the rights to use, copy, modify, merge, publish,
8
+ # distribute, sublicense, and/or sell copies of the Software, and to
9
+ # permit persons to whom the Software is furnished to do so, subject to
10
+ # the following conditions:
11
+ #
12
+ # The above copyright notice and this permission notice shall be
13
+ # included in all copies or substantial portions of the Software.
14
+ #
15
+ # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
16
+ # EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
17
+ # MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
18
+ # NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
19
+ # LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
20
+ # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
21
+ # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
22
+ #
23
+ #
24
+
25
+
26
+ # A hack because there's a bug in add! in Benchmark::Tms
27
+ module Benchmark #:nodoc:
28
+ class Tms #:nodoc:
29
+ def add!(&blk)
30
+ t = Benchmark::measure(&blk)
31
+ @utime = utime + t.utime
32
+ @stime = stime + t.stime
33
+ @cutime = cutime + t.cutime
34
+ @cstime = cstime + t.cstime
35
+ @real = real + t.real
36
+ self
37
+ end
38
+ end
39
+ end
@@ -0,0 +1,1174 @@
1
+ #
2
+ # Copyright (c) 2007-2008 RightScale Inc
3
+ #
4
+ # Permission is hereby granted, free of charge, to any person obtaining
5
+ # a copy of this software and associated documentation files (the
6
+ # "Software"), to deal in the Software without restriction, including
7
+ # without limitation the rights to use, copy, modify, merge, publish,
8
+ # distribute, sublicense, and/or sell copies of the Software, and to
9
+ # permit persons to whom the Software is furnished to do so, subject to
10
+ # the following conditions:
11
+ #
12
+ # The above copyright notice and this permission notice shall be
13
+ # included in all copies or substantial portions of the Software.
14
+ #
15
+ # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
16
+ # EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
17
+ # MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
18
+ # NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
19
+ # LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
20
+ # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
21
+ # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
22
+ #
23
+
24
+ # Test
25
+ module RightAws
26
+ require 'digest/md5'
27
+ require 'pp'
28
+
29
+ class AwsUtils #:nodoc:
30
+ @@digest1 = OpenSSL::Digest::Digest.new("sha1")
31
+ @@digest256 = nil
32
+ if OpenSSL::OPENSSL_VERSION_NUMBER > 0x00908000
33
+ @@digest256 = OpenSSL::Digest::Digest.new("sha256") rescue nil # Some installation may not support sha256
34
+ end
35
+
36
+ def self.utc_iso8601(time)
37
+ if time.is_a?(Fixnum) then time = Time::at(time)
38
+ elsif time.is_a?(String) then time = Time::parse(time)
39
+ end
40
+ time.utc.strftime("%Y-%m-%dT%H:%M:%S.000Z")
41
+ end
42
+
43
+ def self.sign(aws_secret_access_key, auth_string)
44
+ Base64.encode64(OpenSSL::HMAC.digest(@@digest1, aws_secret_access_key, auth_string)).strip
45
+ end
46
+
47
+ # Escape a string accordingly Amazon rulles
48
+ # http://docs.amazonwebservices.com/AmazonSimpleDB/2007-11-07/DeveloperGuide/index.html?REST_RESTAuth.html
49
+ def self.amz_escape(param)
50
+ param.to_s.gsub(/([^a-zA-Z0-9._~-]+)/n) do
51
+ '%' + $1.unpack('H2' * $1.size).join('%').upcase
52
+ end
53
+ end
54
+
55
+ def self.xml_escape(text) # :nodoc:
56
+ REXML::Text::normalize(text)
57
+ end
58
+
59
+ def self.xml_unescape(text) # :nodoc:
60
+ REXML::Text::unnormalize(text)
61
+ end
62
+
63
+ # Set a timestamp and a signature version
64
+ def self.fix_service_params(service_hash, signature)
65
+ service_hash["Timestamp"] ||= utc_iso8601(Time.now) unless service_hash["Expires"]
66
+ service_hash["SignatureVersion"] = signature
67
+ service_hash
68
+ end
69
+
70
+ # Signature Version 0
71
+ # A deprecated guy (should work till septemper 2009)
72
+ def self.sign_request_v0(aws_secret_access_key, service_hash)
73
+ fix_service_params(service_hash, '0')
74
+ string_to_sign = "#{service_hash['Action']}#{service_hash['Timestamp'] || service_hash['Expires']}"
75
+ service_hash['Signature'] = AwsUtils::sign(aws_secret_access_key, string_to_sign)
76
+ service_hash.to_a.collect{|key,val| "#{amz_escape(key)}=#{amz_escape(val.to_s)}" }.join("&")
77
+ end
78
+
79
+ # Signature Version 1
80
+ # Another deprecated guy (should work till septemper 2009)
81
+ def self.sign_request_v1(aws_secret_access_key, service_hash)
82
+ fix_service_params(service_hash, '1')
83
+ string_to_sign = service_hash.sort{|a,b| (a[0].to_s.downcase)<=>(b[0].to_s.downcase)}.to_s
84
+ service_hash['Signature'] = AwsUtils::sign(aws_secret_access_key, string_to_sign)
85
+ service_hash.to_a.collect{|key,val| "#{amz_escape(key)}=#{amz_escape(val.to_s)}" }.join("&")
86
+ end
87
+
88
+ # Signature Version 2
89
+ # EC2, SQS and SDB requests must be signed by this guy.
90
+ # See: http://docs.amazonwebservices.com/AmazonSimpleDB/2007-11-07/DeveloperGuide/index.html?REST_RESTAuth.html
91
+ # http://developer.amazonwebservices.com/connect/entry.jspa?externalID=1928
92
+ def self.sign_request_v2(aws_secret_access_key, service_hash, http_verb, host, uri)
93
+ fix_service_params(service_hash, '2')
94
+ # select a signing method (make an old openssl working with sha1)
95
+ # make 'HmacSHA256' to be a default one
96
+ service_hash['SignatureMethod'] = 'HmacSHA256' unless ['HmacSHA256', 'HmacSHA1'].include?(service_hash['SignatureMethod'])
97
+ service_hash['SignatureMethod'] = 'HmacSHA1' unless @@digest256
98
+ # select a digest
99
+ digest = (service_hash['SignatureMethod'] == 'HmacSHA256' ? @@digest256 : @@digest1)
100
+ # form string to sign
101
+ canonical_string = service_hash.keys.sort.map do |key|
102
+ "#{amz_escape(key)}=#{amz_escape(service_hash[key])}"
103
+ end.join('&')
104
+ string_to_sign = "#{http_verb.to_s.upcase}\n#{host.downcase}\n#{uri}\n#{canonical_string}"
105
+ # sign the string
106
+ signature = amz_escape(Base64.encode64(OpenSSL::HMAC.digest(digest, aws_secret_access_key, string_to_sign)).strip)
107
+ "#{canonical_string}&Signature=#{signature}"
108
+ end
109
+
110
+ # From Amazon's SQS Dev Guide, a brief description of how to escape:
111
+ # "URL encode the computed signature and other query parameters as specified in
112
+ # RFC1738, section 2.2. In addition, because the + character is interpreted as a blank space
113
+ # by Sun Java classes that perform URL decoding, make sure to encode the + character
114
+ # although it is not required by RFC1738."
115
+ # Avoid using CGI::escape to escape URIs.
116
+ # CGI::escape will escape characters in the protocol, host, and port
117
+ # sections of the URI. Only target chars in the query
118
+ # string should be escaped.
119
+ def self.URLencode(raw)
120
+ e = URI.escape(raw)
121
+ e.gsub(/\+/, "%2b")
122
+ end
123
+
124
+ def self.allow_only(allowed_keys, params)
125
+ bogus_args = []
126
+ params.keys.each {|p| bogus_args.push(p) unless allowed_keys.include?(p) }
127
+ raise AwsError.new("The following arguments were given but are not legal for the function call #{caller_method}: #{bogus_args.inspect}") if bogus_args.length > 0
128
+ end
129
+
130
+ def self.mandatory_arguments(required_args, params)
131
+ rargs = required_args.dup
132
+ params.keys.each {|p| rargs.delete(p)}
133
+ raise AwsError.new("The following mandatory arguments were not provided to #{caller_method}: #{rargs.inspect}") if rargs.length > 0
134
+ end
135
+
136
+ def self.caller_method
137
+ caller[1]=~/`(.*?)'/
138
+ $1
139
+ end
140
+
141
+ def self.split_items_and_params(array)
142
+ items = Array(array).flatten.compact
143
+ params = items.last.kind_of?(Hash) ? items.pop : {}
144
+ [items, params]
145
+ end
146
+
147
+ # Generates a token in format of:
148
+ # 1. "1dd8d4e4-db6b-11df-b31d-0025b37efad0 (if UUID gem is loaded)
149
+ # 2. "1287483761-855215-zSv2z-bWGj2-31M5t-ags9m" (if UUID gem is not loaded)
150
+ TOKEN_GENERATOR_CHARSET = ('a'..'z').to_a + ('A'..'Z').to_a + ('0'..'9').to_a
151
+ def self.generate_unique_token
152
+ time = Time.now
153
+ token = "%d-%06d" % [time.to_i, time.usec]
154
+ 4.times do
155
+ token << "-"
156
+ 5.times { token << TOKEN_GENERATOR_CHARSET[rand(TOKEN_GENERATOR_CHARSET.size)] }
157
+ end
158
+ token
159
+ end
160
+ end
161
+
162
+ class AwsBenchmarkingBlock #:nodoc:
163
+ attr_accessor :xml, :service
164
+ def initialize
165
+ # Benchmark::Tms instance for service (Ec2, S3, or SQS) access benchmarking.
166
+ @service = Benchmark::Tms.new()
167
+ # Benchmark::Tms instance for XML parsing benchmarking.
168
+ @xml = Benchmark::Tms.new()
169
+ end
170
+ end
171
+
172
+ class AwsNoChange < RuntimeError
173
+ end
174
+
175
+ class RightAwsBase
176
+
177
+ # Amazon HTTP Error handling
178
+
179
+ # Text, if found in an error message returned by AWS, indicates that this may be a transient
180
+ # error. Transient errors are automatically retried with exponential back-off.
181
+ AMAZON_PROBLEMS = [ 'internal service error',
182
+ 'is currently unavailable',
183
+ 'no response from',
184
+ 'Please try again',
185
+ 'InternalError',
186
+ 'Internal Server Error',
187
+ 'ServiceUnavailable', #from SQS docs
188
+ 'Unavailable',
189
+ 'This application is not currently available',
190
+ 'InsufficientInstanceCapacity'
191
+ ]
192
+ @@amazon_problems = AMAZON_PROBLEMS
193
+ # Returns a list of Amazon service responses which are known to be transient problems.
194
+ # We have to re-request if we get any of them, because the problem will probably disappear.
195
+ # By default this method returns the same value as the AMAZON_PROBLEMS const.
196
+ def self.amazon_problems
197
+ @@amazon_problems
198
+ end
199
+
200
+ # Sets the list of Amazon side problems. Use in conjunction with the
201
+ # getter to append problems.
202
+ def self.amazon_problems=(problems_list)
203
+ @@amazon_problems = problems_list
204
+ end
205
+
206
+ # Raise an exception if a timeout occures while an API call is in progress.
207
+ # This helps to avoid a duplicate resources creation when Amazon hangs for some time and
208
+ # RightHttpConnection is forced to use retries to get a response from it.
209
+ #
210
+ # If an API call action is in the list then no attempts to retry are performed.
211
+ #
212
+ RAISE_ON_TIMEOUT_ON_ACTIONS = %w{
213
+ AllocateAddress
214
+ CreateSnapshot
215
+ CreateVolume
216
+ PurchaseReservedInstancesOffering
217
+ RequestSpotInstances
218
+ RunInstances
219
+ }
220
+ @@raise_on_timeout_on_actions = RAISE_ON_TIMEOUT_ON_ACTIONS.dup
221
+
222
+ def self.raise_on_timeout_on_actions
223
+ @@raise_on_timeout_on_actions
224
+ end
225
+
226
+ def self.raise_on_timeout_on_actions=(actions_list)
227
+ @@raise_on_timeout_on_actions = actions_list
228
+ end
229
+
230
+ end
231
+
232
+ module RightAwsBaseInterface
233
+ DEFAULT_SIGNATURE_VERSION = '2'
234
+
235
+ @@caching = false
236
+ def self.caching
237
+ @@caching
238
+ end
239
+ def self.caching=(caching)
240
+ @@caching = caching
241
+ end
242
+
243
+ # Current aws_access_key_id
244
+ attr_reader :aws_access_key_id
245
+ # Current aws_secret_access_key
246
+ attr_reader :aws_secret_access_key
247
+ # Last HTTP request object
248
+ attr_reader :last_request
249
+ # Last HTTP response object
250
+ attr_reader :last_response
251
+ # Last AWS errors list (used by AWSErrorHandler)
252
+ attr_accessor :last_errors
253
+ # Last AWS request id (used by AWSErrorHandler)
254
+ attr_accessor :last_request_id
255
+ # Logger object
256
+ attr_accessor :logger
257
+ # Initial params hash
258
+ attr_accessor :params
259
+ # RightHttpConnection instance
260
+ attr_reader :connection
261
+ # Cache
262
+ attr_reader :cache
263
+ # Signature version (all services except s3)
264
+ attr_reader :signature_version
265
+
266
+ def init(service_info, aws_access_key_id, aws_secret_access_key, params={}) #:nodoc:
267
+ @params = params
268
+ # If one defines EC2_URL he may forget to use a single slash as an "empty service" path.
269
+ # Amazon does not like this therefore add this bad boy if he is missing...
270
+ service_info[:default_service] = '/' if service_info[:default_service].right_blank?
271
+ raise AwsError.new("AWS access keys are required to operate on #{service_info[:name]}") \
272
+ if aws_access_key_id.right_blank? || aws_secret_access_key.right_blank?
273
+ @aws_access_key_id = aws_access_key_id
274
+ @aws_secret_access_key = aws_secret_access_key
275
+ # if the endpoint was explicitly defined - then use it
276
+ if @params[:endpoint_url]
277
+ @params[:server] = URI.parse(@params[:endpoint_url]).host
278
+ @params[:port] = URI.parse(@params[:endpoint_url]).port
279
+ @params[:service] = URI.parse(@params[:endpoint_url]).path
280
+ # make sure the 'service' path is not empty
281
+ @params[:service] = service_info[:default_service] if @params[:service].right_blank?
282
+ @params[:protocol] = URI.parse(@params[:endpoint_url]).scheme
283
+ @params[:region] = nil
284
+ else
285
+ @params[:server] ||= service_info[:default_host]
286
+ @params[:server] = "#{@params[:region]}.#{@params[:server]}" if @params[:region]
287
+ @params[:port] ||= service_info[:default_port]
288
+ @params[:service] ||= service_info[:default_service]
289
+ @params[:protocol] ||= service_info[:default_protocol]
290
+ end
291
+ # a set of options to be passed to RightHttpConnection object
292
+ @params[:connection_options] = {} unless @params[:connection_options].is_a?(Hash)
293
+ @with_connection_options = {}
294
+ @params[:connections] ||= :shared # || :dedicated
295
+ @params[:max_connections] ||= 10
296
+ @params[:connection_lifetime] ||= 20*60
297
+ @params[:api_version] ||= service_info[:default_api_version]
298
+ @logger = @params[:logger]
299
+ @logger = ::Rails.logger if !@logger && defined?(::Rails) && ::Rails.respond_to?(:logger)
300
+ @logger = RAILS_DEFAULT_LOGGER if !@logger && defined?(RAILS_DEFAULT_LOGGER)
301
+ @logger = Logger.new(STDOUT) if !@logger
302
+ @logger.info "New #{self.class.name} using #{@params[:connections]} connections mode"
303
+ @error_handler = nil
304
+ @cache = {}
305
+ @signature_version = (params[:signature_version] || DEFAULT_SIGNATURE_VERSION).to_s
306
+ end
307
+
308
+ def signed_service_params(aws_secret_access_key, service_hash, http_verb=nil, host=nil, service=nil )
309
+ case signature_version.to_s
310
+ when '0' then AwsUtils::sign_request_v0(aws_secret_access_key, service_hash)
311
+ when '1' then AwsUtils::sign_request_v1(aws_secret_access_key, service_hash)
312
+ when '2' then AwsUtils::sign_request_v2(aws_secret_access_key, service_hash, http_verb, host, service)
313
+ else raise AwsError.new("Unknown signature version (#{signature_version.to_s}) requested")
314
+ end
315
+ end
316
+
317
+ # Returns +true+ if the describe_xxx responses are being cached
318
+ def caching?
319
+ @params.key?(:cache) ? @params[:cache] : @@caching
320
+ end
321
+
322
+ # Check if the aws function response hits the cache or not.
323
+ # If the cache hits:
324
+ # - raises an +AwsNoChange+ exception if +do_raise+ == +:raise+.
325
+ # - returnes parsed response from the cache if it exists or +true+ otherwise.
326
+ # If the cache miss or the caching is off then returns +false+.
327
+ def cache_hits?(function, response, do_raise=:raise)
328
+ result = false
329
+ if caching?
330
+ function = function.to_sym
331
+ # get rid of requestId (this bad boy was added for API 2008-08-08+ and it is uniq for every response)
332
+ # feb 04, 2009 (load balancer uses 'RequestId' hence use 'i' modifier to hit it also)
333
+ response = response.sub(%r{<requestId>.+?</requestId>}i, '')
334
+ response_md5 = MD5.md5(response).to_s
335
+ # check for changes
336
+ unless @cache[function] && @cache[function][:response_md5] == response_md5
337
+ # well, the response is new, reset cache data
338
+ update_cache(function, {:response_md5 => response_md5,
339
+ :timestamp => Time.now,
340
+ :hits => 0,
341
+ :parsed => nil})
342
+ else
343
+ # aha, cache hits, update the data and throw an exception if needed
344
+ @cache[function][:hits] += 1
345
+ if do_raise == :raise
346
+ raise(AwsNoChange, "Cache hit: #{function} response has not changed since "+
347
+ "#{@cache[function][:timestamp].strftime('%Y-%m-%d %H:%M:%S')}, "+
348
+ "hits: #{@cache[function][:hits]}.")
349
+ else
350
+ result = @cache[function][:parsed] || true
351
+ end
352
+ end
353
+ end
354
+ result
355
+ end
356
+
357
+ def update_cache(function, hash)
358
+ (@cache[function.to_sym] ||= {}).merge!(hash) if caching?
359
+ end
360
+
361
+ def on_exception(options={:raise=>true, :log=>true}) # :nodoc:
362
+ raise if $!.is_a?(AwsNoChange)
363
+ AwsError::on_aws_exception(self, options)
364
+ end
365
+
366
+ #----------------------------
367
+ # HTTP Connections handling
368
+ #----------------------------
369
+
370
+ def get_server_url(request) # :nodoc:
371
+ "#{request[:protocol]}://#{request[:server]}:#{request[:port]}"
372
+ end
373
+
374
+ def get_connections_storage(aws_service) # :nodoc:
375
+ case @params[:connections].to_s
376
+ when 'dedicated' then @connections_storage ||= {}
377
+ else Thread.current[aws_service] ||= {}
378
+ end
379
+ end
380
+
381
+ def destroy_connection(request, reason) # :nodoc:
382
+ connections = get_connections_storage(request[:aws_service])
383
+ server_url = get_server_url(request)
384
+ if connections[server_url]
385
+ connections[server_url][:connection].finish(reason)
386
+ connections.delete(server_url)
387
+ end
388
+ end
389
+
390
+ # Expire the connection if it has expired.
391
+ def get_connection(request) # :nodoc:
392
+ server_url = get_server_url(request)
393
+ connection_storage = get_connections_storage(request[:aws_service])
394
+ life_time_scratch = Time.now-@params[:connection_lifetime]
395
+ # Delete out-of-dated connections
396
+ connections_in_list = 0
397
+ connection_storage.to_a.sort{|conn1, conn2| conn2[1][:last_used_at] <=> conn1[1][:last_used_at]}.each do |serv_url, conn_opts|
398
+ if @params[:max_connections] <= connections_in_list
399
+ conn_opts[:connection].finish('out-of-limit')
400
+ connection_storage.delete(server_url)
401
+ elsif conn_opts[:last_used_at] < life_time_scratch
402
+ conn_opts[:connection].finish('out-of-date')
403
+ connection_storage.delete(server_url)
404
+ else
405
+ connections_in_list += 1
406
+ end
407
+ end
408
+ connection = (connection_storage[server_url] ||= {})
409
+ connection[:last_used_at] = Time.now
410
+ connection[:connection] ||= Rightscale::HttpConnection.new(:exception => RightAws::AwsError, :logger => @logger)
411
+ end
412
+
413
+ #----------------------------
414
+ # HTTP Requests handling
415
+ #----------------------------
416
+
417
+ # ACF, AMS, EC2, LBS and SDB uses this guy
418
+ # SQS and S3 use their own methods
419
+ def generate_request_impl(verb, action, options={}) #:nodoc:
420
+ # Form a valid http verb: 'GET' or 'POST' (all the other are not supported now)
421
+ http_verb = verb.to_s.upcase
422
+ # remove empty keys from request options
423
+ options.delete_if { |key, value| value.nil? }
424
+ # prepare service data
425
+ service_hash = {"Action" => action,
426
+ "AWSAccessKeyId" => @aws_access_key_id,
427
+ "Version" => @params[:api_version] }
428
+ service_hash.merge!(options)
429
+ # Sign request options
430
+ service_params = signed_service_params(@aws_secret_access_key, service_hash, http_verb, @params[:server], @params[:service])
431
+ # Use POST if the length of the query string is too large
432
+ # see http://docs.amazonwebservices.com/AmazonSimpleDB/2007-11-07/DeveloperGuide/MakingRESTRequests.html
433
+ if http_verb != 'POST' && service_params.size > 2000
434
+ http_verb = 'POST'
435
+ if signature_version == '2'
436
+ service_params = signed_service_params(@aws_secret_access_key, service_hash, http_verb, @params[:server], @params[:service])
437
+ end
438
+ end
439
+ # create a request
440
+ case http_verb
441
+ when 'GET'
442
+ request = Net::HTTP::Get.new("#{@params[:service]}?#{service_params}")
443
+ when 'POST'
444
+ request = Net::HTTP::Post.new(@params[:service])
445
+ request.body = service_params
446
+ request['Content-Type'] = 'application/x-www-form-urlencoded; charset=utf-8'
447
+ else
448
+ raise "Unsupported HTTP verb #{verb.inspect}!"
449
+ end
450
+ # prepare output hash
451
+ request_hash = { :request => request,
452
+ :server => @params[:server],
453
+ :port => @params[:port],
454
+ :protocol => @params[:protocol] }
455
+ request_hash.merge!(@params[:connection_options])
456
+ request_hash.merge!(@with_connection_options)
457
+
458
+ # If an action is marked as "non-retryable" and there was no :raise_on_timeout option set
459
+ # explicitly then do set that option
460
+ if Array(RightAwsBase::raise_on_timeout_on_actions).include?(action) && !request_hash.has_key?(:raise_on_timeout)
461
+ request_hash.merge!(:raise_on_timeout => true)
462
+ end
463
+
464
+ request_hash
465
+ end
466
+
467
+ # All services uses this guy.
468
+ def request_info_impl(aws_service, benchblock, request, parser, &block) #:nodoc:
469
+ request[:aws_service] = aws_service
470
+ @connection = get_connection(request)
471
+ @last_request = request[:request]
472
+ @last_response = nil
473
+ response = nil
474
+ blockexception = nil
475
+
476
+ if(block != nil)
477
+ # TRB 9/17/07 Careful - because we are passing in blocks, we get a situation where
478
+ # an exception may get thrown in the block body (which is high-level
479
+ # code either here or in the application) but gets caught in the
480
+ # low-level code of HttpConnection. The solution is not to let any
481
+ # exception escape the block that we pass to HttpConnection::request.
482
+ # Exceptions can originate from code directly in the block, or from user
483
+ # code called in the other block which is passed to response.read_body.
484
+ benchblock.service.add! do
485
+ begin
486
+ responsehdr = @connection.request(request) do |response|
487
+ #########
488
+ begin
489
+ @last_response = response
490
+ if response.is_a?(Net::HTTPSuccess)
491
+ @error_handler = nil
492
+ response.read_body(&block)
493
+ else
494
+ @error_handler = AWSErrorHandler.new(self, parser, :errors_list => self.class.amazon_problems) unless @error_handler
495
+ check_result = @error_handler.check(request)
496
+ if check_result
497
+ @error_handler = nil
498
+ return check_result
499
+ end
500
+ raise AwsError.new(@last_errors, @last_response.code, @last_request_id)
501
+ end
502
+ rescue Exception => e
503
+ blockexception = e
504
+ end
505
+ end
506
+ rescue Exception => e
507
+ # Kill a connection if we run into a low level connection error
508
+ destroy_connection(request, "error: #{e.message}")
509
+ raise e
510
+ end
511
+ #########
512
+
513
+ #OK, now we are out of the block passed to the lower level
514
+ if(blockexception)
515
+ raise blockexception
516
+ end
517
+ benchblock.xml.add! do
518
+ parser.parse(responsehdr)
519
+ end
520
+ return parser.result
521
+ end
522
+ else
523
+ benchblock.service.add! do
524
+ begin
525
+ response = @connection.request(request)
526
+ rescue Exception => e
527
+ # Kill a connection if we run into a low level connection error
528
+ destroy_connection(request, "error: #{e.message}")
529
+ raise e
530
+ end
531
+ end
532
+ # check response for errors...
533
+ @last_response = response
534
+ if response.is_a?(Net::HTTPSuccess)
535
+ @error_handler = nil
536
+ benchblock.xml.add! { parser.parse(response) }
537
+ return parser.result
538
+ else
539
+ @error_handler = AWSErrorHandler.new(self, parser, :errors_list => self.class.amazon_problems) unless @error_handler
540
+ check_result = @error_handler.check(request)
541
+ if check_result
542
+ @error_handler = nil
543
+ return check_result
544
+ end
545
+ raise AwsError.new(@last_errors, @last_response.code, @last_request_id)
546
+ end
547
+ end
548
+ rescue
549
+ @error_handler = nil
550
+ raise
551
+ end
552
+
553
+ def request_cache_or_info(method, link, parser_class, benchblock, use_cache=true, &block) #:nodoc:
554
+ # We do not want to break the logic of parsing hence will use a dummy parser to process all the standard
555
+ # steps (errors checking etc). The dummy parser does nothig - just returns back the params it received.
556
+ # If the caching is enabled and hit then throw AwsNoChange.
557
+ # P.S. caching works for the whole images list only! (when the list param is blank)
558
+ # check cache
559
+ response, params = request_info(link, RightDummyParser.new)
560
+ cache_hits?(method.to_sym, response.body) if use_cache
561
+ parser = parser_class.new(:logger => @logger)
562
+ benchblock.xml.add!{ parser.parse(response, params) }
563
+ result = block ? block.call(parser) : parser.result
564
+ # update parsed data
565
+ update_cache(method.to_sym, :parsed => result) if use_cache
566
+ result
567
+ end
568
+
569
+ # Returns Amazons request ID for the latest request
570
+ def last_request_id
571
+ @last_response && @last_response.body.to_s[%r{<requestId>(.+?)</requestId>}i] && $1
572
+ end
573
+
574
+ # Incrementally lists something.
575
+ def incrementally_list_items(action, parser_class, params={}, &block) # :nodoc:
576
+ params = params.dup
577
+ params['MaxItems'] = params.delete(:max_items) if params[:max_items]
578
+ params['Marker'] = params.delete(:marker) if params[:marker]
579
+ last_response = nil
580
+ loop do
581
+ last_response = request_info( generate_request(action, params), parser_class.new(:logger => @logger))
582
+ params['Marker'] = last_response[:marker]
583
+ break unless block && block.call(last_response) && !last_response[:marker].right_blank?
584
+ end
585
+ last_response
586
+ end
587
+
588
+ # Format array of items into Amazons handy hash ('?' is a place holder):
589
+ # Options:
590
+ # :default => "something" : Set a value to "something" when it is nil
591
+ # :default => :skip_nils : Skip nil values
592
+ #
593
+ # amazonize_list('Item', ['a', 'b', 'c']) =>
594
+ # { 'Item.1' => 'a', 'Item.2' => 'b', 'Item.3' => 'c' }
595
+ #
596
+ # amazonize_list('Item.?.instance', ['a', 'c']) #=>
597
+ # { 'Item.1.instance' => 'a', 'Item.2.instance' => 'c' }
598
+ #
599
+ # amazonize_list(['Item.?.Name', 'Item.?.Value'], {'A' => 'a', 'B' => 'b'}) #=>
600
+ # { 'Item.1.Name' => 'A', 'Item.1.Value' => 'a',
601
+ # 'Item.2.Name' => 'B', 'Item.2.Value' => 'b' }
602
+ #
603
+ # amazonize_list(['Item.?.Name', 'Item.?.Value'], [['A','a'], ['B','b']]) #=>
604
+ # { 'Item.1.Name' => 'A', 'Item.1.Value' => 'a',
605
+ # 'Item.2.Name' => 'B', 'Item.2.Value' => 'b' }
606
+ #
607
+ # amazonize_list(['Filter.?.Key', 'Filter.?.Value.?'], {'A' => ['aa','ab'], 'B' => ['ba','bb']}) #=>
608
+ # amazonize_list(['Filter.?.Key', 'Filter.?.Value.?'], [['A',['aa','ab']], ['B',['ba','bb']]]) #=>
609
+ # {"Filter.1.Key"=>"A",
610
+ # "Filter.1.Value.1"=>"aa",
611
+ # "Filter.1.Value.2"=>"ab",
612
+ # "Filter.2.Key"=>"B",
613
+ # "Filter.2.Value.1"=>"ba",
614
+ # "Filter.2.Value.2"=>"bb"}
615
+ def amazonize_list(masks, list, options={}) #:nodoc:
616
+ groups = {}
617
+ Array(list).each_with_index do |list_item, i|
618
+ Array(masks).each_with_index do |mask, mask_idx|
619
+ key = mask[/\?/] ? mask.dup : mask.dup + '.?'
620
+ key.sub!('?', (i+1).to_s)
621
+ value = Array(list_item)[mask_idx]
622
+ if value.is_a?(Array)
623
+ groups.merge!(amazonize_list(key, value, options))
624
+ else
625
+ if value.nil?
626
+ next if options[:default] == :skip_nils
627
+ value = options[:default]
628
+ end
629
+ # Hack to avoid having unhandled '?' in keys : do replace them all with '1':
630
+ # bad: ec2.amazonize_list(['Filter.?.Key', 'Filter.?.Value.?'], { a: => :b }) => {"Filter.1.Key"=>:a, "Filter.1.Value.?"=>1}
631
+ # good: ec2.amazonize_list(['Filter.?.Key', 'Filter.?.Value.?'], { a: => :b }) => {"Filter.1.Key"=>:a, "Filter.1.Value.1"=>1}
632
+ key.gsub!('?', '1')
633
+ groups[key] = value
634
+ end
635
+ end
636
+ end
637
+ groups
638
+ end
639
+
640
+ BLOCK_DEVICE_KEY_MAPPING = { # :nodoc:
641
+ :device_name => 'DeviceName',
642
+ :virtual_name => 'VirtualName',
643
+ :no_device => 'NoDevice',
644
+ :ebs_snapshot_id => 'Ebs.SnapshotId',
645
+ :ebs_volume_size => 'Ebs.VolumeSize',
646
+ :ebs_delete_on_termination => 'Ebs.DeleteOnTermination' }
647
+
648
+ def amazonize_block_device_mappings(block_device_mappings, key = 'BlockDeviceMapping') # :nodoc:
649
+ result = {}
650
+ unless block_device_mappings.right_blank?
651
+ block_device_mappings = [block_device_mappings] unless block_device_mappings.is_a?(Array)
652
+ block_device_mappings.each_with_index do |b, idx|
653
+ BLOCK_DEVICE_KEY_MAPPING.each do |local_name, remote_name|
654
+ value = b[local_name]
655
+ case local_name
656
+ when :no_device then value = value ? '' : nil # allow to pass :no_device as boolean
657
+ end
658
+ result["#{key}.#{idx+1}.#{remote_name}"] = value unless value.nil?
659
+ end
660
+ end
661
+ end
662
+ result
663
+ end
664
+
665
+ # Execute a block of code with custom set of settings for right_http_connection.
666
+ # Accepts next options (see Rightscale::HttpConnection for explanation):
667
+ # :raise_on_timeout
668
+ # :http_connection_retry_count
669
+ # :http_connection_open_timeout
670
+ # :http_connection_read_timeout
671
+ # :http_connection_retry_delay
672
+ # :user_agent
673
+ # :exception
674
+ #
675
+ # Example #1:
676
+ #
677
+ # # Try to create a snapshot but stop with exception if timeout is received
678
+ # # to avoid having a duplicate API calls that create duplicate snapshots.
679
+ # ec2 = Rightscale::Ec2::new(aws_access_key_id, aws_secret_access_key)
680
+ # ec2.with_connection_options(:raise_on_timeout => true) do
681
+ # ec2.create_snapshot('vol-898a6fe0', 'KD: WooHoo!!')
682
+ # end
683
+ #
684
+ # Example #2:
685
+ #
686
+ # # Opposite case when the setting is global:
687
+ # @ec2 = Rightscale::Ec2::new(aws_access_key_id, aws_secret_access_key,
688
+ # :connection_options => { :raise_on_timeout => true })
689
+ # # Create an SSHKey but do tries on timeout
690
+ # ec2.with_connection_options(:raise_on_timeout => false) do
691
+ # new_key = ec2.create_key_pair('my_test_key')
692
+ # end
693
+ #
694
+ # Example #3:
695
+ #
696
+ # # Global settings (HttpConnection level):
697
+ # Rightscale::HttpConnection::params[:http_connection_open_timeout] = 5
698
+ # Rightscale::HttpConnection::params[:http_connection_read_timeout] = 250
699
+ # Rightscale::HttpConnection::params[:http_connection_retry_count] = 2
700
+ #
701
+ # # Local setings (RightAws level)
702
+ # ec2 = Rightscale::Ec2::new(AWS_ID, AWS_KEY,
703
+ # :region => 'us-east-1',
704
+ # :connection_options => {
705
+ # :http_connection_read_timeout => 2,
706
+ # :http_connection_retry_count => 5,
707
+ # :user_agent => 'Mozilla 4.0'
708
+ # })
709
+ #
710
+ # # Custom settings (API call level)
711
+ # ec2.with_connection_options(:raise_on_timeout => true,
712
+ # :http_connection_read_timeout => 10,
713
+ # :user_agent => '') do
714
+ # pp ec2.describe_images
715
+ # end
716
+ #
717
+ def with_connection_options(options, &block)
718
+ @with_connection_options = options
719
+ block.call self
720
+ ensure
721
+ @with_connection_options = {}
722
+ end
723
+ end
724
+
725
+
726
+ # Exception class to signal any Amazon errors. All errors occuring during calls to Amazon's
727
+ # web services raise this type of error.
728
+ # Attribute inherited by RuntimeError:
729
+ # message - the text of the error, generally as returned by AWS in its XML response.
730
+ class AwsError < RuntimeError
731
+
732
+ # either an array of errors where each item is itself an array of [code, message]),
733
+ # or an error string if the error was raised manually, as in <tt>AwsError.new('err_text')</tt>
734
+ attr_reader :errors
735
+
736
+ # Request id (if exists)
737
+ attr_reader :request_id
738
+
739
+ # Response HTTP error code
740
+ attr_reader :http_code
741
+
742
+ def initialize(errors=nil, http_code=nil, request_id=nil)
743
+ @errors = errors
744
+ @request_id = request_id
745
+ @http_code = http_code
746
+ super(@errors.is_a?(Array) ? @errors.map{|code, msg| "#{code}: #{msg}"}.join("; ") : @errors.to_s)
747
+ end
748
+
749
+ # Does any of the error messages include the regexp +pattern+?
750
+ # Used to determine whether to retry request.
751
+ def include?(pattern)
752
+ if @errors.is_a?(Array)
753
+ @errors.each{ |code, msg| return true if code =~ pattern }
754
+ else
755
+ return true if @errors_str =~ pattern
756
+ end
757
+ false
758
+ end
759
+
760
+ # Generic handler for AwsErrors. +aws+ is the RightAws::S3, RightAws::EC2, or RightAws::SQS
761
+ # object that caused the exception (it must provide last_request and last_response). Supported
762
+ # boolean options are:
763
+ # * <tt>:log</tt> print a message into the log using aws.logger to access the Logger
764
+ # * <tt>:puts</tt> do a "puts" of the error
765
+ # * <tt>:raise</tt> re-raise the error after logging
766
+ def self.on_aws_exception(aws, options={:raise=>true, :log=>true})
767
+ # Only log & notify if not user error
768
+ if !options[:raise] || system_error?($!)
769
+ error_text = "#{$!.inspect}\n#{$@}.join('\n')}"
770
+ puts error_text if options[:puts]
771
+ # Log the error
772
+ if options[:log]
773
+ request = aws.last_request ? aws.last_request.path : '-none-'
774
+ response = aws.last_response ? "#{aws.last_response.code} -- #{aws.last_response.message} -- #{aws.last_response.body}" : '-none-'
775
+ aws.logger.error error_text
776
+ aws.logger.error "Request was: #{request}"
777
+ aws.logger.error "Response was: #{response}"
778
+ end
779
+ end
780
+ raise if options[:raise] # re-raise an exception
781
+ return nil
782
+ end
783
+
784
+ # True if e is an AWS system error, i.e. something that is for sure not the caller's fault.
785
+ # Used to force logging.
786
+ def self.system_error?(e)
787
+ !e.is_a?(self) || e.message =~ /InternalError|InsufficientInstanceCapacity|Unavailable/
788
+ end
789
+
790
+ end
791
+
792
+
793
+ class AWSErrorHandler
794
+ # 0-100 (%)
795
+ DEFAULT_CLOSE_ON_4XX_PROBABILITY = 10
796
+
797
+ @@reiteration_start_delay = 0.2
798
+ def self.reiteration_start_delay
799
+ @@reiteration_start_delay
800
+ end
801
+ def self.reiteration_start_delay=(reiteration_start_delay)
802
+ @@reiteration_start_delay = reiteration_start_delay
803
+ end
804
+
805
+ @@reiteration_time = 5
806
+ def self.reiteration_time
807
+ @@reiteration_time
808
+ end
809
+ def self.reiteration_time=(reiteration_time)
810
+ @@reiteration_time = reiteration_time
811
+ end
812
+
813
+ @@close_on_error = true
814
+ def self.close_on_error
815
+ @@close_on_error
816
+ end
817
+ def self.close_on_error=(close_on_error)
818
+ @@close_on_error = close_on_error
819
+ end
820
+
821
+ @@close_on_4xx_probability = DEFAULT_CLOSE_ON_4XX_PROBABILITY
822
+ def self.close_on_4xx_probability
823
+ @@close_on_4xx_probability
824
+ end
825
+ def self.close_on_4xx_probability=(close_on_4xx_probability)
826
+ @@close_on_4xx_probability = close_on_4xx_probability
827
+ end
828
+
829
+ # params:
830
+ # :reiteration_time
831
+ # :errors_list
832
+ # :close_on_error = true | false
833
+ # :close_on_4xx_probability = 1-100
834
+ def initialize(aws, parser, params={}) #:nodoc:
835
+ @aws = aws # Link to RightEc2 | RightSqs | RightS3 instance
836
+ @parser = parser # parser to parse Amazon response
837
+ @started_at = Time.now
838
+ @stop_at = @started_at + (params[:reiteration_time] || @@reiteration_time)
839
+ @errors_list = params[:errors_list] || []
840
+ @reiteration_delay = @@reiteration_start_delay
841
+ @retries = 0
842
+ # close current HTTP(S) connection on 5xx, errors from list and 4xx errors
843
+ @close_on_error = params[:close_on_error].nil? ? @@close_on_error : params[:close_on_error]
844
+ @close_on_4xx_probability = params[:close_on_4xx_probability] || @@close_on_4xx_probability
845
+ end
846
+
847
+ # Returns false if
848
+ def check(request) #:nodoc:
849
+ result = false
850
+ error_found = false
851
+ redirect_detected= false
852
+ error_match = nil
853
+ last_errors_text = ''
854
+ response = @aws.last_response
855
+ # log error
856
+ request_text_data = "#{request[:protocol]}://#{request[:server]}:#{request[:port]}#{request[:request].path}"
857
+ # is this a redirect?
858
+ # yes!
859
+ if response.is_a?(Net::HTTPRedirection)
860
+ redirect_detected = true
861
+ else
862
+ # no, it's an error ...
863
+ @aws.logger.warn("##### #{@aws.class.name} returned an error: #{response.code} #{response.message}\n#{response.body} #####")
864
+ @aws.logger.warn("##### #{@aws.class.name} request: #{request_text_data} ####")
865
+ end
866
+
867
+ # Extract error/redirection message from the response body
868
+ # Amazon claims that a redirection must have a body but somethimes it is nil....
869
+ if response.body && response.body[/^(<\?xml|<ErrorResponse)/]
870
+ error_parser = RightErrorResponseParser.new
871
+ @aws.class.bench_xml.add! do
872
+ error_parser.parse(response.body)
873
+ end
874
+ @aws.last_errors = error_parser.errors
875
+ @aws.last_request_id = error_parser.requestID
876
+ last_errors_text = @aws.last_errors.flatten.join("\n")
877
+ else
878
+ @aws.last_errors = [[response.code, "#{response.message} (#{request_text_data})"]]
879
+ @aws.last_request_id = '-undefined-'
880
+ last_errors_text = response.message
881
+ end
882
+
883
+ # Ok, it is a redirect, find the new destination location
884
+ if redirect_detected
885
+ location = response['location']
886
+ # As for 301 ( Moved Permanently) Amazon does not return a 'Location' header but
887
+ # it is possible to extract a new endpoint from the response body
888
+ if location.right_blank? && response.code=='301' && response.body
889
+ new_endpoint = response.body[/<Endpoint>(.*?)<\/Endpoint>/] && $1
890
+ location = "#{request[:protocol]}://#{new_endpoint}:#{request[:port]}#{request[:request].path}"
891
+ end
892
+ # ... log information and ...
893
+ @aws.logger.info("##### #{@aws.class.name} redirect requested: #{response.code} #{response.message} #####")
894
+ @aws.logger.info(" Old location: #{request_text_data}")
895
+ @aws.logger.info(" New location: #{location}")
896
+ @aws.logger.info(" Request Verb: #{request[:request].class.name}")
897
+ # ... fix the connection data
898
+ request[:server] = URI.parse(location).host
899
+ request[:protocol] = URI.parse(location).scheme
900
+ request[:port] = URI.parse(location).port
901
+ else
902
+ # Not a redirect but an error: try to find the error in our list
903
+ @errors_list.each do |error_to_find|
904
+ if last_errors_text[/#{error_to_find}/i]
905
+ error_found = true
906
+ error_match = error_to_find
907
+ @aws.logger.warn("##### Retry is needed, error pattern match: #{error_to_find} #####")
908
+ break
909
+ end
910
+ end
911
+ end
912
+
913
+ # check the time has gone from the first error come
914
+ if redirect_detected || error_found
915
+ # Close the connection to the server and recreate a new one.
916
+ # It may have a chance that one server is a semi-down and reconnection
917
+ # will help us to connect to the other server
918
+ if !redirect_detected && @close_on_error
919
+ @aws.destroy_connection(request, "#{self.class.name}: error match to pattern '#{error_match}'")
920
+ end
921
+
922
+ if (Time.now < @stop_at)
923
+ @retries += 1
924
+ unless redirect_detected
925
+ @aws.logger.warn("##### Retry ##{@retries} is being performed. Sleeping for #{@reiteration_delay} sec. Whole time: #{Time.now-@started_at} sec ####")
926
+ sleep @reiteration_delay
927
+ @reiteration_delay *= 2
928
+
929
+ # Always make sure that the fp is set to point to the beginning(?)
930
+ # of the File/IO. TODO: it assumes that offset is 0, which is bad.
931
+ if(request[:request].body_stream && request[:request].body_stream.respond_to?(:pos))
932
+ begin
933
+ request[:request].body_stream.pos = 0
934
+ rescue Exception => e
935
+ @logger.warn("Retry may fail due to unable to reset the file pointer" +
936
+ " -- #{self.class.name} : #{e.inspect}")
937
+ end
938
+ end
939
+ else
940
+ @aws.logger.info("##### Retry ##{@retries} is being performed due to a redirect. ####")
941
+ end
942
+ result = @aws.request_info(request, @parser)
943
+ else
944
+ @aws.logger.warn("##### Ooops, time is over... ####")
945
+ end
946
+ # aha, this is unhandled error:
947
+ elsif @close_on_error
948
+ # On 5xx(Server errors), 403(RequestTimeTooSkewed) and 408(Request Timeout) a conection has to be closed
949
+ if @aws.last_response.code.to_s[/^(5\d\d|403|408)$/]
950
+ @aws.destroy_connection(request, "#{self.class.name}: code: #{@aws.last_response.code}: '#{@aws.last_response.message}'")
951
+ # Is this a 4xx error ?
952
+ elsif @aws.last_response.code.to_s[/^4\d\d$/] && @close_on_4xx_probability > rand(100)
953
+ @aws.destroy_connection(request, "#{self.class.name}: code: #{@aws.last_response.code}: '#{@aws.last_response.message}', " +
954
+ "probability: #{@close_on_4xx_probability}%")
955
+ end
956
+ end
957
+ result
958
+ end
959
+
960
+ end
961
+
962
+
963
+ #-----------------------------------------------------------------
964
+
965
+ class RightSaxParserCallbackTemplate #:nodoc:
966
+ def initialize(right_aws_parser)
967
+ @right_aws_parser = right_aws_parser
968
+ end
969
+ def on_characters(chars)
970
+ @right_aws_parser.text(chars)
971
+ end
972
+ def on_start_document; end
973
+ def on_comment(msg); end
974
+ def on_processing_instruction(target, data); end
975
+ def on_cdata_block(cdata); end
976
+ def on_end_document; end
977
+ end
978
+
979
+ class RightSaxParserCallback < RightSaxParserCallbackTemplate
980
+ def self.include_callback
981
+ include XML::SaxParser::Callbacks
982
+ end
983
+ def on_start_element(name, attr_hash)
984
+ @right_aws_parser.tag_start(name, attr_hash)
985
+ end
986
+ def on_end_element(name)
987
+ @right_aws_parser.tag_end(name)
988
+ end
989
+ end
990
+
991
+ class RightSaxParserCallbackNs < RightSaxParserCallbackTemplate
992
+ def on_start_element_ns(name, attr_hash, prefix, uri, namespaces)
993
+ @right_aws_parser.tag_start(name, attr_hash)
994
+ end
995
+ def on_end_element_ns(name, prefix, uri)
996
+ @right_aws_parser.tag_end(name)
997
+ end
998
+ end
999
+
1000
+ class RightAWSParser #:nodoc:
1001
+ # default parsing library
1002
+ DEFAULT_XML_LIBRARY = 'rexml'
1003
+ # a list of supported parsers
1004
+ @@supported_xml_libs = [DEFAULT_XML_LIBRARY, 'libxml']
1005
+
1006
+ @@xml_lib = DEFAULT_XML_LIBRARY # xml library name: 'rexml' | 'libxml'
1007
+ def self.xml_lib
1008
+ @@xml_lib
1009
+ end
1010
+ def self.xml_lib=(new_lib_name)
1011
+ @@xml_lib = new_lib_name
1012
+ end
1013
+
1014
+ attr_accessor :result
1015
+ attr_reader :xmlpath
1016
+ attr_accessor :xml_lib
1017
+ attr_reader :full_tag_name
1018
+ attr_reader :tag
1019
+
1020
+ def initialize(params={})
1021
+ @xmlpath = ''
1022
+ @full_tag_name = ''
1023
+ @result = false
1024
+ @text = ''
1025
+ @tag = ''
1026
+ @xml_lib = params[:xml_lib] || @@xml_lib
1027
+ @logger = params[:logger]
1028
+ reset
1029
+ end
1030
+ def tag_start(name, attributes)
1031
+ @text = ''
1032
+ @tag = name
1033
+ @full_tag_name += @full_tag_name.empty? ? name : "/#{name}"
1034
+ tagstart(name, attributes)
1035
+ @xmlpath = @full_tag_name
1036
+ end
1037
+ def tag_end(name)
1038
+ @xmlpath = @full_tag_name[/^(.*?)\/?#{name}$/] && $1
1039
+ tagend(name)
1040
+ @full_tag_name = @xmlpath
1041
+ end
1042
+ def text(text)
1043
+ @text += text
1044
+ tagtext(text)
1045
+ end
1046
+ # Parser method.
1047
+ # Params:
1048
+ # xml_text - xml message text(String) or Net:HTTPxxx instance (response)
1049
+ # params[:xml_lib] - library name: 'rexml' | 'libxml'
1050
+ def parse(xml_text, params={})
1051
+ # Get response body
1052
+ xml_text = xml_text.body unless xml_text.is_a?(String)
1053
+ @xml_lib = params[:xml_lib] || @xml_lib
1054
+ # check that we had no problems with this library otherwise use default
1055
+ @xml_lib = DEFAULT_XML_LIBRARY unless @@supported_xml_libs.include?(@xml_lib)
1056
+ # load xml library
1057
+ if @xml_lib=='libxml' && !defined?(XML::SaxParser)
1058
+ begin
1059
+ require 'xml/libxml'
1060
+ # Setup SaxParserCallback
1061
+ if XML::Parser::VERSION >= '0.5.1' &&
1062
+ XML::Parser::VERSION < '0.9.7'
1063
+ RightSaxParserCallback.include_callback
1064
+ end
1065
+ rescue LoadError => e
1066
+ @@supported_xml_libs.delete(@xml_lib)
1067
+ @xml_lib = DEFAULT_XML_LIBRARY
1068
+ if @logger
1069
+ @logger.error e.inspect
1070
+ @logger.error e.backtrace
1071
+ @logger.info "Can not load 'libxml' library. '#{DEFAULT_XML_LIBRARY}' is used for parsing."
1072
+ end
1073
+ end
1074
+ end
1075
+ # Parse the xml text
1076
+ case @xml_lib
1077
+ when 'libxml'
1078
+ if XML::Parser::VERSION >= '0.9.9'
1079
+ # avoid warning on every usage
1080
+ xml = XML::SaxParser.string(xml_text)
1081
+ else
1082
+ xml = XML::SaxParser.new
1083
+ xml.string = xml_text
1084
+ end
1085
+ # check libxml-ruby version
1086
+ if XML::Parser::VERSION >= '0.9.7'
1087
+ xml.callbacks = RightSaxParserCallbackNs.new(self)
1088
+ elsif XML::Parser::VERSION >= '0.5.1'
1089
+ xml.callbacks = RightSaxParserCallback.new(self)
1090
+ else
1091
+ xml.on_start_element{|name, attr_hash| self.tag_start(name, attr_hash)}
1092
+ xml.on_characters{ |text| self.text(text)}
1093
+ xml.on_end_element{ |name| self.tag_end(name)}
1094
+ end
1095
+ xml.parse
1096
+ else
1097
+ REXML::Document.parse_stream(xml_text, self)
1098
+ end
1099
+ end
1100
+ # Parser must have a lots of methods
1101
+ # (see /usr/lib/ruby/1.8/rexml/parsers/streamparser.rb)
1102
+ # We dont need most of them in RightAWSParser and method_missing helps us
1103
+ # to skip their definition
1104
+ def method_missing(method, *params)
1105
+ # if the method is one of known - just skip it ...
1106
+ return if [:comment, :attlistdecl, :notationdecl, :elementdecl,
1107
+ :entitydecl, :cdata, :xmldecl, :attlistdecl, :instruction,
1108
+ :doctype].include?(method)
1109
+ # ... else - call super to raise an exception
1110
+ super(method, params)
1111
+ end
1112
+ # the functions to be overriden by children (if nessesery)
1113
+ def reset ; end
1114
+ def tagstart(name, attributes); end
1115
+ def tagend(name) ; end
1116
+ def tagtext(text) ; end
1117
+ end
1118
+
1119
+ #-----------------------------------------------------------------
1120
+ # PARSERS: Errors
1121
+ #-----------------------------------------------------------------
1122
+
1123
+ #<Error>
1124
+ # <Code>TemporaryRedirect</Code>
1125
+ # <Message>Please re-send this request to the specified temporary endpoint. Continue to use the original request endpoint for future requests.</Message>
1126
+ # <RequestId>FD8D5026D1C5ABA3</RequestId>
1127
+ # <Endpoint>bucket-for-k.s3-external-3.amazonaws.com</Endpoint>
1128
+ # <HostId>ItJy8xPFPli1fq/JR3DzQd3iDvFCRqi1LTRmunEdM1Uf6ZtW2r2kfGPWhRE1vtaU</HostId>
1129
+ # <Bucket>bucket-for-k</Bucket>
1130
+ #</Error>
1131
+
1132
+ class RightErrorResponseParser < RightAWSParser #:nodoc:
1133
+ attr_accessor :errors # array of hashes: error/message
1134
+ attr_accessor :requestID
1135
+ # attr_accessor :endpoint, :host_id, :bucket
1136
+ def tagend(name)
1137
+ case name
1138
+ when 'RequestID' ; @requestID = @text
1139
+ when 'Code' ; @code = @text
1140
+ when 'Message' ; @message = @text
1141
+ # when 'Endpoint' ; @endpoint = @text
1142
+ # when 'HostId' ; @host_id = @text
1143
+ # when 'Bucket' ; @bucket = @text
1144
+ when 'Error' ; @errors << [ @code, @message ]
1145
+ end
1146
+ end
1147
+ def reset
1148
+ @errors = []
1149
+ end
1150
+ end
1151
+
1152
+ # Dummy parser - does nothing
1153
+ # Returns the original params back
1154
+ class RightDummyParser # :nodoc:
1155
+ attr_accessor :result
1156
+ def parse(response, params={})
1157
+ @result = [response, params]
1158
+ end
1159
+ end
1160
+
1161
+ class RightHttp2xxParser < RightAWSParser # :nodoc:
1162
+ def parse(response)
1163
+ @result = response.is_a?(Net::HTTPSuccess)
1164
+ end
1165
+ end
1166
+
1167
+ class RightBoolResponseParser < RightAWSParser #:nodoc:
1168
+ def tagend(name)
1169
+ @result = (@text=='true') if name == 'return'
1170
+ end
1171
+ end
1172
+
1173
+ end
1174
+