dustMason-right_aws 2.1.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (70) hide show
  1. data/History.txt +305 -0
  2. data/Manifest.txt +60 -0
  3. data/README.txt +165 -0
  4. data/Rakefile +112 -0
  5. data/lib/acf/right_acf_interface.rb +549 -0
  6. data/lib/acf/right_acf_invalidations.rb +144 -0
  7. data/lib/acf/right_acf_origin_access_identities.rb +230 -0
  8. data/lib/acf/right_acf_streaming_interface.rb +229 -0
  9. data/lib/acw/right_acw_interface.rb +248 -0
  10. data/lib/as/right_as_interface.rb +698 -0
  11. data/lib/awsbase/benchmark_fix.rb +39 -0
  12. data/lib/awsbase/right_awsbase.rb +1174 -0
  13. data/lib/awsbase/support.rb +35 -0
  14. data/lib/awsbase/version.rb +9 -0
  15. data/lib/ec2/right_ec2.rb +458 -0
  16. data/lib/ec2/right_ec2_ebs.rb +465 -0
  17. data/lib/ec2/right_ec2_images.rb +413 -0
  18. data/lib/ec2/right_ec2_instances.rb +785 -0
  19. data/lib/ec2/right_ec2_monitoring.rb +70 -0
  20. data/lib/ec2/right_ec2_placement_groups.rb +108 -0
  21. data/lib/ec2/right_ec2_reserved_instances.rb +174 -0
  22. data/lib/ec2/right_ec2_security_groups.rb +396 -0
  23. data/lib/ec2/right_ec2_spot_instances.rb +425 -0
  24. data/lib/ec2/right_ec2_tags.rb +139 -0
  25. data/lib/ec2/right_ec2_vpc.rb +583 -0
  26. data/lib/ec2/right_ec2_windows_mobility.rb +84 -0
  27. data/lib/elb/right_elb_interface.rb +571 -0
  28. data/lib/iam/right_iam_access_keys.rb +71 -0
  29. data/lib/iam/right_iam_groups.rb +195 -0
  30. data/lib/iam/right_iam_interface.rb +341 -0
  31. data/lib/iam/right_iam_mfa_devices.rb +67 -0
  32. data/lib/iam/right_iam_users.rb +251 -0
  33. data/lib/rds/right_rds_interface.rb +1309 -0
  34. data/lib/right_aws.rb +88 -0
  35. data/lib/route_53/right_route_53_interface.rb +630 -0
  36. data/lib/s3/right_s3.rb +1123 -0
  37. data/lib/s3/right_s3_interface.rb +1198 -0
  38. data/lib/sdb/active_sdb.rb +1107 -0
  39. data/lib/sdb/right_sdb_interface.rb +753 -0
  40. data/lib/sns/right_sns.rb +205 -0
  41. data/lib/sns/right_sns_interface.rb +343 -0
  42. data/lib/sqs/right_sqs.rb +387 -0
  43. data/lib/sqs/right_sqs_gen2.rb +342 -0
  44. data/lib/sqs/right_sqs_gen2_interface.rb +523 -0
  45. data/lib/sqs/right_sqs_interface.rb +593 -0
  46. data/right_aws.gemspec +91 -0
  47. data/test/acf/test_helper.rb +2 -0
  48. data/test/acf/test_right_acf.rb +138 -0
  49. data/test/awsbase/test_helper.rb +2 -0
  50. data/test/awsbase/test_right_awsbase.rb +12 -0
  51. data/test/ec2/test_helper.rb +2 -0
  52. data/test/ec2/test_right_ec2.rb +108 -0
  53. data/test/http_connection.rb +87 -0
  54. data/test/rds/test_helper.rb +2 -0
  55. data/test/rds/test_right_rds.rb +120 -0
  56. data/test/s3/test_helper.rb +2 -0
  57. data/test/s3/test_right_s3.rb +421 -0
  58. data/test/s3/test_right_s3_stubbed.rb +97 -0
  59. data/test/sdb/test_active_sdb.rb +357 -0
  60. data/test/sdb/test_batch_put_attributes.rb +54 -0
  61. data/test/sdb/test_helper.rb +3 -0
  62. data/test/sdb/test_right_sdb.rb +253 -0
  63. data/test/sns/test_helper.rb +2 -0
  64. data/test/sns/test_right_sns.rb +73 -0
  65. data/test/sqs/test_helper.rb +2 -0
  66. data/test/sqs/test_right_sqs.rb +285 -0
  67. data/test/sqs/test_right_sqs_gen2.rb +264 -0
  68. data/test/test_credentials.rb +37 -0
  69. data/test/ts_right_aws.rb +14 -0
  70. metadata +244 -0
@@ -0,0 +1,39 @@
1
+ #
2
+ # Copyright (c) 2007-2008 RightScale Inc
3
+ #
4
+ # Permission is hereby granted, free of charge, to any person obtaining
5
+ # a copy of this software and associated documentation files (the
6
+ # "Software"), to deal in the Software without restriction, including
7
+ # without limitation the rights to use, copy, modify, merge, publish,
8
+ # distribute, sublicense, and/or sell copies of the Software, and to
9
+ # permit persons to whom the Software is furnished to do so, subject to
10
+ # the following conditions:
11
+ #
12
+ # The above copyright notice and this permission notice shall be
13
+ # included in all copies or substantial portions of the Software.
14
+ #
15
+ # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
16
+ # EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
17
+ # MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
18
+ # NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
19
+ # LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
20
+ # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
21
+ # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
22
+ #
23
+ #
24
+
25
+
26
+ # A hack because there's a bug in add! in Benchmark::Tms
27
+ module Benchmark #:nodoc:
28
+ class Tms #:nodoc:
29
+ def add!(&blk)
30
+ t = Benchmark::measure(&blk)
31
+ @utime = utime + t.utime
32
+ @stime = stime + t.stime
33
+ @cutime = cutime + t.cutime
34
+ @cstime = cstime + t.cstime
35
+ @real = real + t.real
36
+ self
37
+ end
38
+ end
39
+ end
@@ -0,0 +1,1174 @@
1
+ #
2
+ # Copyright (c) 2007-2008 RightScale Inc
3
+ #
4
+ # Permission is hereby granted, free of charge, to any person obtaining
5
+ # a copy of this software and associated documentation files (the
6
+ # "Software"), to deal in the Software without restriction, including
7
+ # without limitation the rights to use, copy, modify, merge, publish,
8
+ # distribute, sublicense, and/or sell copies of the Software, and to
9
+ # permit persons to whom the Software is furnished to do so, subject to
10
+ # the following conditions:
11
+ #
12
+ # The above copyright notice and this permission notice shall be
13
+ # included in all copies or substantial portions of the Software.
14
+ #
15
+ # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
16
+ # EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
17
+ # MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
18
+ # NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
19
+ # LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
20
+ # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
21
+ # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
22
+ #
23
+
24
+ # Test
25
+ module RightAws
26
+ require 'digest/md5'
27
+ require 'pp'
28
+
29
+ class AwsUtils #:nodoc:
30
+ @@digest1 = OpenSSL::Digest::Digest.new("sha1")
31
+ @@digest256 = nil
32
+ if OpenSSL::OPENSSL_VERSION_NUMBER > 0x00908000
33
+ @@digest256 = OpenSSL::Digest::Digest.new("sha256") rescue nil # Some installation may not support sha256
34
+ end
35
+
36
+ def self.utc_iso8601(time)
37
+ if time.is_a?(Fixnum) then time = Time::at(time)
38
+ elsif time.is_a?(String) then time = Time::parse(time)
39
+ end
40
+ time.utc.strftime("%Y-%m-%dT%H:%M:%S.000Z")
41
+ end
42
+
43
+ def self.sign(aws_secret_access_key, auth_string)
44
+ Base64.encode64(OpenSSL::HMAC.digest(@@digest1, aws_secret_access_key, auth_string)).strip
45
+ end
46
+
47
+ # Escape a string accordingly Amazon rulles
48
+ # http://docs.amazonwebservices.com/AmazonSimpleDB/2007-11-07/DeveloperGuide/index.html?REST_RESTAuth.html
49
+ def self.amz_escape(param)
50
+ param.to_s.gsub(/([^a-zA-Z0-9._~-]+)/n) do
51
+ '%' + $1.unpack('H2' * $1.size).join('%').upcase
52
+ end
53
+ end
54
+
55
+ def self.xml_escape(text) # :nodoc:
56
+ REXML::Text::normalize(text)
57
+ end
58
+
59
+ def self.xml_unescape(text) # :nodoc:
60
+ REXML::Text::unnormalize(text)
61
+ end
62
+
63
+ # Set a timestamp and a signature version
64
+ def self.fix_service_params(service_hash, signature)
65
+ service_hash["Timestamp"] ||= utc_iso8601(Time.now) unless service_hash["Expires"]
66
+ service_hash["SignatureVersion"] = signature
67
+ service_hash
68
+ end
69
+
70
+ # Signature Version 0
71
+ # A deprecated guy (should work till septemper 2009)
72
+ def self.sign_request_v0(aws_secret_access_key, service_hash)
73
+ fix_service_params(service_hash, '0')
74
+ string_to_sign = "#{service_hash['Action']}#{service_hash['Timestamp'] || service_hash['Expires']}"
75
+ service_hash['Signature'] = AwsUtils::sign(aws_secret_access_key, string_to_sign)
76
+ service_hash.to_a.collect{|key,val| "#{amz_escape(key)}=#{amz_escape(val.to_s)}" }.join("&")
77
+ end
78
+
79
+ # Signature Version 1
80
+ # Another deprecated guy (should work till septemper 2009)
81
+ def self.sign_request_v1(aws_secret_access_key, service_hash)
82
+ fix_service_params(service_hash, '1')
83
+ string_to_sign = service_hash.sort{|a,b| (a[0].to_s.downcase)<=>(b[0].to_s.downcase)}.to_s
84
+ service_hash['Signature'] = AwsUtils::sign(aws_secret_access_key, string_to_sign)
85
+ service_hash.to_a.collect{|key,val| "#{amz_escape(key)}=#{amz_escape(val.to_s)}" }.join("&")
86
+ end
87
+
88
+ # Signature Version 2
89
+ # EC2, SQS and SDB requests must be signed by this guy.
90
+ # See: http://docs.amazonwebservices.com/AmazonSimpleDB/2007-11-07/DeveloperGuide/index.html?REST_RESTAuth.html
91
+ # http://developer.amazonwebservices.com/connect/entry.jspa?externalID=1928
92
+ def self.sign_request_v2(aws_secret_access_key, service_hash, http_verb, host, uri)
93
+ fix_service_params(service_hash, '2')
94
+ # select a signing method (make an old openssl working with sha1)
95
+ # make 'HmacSHA256' to be a default one
96
+ service_hash['SignatureMethod'] = 'HmacSHA256' unless ['HmacSHA256', 'HmacSHA1'].include?(service_hash['SignatureMethod'])
97
+ service_hash['SignatureMethod'] = 'HmacSHA1' unless @@digest256
98
+ # select a digest
99
+ digest = (service_hash['SignatureMethod'] == 'HmacSHA256' ? @@digest256 : @@digest1)
100
+ # form string to sign
101
+ canonical_string = service_hash.keys.sort.map do |key|
102
+ "#{amz_escape(key)}=#{amz_escape(service_hash[key])}"
103
+ end.join('&')
104
+ string_to_sign = "#{http_verb.to_s.upcase}\n#{host.downcase}\n#{uri}\n#{canonical_string}"
105
+ # sign the string
106
+ signature = amz_escape(Base64.encode64(OpenSSL::HMAC.digest(digest, aws_secret_access_key, string_to_sign)).strip)
107
+ "#{canonical_string}&Signature=#{signature}"
108
+ end
109
+
110
+ # From Amazon's SQS Dev Guide, a brief description of how to escape:
111
+ # "URL encode the computed signature and other query parameters as specified in
112
+ # RFC1738, section 2.2. In addition, because the + character is interpreted as a blank space
113
+ # by Sun Java classes that perform URL decoding, make sure to encode the + character
114
+ # although it is not required by RFC1738."
115
+ # Avoid using CGI::escape to escape URIs.
116
+ # CGI::escape will escape characters in the protocol, host, and port
117
+ # sections of the URI. Only target chars in the query
118
+ # string should be escaped.
119
+ def self.URLencode(raw)
120
+ e = URI.escape(raw)
121
+ e.gsub(/\+/, "%2b")
122
+ end
123
+
124
+ def self.allow_only(allowed_keys, params)
125
+ bogus_args = []
126
+ params.keys.each {|p| bogus_args.push(p) unless allowed_keys.include?(p) }
127
+ raise AwsError.new("The following arguments were given but are not legal for the function call #{caller_method}: #{bogus_args.inspect}") if bogus_args.length > 0
128
+ end
129
+
130
+ def self.mandatory_arguments(required_args, params)
131
+ rargs = required_args.dup
132
+ params.keys.each {|p| rargs.delete(p)}
133
+ raise AwsError.new("The following mandatory arguments were not provided to #{caller_method}: #{rargs.inspect}") if rargs.length > 0
134
+ end
135
+
136
+ def self.caller_method
137
+ caller[1]=~/`(.*?)'/
138
+ $1
139
+ end
140
+
141
+ def self.split_items_and_params(array)
142
+ items = Array(array).flatten.compact
143
+ params = items.last.kind_of?(Hash) ? items.pop : {}
144
+ [items, params]
145
+ end
146
+
147
+ # Generates a token in format of:
148
+ # 1. "1dd8d4e4-db6b-11df-b31d-0025b37efad0 (if UUID gem is loaded)
149
+ # 2. "1287483761-855215-zSv2z-bWGj2-31M5t-ags9m" (if UUID gem is not loaded)
150
+ TOKEN_GENERATOR_CHARSET = ('a'..'z').to_a + ('A'..'Z').to_a + ('0'..'9').to_a
151
+ def self.generate_unique_token
152
+ time = Time.now
153
+ token = "%d-%06d" % [time.to_i, time.usec]
154
+ 4.times do
155
+ token << "-"
156
+ 5.times { token << TOKEN_GENERATOR_CHARSET[rand(TOKEN_GENERATOR_CHARSET.size)] }
157
+ end
158
+ token
159
+ end
160
+ end
161
+
162
+ class AwsBenchmarkingBlock #:nodoc:
163
+ attr_accessor :xml, :service
164
+ def initialize
165
+ # Benchmark::Tms instance for service (Ec2, S3, or SQS) access benchmarking.
166
+ @service = Benchmark::Tms.new()
167
+ # Benchmark::Tms instance for XML parsing benchmarking.
168
+ @xml = Benchmark::Tms.new()
169
+ end
170
+ end
171
+
172
+ class AwsNoChange < RuntimeError
173
+ end
174
+
175
+ class RightAwsBase
176
+
177
+ # Amazon HTTP Error handling
178
+
179
+ # Text, if found in an error message returned by AWS, indicates that this may be a transient
180
+ # error. Transient errors are automatically retried with exponential back-off.
181
+ AMAZON_PROBLEMS = [ 'internal service error',
182
+ 'is currently unavailable',
183
+ 'no response from',
184
+ 'Please try again',
185
+ 'InternalError',
186
+ 'Internal Server Error',
187
+ 'ServiceUnavailable', #from SQS docs
188
+ 'Unavailable',
189
+ 'This application is not currently available',
190
+ 'InsufficientInstanceCapacity'
191
+ ]
192
+ @@amazon_problems = AMAZON_PROBLEMS
193
+ # Returns a list of Amazon service responses which are known to be transient problems.
194
+ # We have to re-request if we get any of them, because the problem will probably disappear.
195
+ # By default this method returns the same value as the AMAZON_PROBLEMS const.
196
+ def self.amazon_problems
197
+ @@amazon_problems
198
+ end
199
+
200
+ # Sets the list of Amazon side problems. Use in conjunction with the
201
+ # getter to append problems.
202
+ def self.amazon_problems=(problems_list)
203
+ @@amazon_problems = problems_list
204
+ end
205
+
206
+ # Raise an exception if a timeout occures while an API call is in progress.
207
+ # This helps to avoid a duplicate resources creation when Amazon hangs for some time and
208
+ # RightHttpConnection is forced to use retries to get a response from it.
209
+ #
210
+ # If an API call action is in the list then no attempts to retry are performed.
211
+ #
212
+ RAISE_ON_TIMEOUT_ON_ACTIONS = %w{
213
+ AllocateAddress
214
+ CreateSnapshot
215
+ CreateVolume
216
+ PurchaseReservedInstancesOffering
217
+ RequestSpotInstances
218
+ RunInstances
219
+ }
220
+ @@raise_on_timeout_on_actions = RAISE_ON_TIMEOUT_ON_ACTIONS.dup
221
+
222
+ def self.raise_on_timeout_on_actions
223
+ @@raise_on_timeout_on_actions
224
+ end
225
+
226
+ def self.raise_on_timeout_on_actions=(actions_list)
227
+ @@raise_on_timeout_on_actions = actions_list
228
+ end
229
+
230
+ end
231
+
232
+ module RightAwsBaseInterface
233
+ DEFAULT_SIGNATURE_VERSION = '2'
234
+
235
+ @@caching = false
236
+ def self.caching
237
+ @@caching
238
+ end
239
+ def self.caching=(caching)
240
+ @@caching = caching
241
+ end
242
+
243
+ # Current aws_access_key_id
244
+ attr_reader :aws_access_key_id
245
+ # Current aws_secret_access_key
246
+ attr_reader :aws_secret_access_key
247
+ # Last HTTP request object
248
+ attr_reader :last_request
249
+ # Last HTTP response object
250
+ attr_reader :last_response
251
+ # Last AWS errors list (used by AWSErrorHandler)
252
+ attr_accessor :last_errors
253
+ # Last AWS request id (used by AWSErrorHandler)
254
+ attr_accessor :last_request_id
255
+ # Logger object
256
+ attr_accessor :logger
257
+ # Initial params hash
258
+ attr_accessor :params
259
+ # RightHttpConnection instance
260
+ attr_reader :connection
261
+ # Cache
262
+ attr_reader :cache
263
+ # Signature version (all services except s3)
264
+ attr_reader :signature_version
265
+
266
+ def init(service_info, aws_access_key_id, aws_secret_access_key, params={}) #:nodoc:
267
+ @params = params
268
+ # If one defines EC2_URL he may forget to use a single slash as an "empty service" path.
269
+ # Amazon does not like this therefore add this bad boy if he is missing...
270
+ service_info[:default_service] = '/' if service_info[:default_service].right_blank?
271
+ raise AwsError.new("AWS access keys are required to operate on #{service_info[:name]}") \
272
+ if aws_access_key_id.right_blank? || aws_secret_access_key.right_blank?
273
+ @aws_access_key_id = aws_access_key_id
274
+ @aws_secret_access_key = aws_secret_access_key
275
+ # if the endpoint was explicitly defined - then use it
276
+ if @params[:endpoint_url]
277
+ @params[:server] = URI.parse(@params[:endpoint_url]).host
278
+ @params[:port] = URI.parse(@params[:endpoint_url]).port
279
+ @params[:service] = URI.parse(@params[:endpoint_url]).path
280
+ # make sure the 'service' path is not empty
281
+ @params[:service] = service_info[:default_service] if @params[:service].right_blank?
282
+ @params[:protocol] = URI.parse(@params[:endpoint_url]).scheme
283
+ @params[:region] = nil
284
+ else
285
+ @params[:server] ||= service_info[:default_host]
286
+ @params[:server] = "#{@params[:region]}.#{@params[:server]}" if @params[:region]
287
+ @params[:port] ||= service_info[:default_port]
288
+ @params[:service] ||= service_info[:default_service]
289
+ @params[:protocol] ||= service_info[:default_protocol]
290
+ end
291
+ # a set of options to be passed to RightHttpConnection object
292
+ @params[:connection_options] = {} unless @params[:connection_options].is_a?(Hash)
293
+ @with_connection_options = {}
294
+ @params[:connections] ||= :shared # || :dedicated
295
+ @params[:max_connections] ||= 10
296
+ @params[:connection_lifetime] ||= 20*60
297
+ @params[:api_version] ||= service_info[:default_api_version]
298
+ @logger = @params[:logger]
299
+ @logger = ::Rails.logger if !@logger && defined?(::Rails) && ::Rails.respond_to?(:logger)
300
+ @logger = RAILS_DEFAULT_LOGGER if !@logger && defined?(RAILS_DEFAULT_LOGGER)
301
+ @logger = Logger.new(STDOUT) if !@logger
302
+ @logger.info "New #{self.class.name} using #{@params[:connections]} connections mode"
303
+ @error_handler = nil
304
+ @cache = {}
305
+ @signature_version = (params[:signature_version] || DEFAULT_SIGNATURE_VERSION).to_s
306
+ end
307
+
308
+ def signed_service_params(aws_secret_access_key, service_hash, http_verb=nil, host=nil, service=nil )
309
+ case signature_version.to_s
310
+ when '0' then AwsUtils::sign_request_v0(aws_secret_access_key, service_hash)
311
+ when '1' then AwsUtils::sign_request_v1(aws_secret_access_key, service_hash)
312
+ when '2' then AwsUtils::sign_request_v2(aws_secret_access_key, service_hash, http_verb, host, service)
313
+ else raise AwsError.new("Unknown signature version (#{signature_version.to_s}) requested")
314
+ end
315
+ end
316
+
317
+ # Returns +true+ if the describe_xxx responses are being cached
318
+ def caching?
319
+ @params.key?(:cache) ? @params[:cache] : @@caching
320
+ end
321
+
322
+ # Check if the aws function response hits the cache or not.
323
+ # If the cache hits:
324
+ # - raises an +AwsNoChange+ exception if +do_raise+ == +:raise+.
325
+ # - returnes parsed response from the cache if it exists or +true+ otherwise.
326
+ # If the cache miss or the caching is off then returns +false+.
327
+ def cache_hits?(function, response, do_raise=:raise)
328
+ result = false
329
+ if caching?
330
+ function = function.to_sym
331
+ # get rid of requestId (this bad boy was added for API 2008-08-08+ and it is uniq for every response)
332
+ # feb 04, 2009 (load balancer uses 'RequestId' hence use 'i' modifier to hit it also)
333
+ response = response.sub(%r{<requestId>.+?</requestId>}i, '')
334
+ response_md5 = MD5.md5(response).to_s
335
+ # check for changes
336
+ unless @cache[function] && @cache[function][:response_md5] == response_md5
337
+ # well, the response is new, reset cache data
338
+ update_cache(function, {:response_md5 => response_md5,
339
+ :timestamp => Time.now,
340
+ :hits => 0,
341
+ :parsed => nil})
342
+ else
343
+ # aha, cache hits, update the data and throw an exception if needed
344
+ @cache[function][:hits] += 1
345
+ if do_raise == :raise
346
+ raise(AwsNoChange, "Cache hit: #{function} response has not changed since "+
347
+ "#{@cache[function][:timestamp].strftime('%Y-%m-%d %H:%M:%S')}, "+
348
+ "hits: #{@cache[function][:hits]}.")
349
+ else
350
+ result = @cache[function][:parsed] || true
351
+ end
352
+ end
353
+ end
354
+ result
355
+ end
356
+
357
+ def update_cache(function, hash)
358
+ (@cache[function.to_sym] ||= {}).merge!(hash) if caching?
359
+ end
360
+
361
+ def on_exception(options={:raise=>true, :log=>true}) # :nodoc:
362
+ raise if $!.is_a?(AwsNoChange)
363
+ AwsError::on_aws_exception(self, options)
364
+ end
365
+
366
+ #----------------------------
367
+ # HTTP Connections handling
368
+ #----------------------------
369
+
370
+ def get_server_url(request) # :nodoc:
371
+ "#{request[:protocol]}://#{request[:server]}:#{request[:port]}"
372
+ end
373
+
374
+ def get_connections_storage(aws_service) # :nodoc:
375
+ case @params[:connections].to_s
376
+ when 'dedicated' then @connections_storage ||= {}
377
+ else Thread.current[aws_service] ||= {}
378
+ end
379
+ end
380
+
381
+ def destroy_connection(request, reason) # :nodoc:
382
+ connections = get_connections_storage(request[:aws_service])
383
+ server_url = get_server_url(request)
384
+ if connections[server_url]
385
+ connections[server_url][:connection].finish(reason)
386
+ connections.delete(server_url)
387
+ end
388
+ end
389
+
390
+ # Expire the connection if it has expired.
391
+ def get_connection(request) # :nodoc:
392
+ server_url = get_server_url(request)
393
+ connection_storage = get_connections_storage(request[:aws_service])
394
+ life_time_scratch = Time.now-@params[:connection_lifetime]
395
+ # Delete out-of-dated connections
396
+ connections_in_list = 0
397
+ connection_storage.to_a.sort{|conn1, conn2| conn2[1][:last_used_at] <=> conn1[1][:last_used_at]}.each do |serv_url, conn_opts|
398
+ if @params[:max_connections] <= connections_in_list
399
+ conn_opts[:connection].finish('out-of-limit')
400
+ connection_storage.delete(server_url)
401
+ elsif conn_opts[:last_used_at] < life_time_scratch
402
+ conn_opts[:connection].finish('out-of-date')
403
+ connection_storage.delete(server_url)
404
+ else
405
+ connections_in_list += 1
406
+ end
407
+ end
408
+ connection = (connection_storage[server_url] ||= {})
409
+ connection[:last_used_at] = Time.now
410
+ connection[:connection] ||= Rightscale::HttpConnection.new(:exception => RightAws::AwsError, :logger => @logger)
411
+ end
412
+
413
+ #----------------------------
414
+ # HTTP Requests handling
415
+ #----------------------------
416
+
417
+ # ACF, AMS, EC2, LBS and SDB uses this guy
418
+ # SQS and S3 use their own methods
419
+ def generate_request_impl(verb, action, options={}) #:nodoc:
420
+ # Form a valid http verb: 'GET' or 'POST' (all the other are not supported now)
421
+ http_verb = verb.to_s.upcase
422
+ # remove empty keys from request options
423
+ options.delete_if { |key, value| value.nil? }
424
+ # prepare service data
425
+ service_hash = {"Action" => action,
426
+ "AWSAccessKeyId" => @aws_access_key_id,
427
+ "Version" => @params[:api_version] }
428
+ service_hash.merge!(options)
429
+ # Sign request options
430
+ service_params = signed_service_params(@aws_secret_access_key, service_hash, http_verb, @params[:server], @params[:service])
431
+ # Use POST if the length of the query string is too large
432
+ # see http://docs.amazonwebservices.com/AmazonSimpleDB/2007-11-07/DeveloperGuide/MakingRESTRequests.html
433
+ if http_verb != 'POST' && service_params.size > 2000
434
+ http_verb = 'POST'
435
+ if signature_version == '2'
436
+ service_params = signed_service_params(@aws_secret_access_key, service_hash, http_verb, @params[:server], @params[:service])
437
+ end
438
+ end
439
+ # create a request
440
+ case http_verb
441
+ when 'GET'
442
+ request = Net::HTTP::Get.new("#{@params[:service]}?#{service_params}")
443
+ when 'POST'
444
+ request = Net::HTTP::Post.new(@params[:service])
445
+ request.body = service_params
446
+ request['Content-Type'] = 'application/x-www-form-urlencoded; charset=utf-8'
447
+ else
448
+ raise "Unsupported HTTP verb #{verb.inspect}!"
449
+ end
450
+ # prepare output hash
451
+ request_hash = { :request => request,
452
+ :server => @params[:server],
453
+ :port => @params[:port],
454
+ :protocol => @params[:protocol] }
455
+ request_hash.merge!(@params[:connection_options])
456
+ request_hash.merge!(@with_connection_options)
457
+
458
+ # If an action is marked as "non-retryable" and there was no :raise_on_timeout option set
459
+ # explicitly then do set that option
460
+ if Array(RightAwsBase::raise_on_timeout_on_actions).include?(action) && !request_hash.has_key?(:raise_on_timeout)
461
+ request_hash.merge!(:raise_on_timeout => true)
462
+ end
463
+
464
+ request_hash
465
+ end
466
+
467
+ # All services uses this guy.
468
+ def request_info_impl(aws_service, benchblock, request, parser, &block) #:nodoc:
469
+ request[:aws_service] = aws_service
470
+ @connection = get_connection(request)
471
+ @last_request = request[:request]
472
+ @last_response = nil
473
+ response = nil
474
+ blockexception = nil
475
+
476
+ if(block != nil)
477
+ # TRB 9/17/07 Careful - because we are passing in blocks, we get a situation where
478
+ # an exception may get thrown in the block body (which is high-level
479
+ # code either here or in the application) but gets caught in the
480
+ # low-level code of HttpConnection. The solution is not to let any
481
+ # exception escape the block that we pass to HttpConnection::request.
482
+ # Exceptions can originate from code directly in the block, or from user
483
+ # code called in the other block which is passed to response.read_body.
484
+ benchblock.service.add! do
485
+ begin
486
+ responsehdr = @connection.request(request) do |response|
487
+ #########
488
+ begin
489
+ @last_response = response
490
+ if response.is_a?(Net::HTTPSuccess)
491
+ @error_handler = nil
492
+ response.read_body(&block)
493
+ else
494
+ @error_handler = AWSErrorHandler.new(self, parser, :errors_list => self.class.amazon_problems) unless @error_handler
495
+ check_result = @error_handler.check(request)
496
+ if check_result
497
+ @error_handler = nil
498
+ return check_result
499
+ end
500
+ raise AwsError.new(@last_errors, @last_response.code, @last_request_id)
501
+ end
502
+ rescue Exception => e
503
+ blockexception = e
504
+ end
505
+ end
506
+ rescue Exception => e
507
+ # Kill a connection if we run into a low level connection error
508
+ destroy_connection(request, "error: #{e.message}")
509
+ raise e
510
+ end
511
+ #########
512
+
513
+ #OK, now we are out of the block passed to the lower level
514
+ if(blockexception)
515
+ raise blockexception
516
+ end
517
+ benchblock.xml.add! do
518
+ parser.parse(responsehdr)
519
+ end
520
+ return parser.result
521
+ end
522
+ else
523
+ benchblock.service.add! do
524
+ begin
525
+ response = @connection.request(request)
526
+ rescue Exception => e
527
+ # Kill a connection if we run into a low level connection error
528
+ destroy_connection(request, "error: #{e.message}")
529
+ raise e
530
+ end
531
+ end
532
+ # check response for errors...
533
+ @last_response = response
534
+ if response.is_a?(Net::HTTPSuccess)
535
+ @error_handler = nil
536
+ benchblock.xml.add! { parser.parse(response) }
537
+ return parser.result
538
+ else
539
+ @error_handler = AWSErrorHandler.new(self, parser, :errors_list => self.class.amazon_problems) unless @error_handler
540
+ check_result = @error_handler.check(request)
541
+ if check_result
542
+ @error_handler = nil
543
+ return check_result
544
+ end
545
+ raise AwsError.new(@last_errors, @last_response.code, @last_request_id)
546
+ end
547
+ end
548
+ rescue
549
+ @error_handler = nil
550
+ raise
551
+ end
552
+
553
+ def request_cache_or_info(method, link, parser_class, benchblock, use_cache=true, &block) #:nodoc:
554
+ # We do not want to break the logic of parsing hence will use a dummy parser to process all the standard
555
+ # steps (errors checking etc). The dummy parser does nothig - just returns back the params it received.
556
+ # If the caching is enabled and hit then throw AwsNoChange.
557
+ # P.S. caching works for the whole images list only! (when the list param is blank)
558
+ # check cache
559
+ response, params = request_info(link, RightDummyParser.new)
560
+ cache_hits?(method.to_sym, response.body) if use_cache
561
+ parser = parser_class.new(:logger => @logger)
562
+ benchblock.xml.add!{ parser.parse(response, params) }
563
+ result = block ? block.call(parser) : parser.result
564
+ # update parsed data
565
+ update_cache(method.to_sym, :parsed => result) if use_cache
566
+ result
567
+ end
568
+
569
+ # Returns Amazons request ID for the latest request
570
+ def last_request_id
571
+ @last_response && @last_response.body.to_s[%r{<requestId>(.+?)</requestId>}i] && $1
572
+ end
573
+
574
+ # Incrementally lists something.
575
+ def incrementally_list_items(action, parser_class, params={}, &block) # :nodoc:
576
+ params = params.dup
577
+ params['MaxItems'] = params.delete(:max_items) if params[:max_items]
578
+ params['Marker'] = params.delete(:marker) if params[:marker]
579
+ last_response = nil
580
+ loop do
581
+ last_response = request_info( generate_request(action, params), parser_class.new(:logger => @logger))
582
+ params['Marker'] = last_response[:marker]
583
+ break unless block && block.call(last_response) && !last_response[:marker].right_blank?
584
+ end
585
+ last_response
586
+ end
587
+
588
+ # Format array of items into Amazons handy hash ('?' is a place holder):
589
+ # Options:
590
+ # :default => "something" : Set a value to "something" when it is nil
591
+ # :default => :skip_nils : Skip nil values
592
+ #
593
+ # amazonize_list('Item', ['a', 'b', 'c']) =>
594
+ # { 'Item.1' => 'a', 'Item.2' => 'b', 'Item.3' => 'c' }
595
+ #
596
+ # amazonize_list('Item.?.instance', ['a', 'c']) #=>
597
+ # { 'Item.1.instance' => 'a', 'Item.2.instance' => 'c' }
598
+ #
599
+ # amazonize_list(['Item.?.Name', 'Item.?.Value'], {'A' => 'a', 'B' => 'b'}) #=>
600
+ # { 'Item.1.Name' => 'A', 'Item.1.Value' => 'a',
601
+ # 'Item.2.Name' => 'B', 'Item.2.Value' => 'b' }
602
+ #
603
+ # amazonize_list(['Item.?.Name', 'Item.?.Value'], [['A','a'], ['B','b']]) #=>
604
+ # { 'Item.1.Name' => 'A', 'Item.1.Value' => 'a',
605
+ # 'Item.2.Name' => 'B', 'Item.2.Value' => 'b' }
606
+ #
607
+ # amazonize_list(['Filter.?.Key', 'Filter.?.Value.?'], {'A' => ['aa','ab'], 'B' => ['ba','bb']}) #=>
608
+ # amazonize_list(['Filter.?.Key', 'Filter.?.Value.?'], [['A',['aa','ab']], ['B',['ba','bb']]]) #=>
609
+ # {"Filter.1.Key"=>"A",
610
+ # "Filter.1.Value.1"=>"aa",
611
+ # "Filter.1.Value.2"=>"ab",
612
+ # "Filter.2.Key"=>"B",
613
+ # "Filter.2.Value.1"=>"ba",
614
+ # "Filter.2.Value.2"=>"bb"}
615
+ def amazonize_list(masks, list, options={}) #:nodoc:
616
+ groups = {}
617
+ Array(list).each_with_index do |list_item, i|
618
+ Array(masks).each_with_index do |mask, mask_idx|
619
+ key = mask[/\?/] ? mask.dup : mask.dup + '.?'
620
+ key.sub!('?', (i+1).to_s)
621
+ value = Array(list_item)[mask_idx]
622
+ if value.is_a?(Array)
623
+ groups.merge!(amazonize_list(key, value, options))
624
+ else
625
+ if value.nil?
626
+ next if options[:default] == :skip_nils
627
+ value = options[:default]
628
+ end
629
+ # Hack to avoid having unhandled '?' in keys : do replace them all with '1':
630
+ # bad: ec2.amazonize_list(['Filter.?.Key', 'Filter.?.Value.?'], { a: => :b }) => {"Filter.1.Key"=>:a, "Filter.1.Value.?"=>1}
631
+ # good: ec2.amazonize_list(['Filter.?.Key', 'Filter.?.Value.?'], { a: => :b }) => {"Filter.1.Key"=>:a, "Filter.1.Value.1"=>1}
632
+ key.gsub!('?', '1')
633
+ groups[key] = value
634
+ end
635
+ end
636
+ end
637
+ groups
638
+ end
639
+
640
+ BLOCK_DEVICE_KEY_MAPPING = { # :nodoc:
641
+ :device_name => 'DeviceName',
642
+ :virtual_name => 'VirtualName',
643
+ :no_device => 'NoDevice',
644
+ :ebs_snapshot_id => 'Ebs.SnapshotId',
645
+ :ebs_volume_size => 'Ebs.VolumeSize',
646
+ :ebs_delete_on_termination => 'Ebs.DeleteOnTermination' }
647
+
648
+ def amazonize_block_device_mappings(block_device_mappings, key = 'BlockDeviceMapping') # :nodoc:
649
+ result = {}
650
+ unless block_device_mappings.right_blank?
651
+ block_device_mappings = [block_device_mappings] unless block_device_mappings.is_a?(Array)
652
+ block_device_mappings.each_with_index do |b, idx|
653
+ BLOCK_DEVICE_KEY_MAPPING.each do |local_name, remote_name|
654
+ value = b[local_name]
655
+ case local_name
656
+ when :no_device then value = value ? '' : nil # allow to pass :no_device as boolean
657
+ end
658
+ result["#{key}.#{idx+1}.#{remote_name}"] = value unless value.nil?
659
+ end
660
+ end
661
+ end
662
+ result
663
+ end
664
+
665
+ # Execute a block of code with custom set of settings for right_http_connection.
666
+ # Accepts next options (see Rightscale::HttpConnection for explanation):
667
+ # :raise_on_timeout
668
+ # :http_connection_retry_count
669
+ # :http_connection_open_timeout
670
+ # :http_connection_read_timeout
671
+ # :http_connection_retry_delay
672
+ # :user_agent
673
+ # :exception
674
+ #
675
+ # Example #1:
676
+ #
677
+ # # Try to create a snapshot but stop with exception if timeout is received
678
+ # # to avoid having a duplicate API calls that create duplicate snapshots.
679
+ # ec2 = Rightscale::Ec2::new(aws_access_key_id, aws_secret_access_key)
680
+ # ec2.with_connection_options(:raise_on_timeout => true) do
681
+ # ec2.create_snapshot('vol-898a6fe0', 'KD: WooHoo!!')
682
+ # end
683
+ #
684
+ # Example #2:
685
+ #
686
+ # # Opposite case when the setting is global:
687
+ # @ec2 = Rightscale::Ec2::new(aws_access_key_id, aws_secret_access_key,
688
+ # :connection_options => { :raise_on_timeout => true })
689
+ # # Create an SSHKey but do tries on timeout
690
+ # ec2.with_connection_options(:raise_on_timeout => false) do
691
+ # new_key = ec2.create_key_pair('my_test_key')
692
+ # end
693
+ #
694
+ # Example #3:
695
+ #
696
+ # # Global settings (HttpConnection level):
697
+ # Rightscale::HttpConnection::params[:http_connection_open_timeout] = 5
698
+ # Rightscale::HttpConnection::params[:http_connection_read_timeout] = 250
699
+ # Rightscale::HttpConnection::params[:http_connection_retry_count] = 2
700
+ #
701
+ # # Local setings (RightAws level)
702
+ # ec2 = Rightscale::Ec2::new(AWS_ID, AWS_KEY,
703
+ # :region => 'us-east-1',
704
+ # :connection_options => {
705
+ # :http_connection_read_timeout => 2,
706
+ # :http_connection_retry_count => 5,
707
+ # :user_agent => 'Mozilla 4.0'
708
+ # })
709
+ #
710
+ # # Custom settings (API call level)
711
+ # ec2.with_connection_options(:raise_on_timeout => true,
712
+ # :http_connection_read_timeout => 10,
713
+ # :user_agent => '') do
714
+ # pp ec2.describe_images
715
+ # end
716
+ #
717
+ def with_connection_options(options, &block)
718
+ @with_connection_options = options
719
+ block.call self
720
+ ensure
721
+ @with_connection_options = {}
722
+ end
723
+ end
724
+
725
+
726
+ # Exception class to signal any Amazon errors. All errors occuring during calls to Amazon's
727
+ # web services raise this type of error.
728
+ # Attribute inherited by RuntimeError:
729
+ # message - the text of the error, generally as returned by AWS in its XML response.
730
+ class AwsError < RuntimeError
731
+
732
+ # either an array of errors where each item is itself an array of [code, message]),
733
+ # or an error string if the error was raised manually, as in <tt>AwsError.new('err_text')</tt>
734
+ attr_reader :errors
735
+
736
+ # Request id (if exists)
737
+ attr_reader :request_id
738
+
739
+ # Response HTTP error code
740
+ attr_reader :http_code
741
+
742
+ def initialize(errors=nil, http_code=nil, request_id=nil)
743
+ @errors = errors
744
+ @request_id = request_id
745
+ @http_code = http_code
746
+ super(@errors.is_a?(Array) ? @errors.map{|code, msg| "#{code}: #{msg}"}.join("; ") : @errors.to_s)
747
+ end
748
+
749
+ # Does any of the error messages include the regexp +pattern+?
750
+ # Used to determine whether to retry request.
751
+ def include?(pattern)
752
+ if @errors.is_a?(Array)
753
+ @errors.each{ |code, msg| return true if code =~ pattern }
754
+ else
755
+ return true if @errors_str =~ pattern
756
+ end
757
+ false
758
+ end
759
+
760
+ # Generic handler for AwsErrors. +aws+ is the RightAws::S3, RightAws::EC2, or RightAws::SQS
761
+ # object that caused the exception (it must provide last_request and last_response). Supported
762
+ # boolean options are:
763
+ # * <tt>:log</tt> print a message into the log using aws.logger to access the Logger
764
+ # * <tt>:puts</tt> do a "puts" of the error
765
+ # * <tt>:raise</tt> re-raise the error after logging
766
+ def self.on_aws_exception(aws, options={:raise=>true, :log=>true})
767
+ # Only log & notify if not user error
768
+ if !options[:raise] || system_error?($!)
769
+ error_text = "#{$!.inspect}\n#{$@}.join('\n')}"
770
+ puts error_text if options[:puts]
771
+ # Log the error
772
+ if options[:log]
773
+ request = aws.last_request ? aws.last_request.path : '-none-'
774
+ response = aws.last_response ? "#{aws.last_response.code} -- #{aws.last_response.message} -- #{aws.last_response.body}" : '-none-'
775
+ aws.logger.error error_text
776
+ aws.logger.error "Request was: #{request}"
777
+ aws.logger.error "Response was: #{response}"
778
+ end
779
+ end
780
+ raise if options[:raise] # re-raise an exception
781
+ return nil
782
+ end
783
+
784
+ # True if e is an AWS system error, i.e. something that is for sure not the caller's fault.
785
+ # Used to force logging.
786
+ def self.system_error?(e)
787
+ !e.is_a?(self) || e.message =~ /InternalError|InsufficientInstanceCapacity|Unavailable/
788
+ end
789
+
790
+ end
791
+
792
+
793
+ class AWSErrorHandler
794
+ # 0-100 (%)
795
+ DEFAULT_CLOSE_ON_4XX_PROBABILITY = 10
796
+
797
+ @@reiteration_start_delay = 0.2
798
+ def self.reiteration_start_delay
799
+ @@reiteration_start_delay
800
+ end
801
+ def self.reiteration_start_delay=(reiteration_start_delay)
802
+ @@reiteration_start_delay = reiteration_start_delay
803
+ end
804
+
805
+ @@reiteration_time = 5
806
+ def self.reiteration_time
807
+ @@reiteration_time
808
+ end
809
+ def self.reiteration_time=(reiteration_time)
810
+ @@reiteration_time = reiteration_time
811
+ end
812
+
813
+ @@close_on_error = true
814
+ def self.close_on_error
815
+ @@close_on_error
816
+ end
817
+ def self.close_on_error=(close_on_error)
818
+ @@close_on_error = close_on_error
819
+ end
820
+
821
+ @@close_on_4xx_probability = DEFAULT_CLOSE_ON_4XX_PROBABILITY
822
+ def self.close_on_4xx_probability
823
+ @@close_on_4xx_probability
824
+ end
825
+ def self.close_on_4xx_probability=(close_on_4xx_probability)
826
+ @@close_on_4xx_probability = close_on_4xx_probability
827
+ end
828
+
829
+ # params:
830
+ # :reiteration_time
831
+ # :errors_list
832
+ # :close_on_error = true | false
833
+ # :close_on_4xx_probability = 1-100
834
+ def initialize(aws, parser, params={}) #:nodoc:
835
+ @aws = aws # Link to RightEc2 | RightSqs | RightS3 instance
836
+ @parser = parser # parser to parse Amazon response
837
+ @started_at = Time.now
838
+ @stop_at = @started_at + (params[:reiteration_time] || @@reiteration_time)
839
+ @errors_list = params[:errors_list] || []
840
+ @reiteration_delay = @@reiteration_start_delay
841
+ @retries = 0
842
+ # close current HTTP(S) connection on 5xx, errors from list and 4xx errors
843
+ @close_on_error = params[:close_on_error].nil? ? @@close_on_error : params[:close_on_error]
844
+ @close_on_4xx_probability = params[:close_on_4xx_probability] || @@close_on_4xx_probability
845
+ end
846
+
847
+ # Returns false if
848
+ def check(request) #:nodoc:
849
+ result = false
850
+ error_found = false
851
+ redirect_detected= false
852
+ error_match = nil
853
+ last_errors_text = ''
854
+ response = @aws.last_response
855
+ # log error
856
+ request_text_data = "#{request[:protocol]}://#{request[:server]}:#{request[:port]}#{request[:request].path}"
857
+ # is this a redirect?
858
+ # yes!
859
+ if response.is_a?(Net::HTTPRedirection)
860
+ redirect_detected = true
861
+ else
862
+ # no, it's an error ...
863
+ @aws.logger.warn("##### #{@aws.class.name} returned an error: #{response.code} #{response.message}\n#{response.body} #####")
864
+ @aws.logger.warn("##### #{@aws.class.name} request: #{request_text_data} ####")
865
+ end
866
+
867
+ # Extract error/redirection message from the response body
868
+ # Amazon claims that a redirection must have a body but somethimes it is nil....
869
+ if response.body && response.body[/^(<\?xml|<ErrorResponse)/]
870
+ error_parser = RightErrorResponseParser.new
871
+ @aws.class.bench_xml.add! do
872
+ error_parser.parse(response.body)
873
+ end
874
+ @aws.last_errors = error_parser.errors
875
+ @aws.last_request_id = error_parser.requestID
876
+ last_errors_text = @aws.last_errors.flatten.join("\n")
877
+ else
878
+ @aws.last_errors = [[response.code, "#{response.message} (#{request_text_data})"]]
879
+ @aws.last_request_id = '-undefined-'
880
+ last_errors_text = response.message
881
+ end
882
+
883
+ # Ok, it is a redirect, find the new destination location
884
+ if redirect_detected
885
+ location = response['location']
886
+ # As for 301 ( Moved Permanently) Amazon does not return a 'Location' header but
887
+ # it is possible to extract a new endpoint from the response body
888
+ if location.right_blank? && response.code=='301' && response.body
889
+ new_endpoint = response.body[/<Endpoint>(.*?)<\/Endpoint>/] && $1
890
+ location = "#{request[:protocol]}://#{new_endpoint}:#{request[:port]}#{request[:request].path}"
891
+ end
892
+ # ... log information and ...
893
+ @aws.logger.info("##### #{@aws.class.name} redirect requested: #{response.code} #{response.message} #####")
894
+ @aws.logger.info(" Old location: #{request_text_data}")
895
+ @aws.logger.info(" New location: #{location}")
896
+ @aws.logger.info(" Request Verb: #{request[:request].class.name}")
897
+ # ... fix the connection data
898
+ request[:server] = URI.parse(location).host
899
+ request[:protocol] = URI.parse(location).scheme
900
+ request[:port] = URI.parse(location).port
901
+ else
902
+ # Not a redirect but an error: try to find the error in our list
903
+ @errors_list.each do |error_to_find|
904
+ if last_errors_text[/#{error_to_find}/i]
905
+ error_found = true
906
+ error_match = error_to_find
907
+ @aws.logger.warn("##### Retry is needed, error pattern match: #{error_to_find} #####")
908
+ break
909
+ end
910
+ end
911
+ end
912
+
913
+ # check the time has gone from the first error come
914
+ if redirect_detected || error_found
915
+ # Close the connection to the server and recreate a new one.
916
+ # It may have a chance that one server is a semi-down and reconnection
917
+ # will help us to connect to the other server
918
+ if !redirect_detected && @close_on_error
919
+ @aws.destroy_connection(request, "#{self.class.name}: error match to pattern '#{error_match}'")
920
+ end
921
+
922
+ if (Time.now < @stop_at)
923
+ @retries += 1
924
+ unless redirect_detected
925
+ @aws.logger.warn("##### Retry ##{@retries} is being performed. Sleeping for #{@reiteration_delay} sec. Whole time: #{Time.now-@started_at} sec ####")
926
+ sleep @reiteration_delay
927
+ @reiteration_delay *= 2
928
+
929
+ # Always make sure that the fp is set to point to the beginning(?)
930
+ # of the File/IO. TODO: it assumes that offset is 0, which is bad.
931
+ if(request[:request].body_stream && request[:request].body_stream.respond_to?(:pos))
932
+ begin
933
+ request[:request].body_stream.pos = 0
934
+ rescue Exception => e
935
+ @logger.warn("Retry may fail due to unable to reset the file pointer" +
936
+ " -- #{self.class.name} : #{e.inspect}")
937
+ end
938
+ end
939
+ else
940
+ @aws.logger.info("##### Retry ##{@retries} is being performed due to a redirect. ####")
941
+ end
942
+ result = @aws.request_info(request, @parser)
943
+ else
944
+ @aws.logger.warn("##### Ooops, time is over... ####")
945
+ end
946
+ # aha, this is unhandled error:
947
+ elsif @close_on_error
948
+ # On 5xx(Server errors), 403(RequestTimeTooSkewed) and 408(Request Timeout) a conection has to be closed
949
+ if @aws.last_response.code.to_s[/^(5\d\d|403|408)$/]
950
+ @aws.destroy_connection(request, "#{self.class.name}: code: #{@aws.last_response.code}: '#{@aws.last_response.message}'")
951
+ # Is this a 4xx error ?
952
+ elsif @aws.last_response.code.to_s[/^4\d\d$/] && @close_on_4xx_probability > rand(100)
953
+ @aws.destroy_connection(request, "#{self.class.name}: code: #{@aws.last_response.code}: '#{@aws.last_response.message}', " +
954
+ "probability: #{@close_on_4xx_probability}%")
955
+ end
956
+ end
957
+ result
958
+ end
959
+
960
+ end
961
+
962
+
963
+ #-----------------------------------------------------------------
964
+
965
+ class RightSaxParserCallbackTemplate #:nodoc:
966
+ def initialize(right_aws_parser)
967
+ @right_aws_parser = right_aws_parser
968
+ end
969
+ def on_characters(chars)
970
+ @right_aws_parser.text(chars)
971
+ end
972
+ def on_start_document; end
973
+ def on_comment(msg); end
974
+ def on_processing_instruction(target, data); end
975
+ def on_cdata_block(cdata); end
976
+ def on_end_document; end
977
+ end
978
+
979
+ class RightSaxParserCallback < RightSaxParserCallbackTemplate
980
+ def self.include_callback
981
+ include XML::SaxParser::Callbacks
982
+ end
983
+ def on_start_element(name, attr_hash)
984
+ @right_aws_parser.tag_start(name, attr_hash)
985
+ end
986
+ def on_end_element(name)
987
+ @right_aws_parser.tag_end(name)
988
+ end
989
+ end
990
+
991
+ class RightSaxParserCallbackNs < RightSaxParserCallbackTemplate
992
+ def on_start_element_ns(name, attr_hash, prefix, uri, namespaces)
993
+ @right_aws_parser.tag_start(name, attr_hash)
994
+ end
995
+ def on_end_element_ns(name, prefix, uri)
996
+ @right_aws_parser.tag_end(name)
997
+ end
998
+ end
999
+
1000
+ class RightAWSParser #:nodoc:
1001
+ # default parsing library
1002
+ DEFAULT_XML_LIBRARY = 'rexml'
1003
+ # a list of supported parsers
1004
+ @@supported_xml_libs = [DEFAULT_XML_LIBRARY, 'libxml']
1005
+
1006
+ @@xml_lib = DEFAULT_XML_LIBRARY # xml library name: 'rexml' | 'libxml'
1007
+ def self.xml_lib
1008
+ @@xml_lib
1009
+ end
1010
+ def self.xml_lib=(new_lib_name)
1011
+ @@xml_lib = new_lib_name
1012
+ end
1013
+
1014
+ attr_accessor :result
1015
+ attr_reader :xmlpath
1016
+ attr_accessor :xml_lib
1017
+ attr_reader :full_tag_name
1018
+ attr_reader :tag
1019
+
1020
+ def initialize(params={})
1021
+ @xmlpath = ''
1022
+ @full_tag_name = ''
1023
+ @result = false
1024
+ @text = ''
1025
+ @tag = ''
1026
+ @xml_lib = params[:xml_lib] || @@xml_lib
1027
+ @logger = params[:logger]
1028
+ reset
1029
+ end
1030
+ def tag_start(name, attributes)
1031
+ @text = ''
1032
+ @tag = name
1033
+ @full_tag_name += @full_tag_name.empty? ? name : "/#{name}"
1034
+ tagstart(name, attributes)
1035
+ @xmlpath = @full_tag_name
1036
+ end
1037
+ def tag_end(name)
1038
+ @xmlpath = @full_tag_name[/^(.*?)\/?#{name}$/] && $1
1039
+ tagend(name)
1040
+ @full_tag_name = @xmlpath
1041
+ end
1042
+ def text(text)
1043
+ @text += text
1044
+ tagtext(text)
1045
+ end
1046
+ # Parser method.
1047
+ # Params:
1048
+ # xml_text - xml message text(String) or Net:HTTPxxx instance (response)
1049
+ # params[:xml_lib] - library name: 'rexml' | 'libxml'
1050
+ def parse(xml_text, params={})
1051
+ # Get response body
1052
+ xml_text = xml_text.body unless xml_text.is_a?(String)
1053
+ @xml_lib = params[:xml_lib] || @xml_lib
1054
+ # check that we had no problems with this library otherwise use default
1055
+ @xml_lib = DEFAULT_XML_LIBRARY unless @@supported_xml_libs.include?(@xml_lib)
1056
+ # load xml library
1057
+ if @xml_lib=='libxml' && !defined?(XML::SaxParser)
1058
+ begin
1059
+ require 'xml/libxml'
1060
+ # Setup SaxParserCallback
1061
+ if XML::Parser::VERSION >= '0.5.1' &&
1062
+ XML::Parser::VERSION < '0.9.7'
1063
+ RightSaxParserCallback.include_callback
1064
+ end
1065
+ rescue LoadError => e
1066
+ @@supported_xml_libs.delete(@xml_lib)
1067
+ @xml_lib = DEFAULT_XML_LIBRARY
1068
+ if @logger
1069
+ @logger.error e.inspect
1070
+ @logger.error e.backtrace
1071
+ @logger.info "Can not load 'libxml' library. '#{DEFAULT_XML_LIBRARY}' is used for parsing."
1072
+ end
1073
+ end
1074
+ end
1075
+ # Parse the xml text
1076
+ case @xml_lib
1077
+ when 'libxml'
1078
+ if XML::Parser::VERSION >= '0.9.9'
1079
+ # avoid warning on every usage
1080
+ xml = XML::SaxParser.string(xml_text)
1081
+ else
1082
+ xml = XML::SaxParser.new
1083
+ xml.string = xml_text
1084
+ end
1085
+ # check libxml-ruby version
1086
+ if XML::Parser::VERSION >= '0.9.7'
1087
+ xml.callbacks = RightSaxParserCallbackNs.new(self)
1088
+ elsif XML::Parser::VERSION >= '0.5.1'
1089
+ xml.callbacks = RightSaxParserCallback.new(self)
1090
+ else
1091
+ xml.on_start_element{|name, attr_hash| self.tag_start(name, attr_hash)}
1092
+ xml.on_characters{ |text| self.text(text)}
1093
+ xml.on_end_element{ |name| self.tag_end(name)}
1094
+ end
1095
+ xml.parse
1096
+ else
1097
+ REXML::Document.parse_stream(xml_text, self)
1098
+ end
1099
+ end
1100
+ # Parser must have a lots of methods
1101
+ # (see /usr/lib/ruby/1.8/rexml/parsers/streamparser.rb)
1102
+ # We dont need most of them in RightAWSParser and method_missing helps us
1103
+ # to skip their definition
1104
+ def method_missing(method, *params)
1105
+ # if the method is one of known - just skip it ...
1106
+ return if [:comment, :attlistdecl, :notationdecl, :elementdecl,
1107
+ :entitydecl, :cdata, :xmldecl, :attlistdecl, :instruction,
1108
+ :doctype].include?(method)
1109
+ # ... else - call super to raise an exception
1110
+ super(method, params)
1111
+ end
1112
+ # the functions to be overriden by children (if nessesery)
1113
+ def reset ; end
1114
+ def tagstart(name, attributes); end
1115
+ def tagend(name) ; end
1116
+ def tagtext(text) ; end
1117
+ end
1118
+
1119
+ #-----------------------------------------------------------------
1120
+ # PARSERS: Errors
1121
+ #-----------------------------------------------------------------
1122
+
1123
+ #<Error>
1124
+ # <Code>TemporaryRedirect</Code>
1125
+ # <Message>Please re-send this request to the specified temporary endpoint. Continue to use the original request endpoint for future requests.</Message>
1126
+ # <RequestId>FD8D5026D1C5ABA3</RequestId>
1127
+ # <Endpoint>bucket-for-k.s3-external-3.amazonaws.com</Endpoint>
1128
+ # <HostId>ItJy8xPFPli1fq/JR3DzQd3iDvFCRqi1LTRmunEdM1Uf6ZtW2r2kfGPWhRE1vtaU</HostId>
1129
+ # <Bucket>bucket-for-k</Bucket>
1130
+ #</Error>
1131
+
1132
+ class RightErrorResponseParser < RightAWSParser #:nodoc:
1133
+ attr_accessor :errors # array of hashes: error/message
1134
+ attr_accessor :requestID
1135
+ # attr_accessor :endpoint, :host_id, :bucket
1136
+ def tagend(name)
1137
+ case name
1138
+ when 'RequestID' ; @requestID = @text
1139
+ when 'Code' ; @code = @text
1140
+ when 'Message' ; @message = @text
1141
+ # when 'Endpoint' ; @endpoint = @text
1142
+ # when 'HostId' ; @host_id = @text
1143
+ # when 'Bucket' ; @bucket = @text
1144
+ when 'Error' ; @errors << [ @code, @message ]
1145
+ end
1146
+ end
1147
+ def reset
1148
+ @errors = []
1149
+ end
1150
+ end
1151
+
1152
+ # Dummy parser - does nothing
1153
+ # Returns the original params back
1154
+ class RightDummyParser # :nodoc:
1155
+ attr_accessor :result
1156
+ def parse(response, params={})
1157
+ @result = [response, params]
1158
+ end
1159
+ end
1160
+
1161
+ class RightHttp2xxParser < RightAWSParser # :nodoc:
1162
+ def parse(response)
1163
+ @result = response.is_a?(Net::HTTPSuccess)
1164
+ end
1165
+ end
1166
+
1167
+ class RightBoolResponseParser < RightAWSParser #:nodoc:
1168
+ def tagend(name)
1169
+ @result = (@text=='true') if name == 'return'
1170
+ end
1171
+ end
1172
+
1173
+ end
1174
+