aboisvert_aws 3.0.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (78) hide show
  1. data/History.txt +329 -0
  2. data/Manifest.txt +61 -0
  3. data/README.txt +163 -0
  4. data/Rakefile +130 -0
  5. data/lib/acf/right_acf_interface.rb +549 -0
  6. data/lib/acf/right_acf_invalidations.rb +144 -0
  7. data/lib/acf/right_acf_origin_access_identities.rb +230 -0
  8. data/lib/acf/right_acf_streaming_interface.rb +229 -0
  9. data/lib/acw/right_acw_interface.rb +248 -0
  10. data/lib/as/right_as_interface.rb +698 -0
  11. data/lib/awsbase/benchmark_fix.rb +39 -0
  12. data/lib/awsbase/right_awsbase.rb +1343 -0
  13. data/lib/awsbase/support.rb +35 -0
  14. data/lib/awsbase/version.rb +9 -0
  15. data/lib/ec2/right_ec2.rb +541 -0
  16. data/lib/ec2/right_ec2_ebs.rb +481 -0
  17. data/lib/ec2/right_ec2_images.rb +444 -0
  18. data/lib/ec2/right_ec2_instances.rb +788 -0
  19. data/lib/ec2/right_ec2_monitoring.rb +70 -0
  20. data/lib/ec2/right_ec2_placement_groups.rb +108 -0
  21. data/lib/ec2/right_ec2_reserved_instances.rb +184 -0
  22. data/lib/ec2/right_ec2_security_groups.rb +491 -0
  23. data/lib/ec2/right_ec2_spot_instances.rb +422 -0
  24. data/lib/ec2/right_ec2_tags.rb +139 -0
  25. data/lib/ec2/right_ec2_vpc.rb +590 -0
  26. data/lib/ec2/right_ec2_vpc2.rb +381 -0
  27. data/lib/ec2/right_ec2_windows_mobility.rb +84 -0
  28. data/lib/elb/right_elb_interface.rb +573 -0
  29. data/lib/emr/right_emr_interface.rb +727 -0
  30. data/lib/iam/right_iam_access_keys.rb +71 -0
  31. data/lib/iam/right_iam_groups.rb +195 -0
  32. data/lib/iam/right_iam_interface.rb +341 -0
  33. data/lib/iam/right_iam_mfa_devices.rb +67 -0
  34. data/lib/iam/right_iam_users.rb +251 -0
  35. data/lib/rds/right_rds_interface.rb +1384 -0
  36. data/lib/right_aws.rb +86 -0
  37. data/lib/route_53/right_route_53_interface.rb +640 -0
  38. data/lib/s3/right_s3.rb +1138 -0
  39. data/lib/s3/right_s3_interface.rb +1278 -0
  40. data/lib/sdb/active_sdb.rb +1107 -0
  41. data/lib/sdb/right_sdb_interface.rb +762 -0
  42. data/lib/sns/right_sns_interface.rb +286 -0
  43. data/lib/sqs/right_sqs.rb +387 -0
  44. data/lib/sqs/right_sqs_gen2.rb +342 -0
  45. data/lib/sqs/right_sqs_gen2_interface.rb +523 -0
  46. data/lib/sqs/right_sqs_interface.rb +593 -0
  47. data/right_aws.gemspec +90 -0
  48. data/test/README.mdown +39 -0
  49. data/test/acf/test_helper.rb +2 -0
  50. data/test/acf/test_right_acf.rb +138 -0
  51. data/test/awsbase/test_helper.rb +2 -0
  52. data/test/awsbase/test_right_awsbase.rb +11 -0
  53. data/test/ec2/test_helper.rb +2 -0
  54. data/test/ec2/test_right_ec2.rb +107 -0
  55. data/test/elb/test_helper.rb +2 -0
  56. data/test/elb/test_right_elb.rb +43 -0
  57. data/test/http_connection.rb +87 -0
  58. data/test/rds/test_helper.rb +2 -0
  59. data/test/rds/test_right_rds.rb +120 -0
  60. data/test/route_53/fixtures/a_record.xml +18 -0
  61. data/test/route_53/fixtures/alias_record.xml +18 -0
  62. data/test/route_53/test_helper.rb +2 -0
  63. data/test/route_53/test_right_route_53.rb +141 -0
  64. data/test/s3/test_helper.rb +2 -0
  65. data/test/s3/test_right_s3.rb +528 -0
  66. data/test/s3/test_right_s3_stubbed.rb +97 -0
  67. data/test/sdb/test_active_sdb.rb +357 -0
  68. data/test/sdb/test_batch_put_attributes.rb +54 -0
  69. data/test/sdb/test_helper.rb +3 -0
  70. data/test/sdb/test_right_sdb.rb +253 -0
  71. data/test/sns/test_helper.rb +2 -0
  72. data/test/sns/test_right_sns.rb +153 -0
  73. data/test/sqs/test_helper.rb +2 -0
  74. data/test/sqs/test_right_sqs.rb +285 -0
  75. data/test/sqs/test_right_sqs_gen2.rb +264 -0
  76. data/test/test_credentials.rb +37 -0
  77. data/test/ts_right_aws.rb +15 -0
  78. metadata +257 -0
@@ -0,0 +1,39 @@
1
+ #
2
+ # Copyright (c) 2007-2008 RightScale Inc
3
+ #
4
+ # Permission is hereby granted, free of charge, to any person obtaining
5
+ # a copy of this software and associated documentation files (the
6
+ # "Software"), to deal in the Software without restriction, including
7
+ # without limitation the rights to use, copy, modify, merge, publish,
8
+ # distribute, sublicense, and/or sell copies of the Software, and to
9
+ # permit persons to whom the Software is furnished to do so, subject to
10
+ # the following conditions:
11
+ #
12
+ # The above copyright notice and this permission notice shall be
13
+ # included in all copies or substantial portions of the Software.
14
+ #
15
+ # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
16
+ # EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
17
+ # MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
18
+ # NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
19
+ # LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
20
+ # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
21
+ # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
22
+ #
23
+ #
24
+
25
+
26
+ # A hack because there's a bug in add! in Benchmark::Tms
27
+ module Benchmark #:nodoc:
28
+ class Tms #:nodoc:
29
+ def add!(&blk)
30
+ t = Benchmark::measure(&blk)
31
+ @utime = utime + t.utime
32
+ @stime = stime + t.stime
33
+ @cutime = cutime + t.cutime
34
+ @cstime = cstime + t.cstime
35
+ @real = real + t.real
36
+ self
37
+ end
38
+ end
39
+ end
@@ -0,0 +1,1343 @@
1
+ #
2
+ # Copyright (c) 2007-2008 RightScale Inc
3
+ #
4
+ # Permission is hereby granted, free of charge, to any person obtaining
5
+ # a copy of this software and associated documentation files (the
6
+ # "Software"), to deal in the Software without restriction, including
7
+ # without limitation the rights to use, copy, modify, merge, publish,
8
+ # distribute, sublicense, and/or sell copies of the Software, and to
9
+ # permit persons to whom the Software is furnished to do so, subject to
10
+ # the following conditions:
11
+ #
12
+ # The above copyright notice and this permission notice shall be
13
+ # included in all copies or substantial portions of the Software.
14
+ #
15
+ # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
16
+ # EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
17
+ # MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
18
+ # NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
19
+ # LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
20
+ # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
21
+ # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
22
+ #
23
+
24
+ # Test
25
+ module RightAws
26
+ require 'digest/md5'
27
+
28
+ class AwsUtils #:nodoc:
29
+ @@digest1 = OpenSSL::Digest::Digest.new("sha1")
30
+ @@digest256 = nil
31
+ if OpenSSL::OPENSSL_VERSION_NUMBER > 0x00908000
32
+ @@digest256 = OpenSSL::Digest::Digest.new("sha256") rescue nil # Some installation may not support sha256
33
+ end
34
+
35
+ def self.utc_iso8601(time)
36
+ if time.is_a?(Fixnum) then time = Time::at(time)
37
+ elsif time.is_a?(String) then time = Time::parse(time)
38
+ end
39
+ time.utc.strftime("%Y-%m-%dT%H:%M:%S.000Z")
40
+ end
41
+
42
+ def self.sign(aws_secret_access_key, auth_string)
43
+ Base64.encode64(OpenSSL::HMAC.digest(@@digest1, aws_secret_access_key, auth_string)).strip
44
+ end
45
+
46
+ # Calculates 'Content-MD5' header value for some content
47
+ def self.content_md5(content)
48
+ Base64.encode64(Digest::MD5::new.update(content).digest).strip
49
+ end
50
+
51
+ # Escape a string accordingly Amazon rulles
52
+ # http://docs.amazonwebservices.com/AmazonSimpleDB/2007-11-07/DeveloperGuide/index.html?REST_RESTAuth.html
53
+ def self.amz_escape(param)
54
+ param.to_s.gsub(/([^a-zA-Z0-9._~-]+)/n) do
55
+ '%' + $1.unpack('H2' * $1.size).join('%').upcase
56
+ end
57
+ end
58
+
59
+ def self.xml_escape(text) # :nodoc:
60
+ REXML::Text::normalize(text)
61
+ end
62
+
63
+ def self.xml_unescape(text) # :nodoc:
64
+ REXML::Text::unnormalize(text)
65
+ end
66
+
67
+ # Set a timestamp and a signature version
68
+ def self.fix_service_params(service_hash, signature)
69
+ service_hash["Timestamp"] ||= utc_iso8601(Time.now) unless service_hash["Expires"]
70
+ service_hash["SignatureVersion"] = signature
71
+ service_hash
72
+ end
73
+
74
+ # Signature Version 0
75
+ # A deprecated guy (should work till septemper 2009)
76
+ def self.sign_request_v0(aws_secret_access_key, service_hash)
77
+ fix_service_params(service_hash, '0')
78
+ string_to_sign = "#{service_hash['Action']}#{service_hash['Timestamp'] || service_hash['Expires']}"
79
+ service_hash['Signature'] = AwsUtils::sign(aws_secret_access_key, string_to_sign)
80
+ service_hash.to_a.collect{|key,val| "#{amz_escape(key)}=#{amz_escape(val.to_s)}" }.join("&")
81
+ end
82
+
83
+ # Signature Version 1
84
+ # Another deprecated guy (should work till septemper 2009)
85
+ def self.sign_request_v1(aws_secret_access_key, service_hash)
86
+ fix_service_params(service_hash, '1')
87
+ string_to_sign = service_hash.sort{|a,b| (a[0].to_s.downcase)<=>(b[0].to_s.downcase)}.to_s
88
+ service_hash['Signature'] = AwsUtils::sign(aws_secret_access_key, string_to_sign)
89
+ service_hash.to_a.collect{|key,val| "#{amz_escape(key)}=#{amz_escape(val.to_s)}" }.join("&")
90
+ end
91
+
92
+ # Signature Version 2
93
+ # EC2, SQS and SDB requests must be signed by this guy.
94
+ # See: http://docs.amazonwebservices.com/AmazonSimpleDB/2007-11-07/DeveloperGuide/index.html?REST_RESTAuth.html
95
+ # http://developer.amazonwebservices.com/connect/entry.jspa?externalID=1928
96
+ def self.sign_request_v2(aws_secret_access_key, service_hash, http_verb, host, uri)
97
+ fix_service_params(service_hash, '2')
98
+ # select a signing method (make an old openssl working with sha1)
99
+ # make 'HmacSHA256' to be a default one
100
+ service_hash['SignatureMethod'] = 'HmacSHA256' unless ['HmacSHA256', 'HmacSHA1'].include?(service_hash['SignatureMethod'])
101
+ service_hash['SignatureMethod'] = 'HmacSHA1' unless @@digest256
102
+ # select a digest
103
+ digest = (service_hash['SignatureMethod'] == 'HmacSHA256' ? @@digest256 : @@digest1)
104
+ # form string to sign
105
+ canonical_string = service_hash.keys.sort.map do |key|
106
+ "#{amz_escape(key)}=#{amz_escape(service_hash[key])}"
107
+ end.join('&')
108
+ string_to_sign = "#{http_verb.to_s.upcase}\n#{host.downcase}\n#{uri}\n#{canonical_string}"
109
+ # sign the string
110
+ signature = amz_escape(Base64.encode64(OpenSSL::HMAC.digest(digest, aws_secret_access_key, string_to_sign)).strip)
111
+ "#{canonical_string}&Signature=#{signature}"
112
+ end
113
+
114
+ # From Amazon's SQS Dev Guide, a brief description of how to escape:
115
+ # "URL encode the computed signature and other query parameters as specified in
116
+ # RFC1738, section 2.2. In addition, because the + character is interpreted as a blank space
117
+ # by Sun Java classes that perform URL decoding, make sure to encode the + character
118
+ # although it is not required by RFC1738."
119
+ # Avoid using CGI::escape to escape URIs.
120
+ # CGI::escape will escape characters in the protocol, host, and port
121
+ # sections of the URI. Only target chars in the query
122
+ # string should be escaped.
123
+ def self.URLencode(raw)
124
+ e = URI.escape(raw)
125
+ e.gsub(/\+/, "%2b")
126
+ end
127
+
128
+ def self.allow_only(allowed_keys, params)
129
+ bogus_args = []
130
+ params.keys.each {|p| bogus_args.push(p) unless allowed_keys.include?(p) }
131
+ raise AwsError.new("The following arguments were given but are not legal for the function call #{caller_method}: #{bogus_args.inspect}") if bogus_args.length > 0
132
+ end
133
+
134
+ def self.mandatory_arguments(required_args, params)
135
+ rargs = required_args.dup
136
+ params.keys.each {|p| rargs.delete(p)}
137
+ raise AwsError.new("The following mandatory arguments were not provided to #{caller_method}: #{rargs.inspect}") if rargs.length > 0
138
+ end
139
+
140
+ def self.caller_method
141
+ caller[1]=~/`(.*?)'/
142
+ $1
143
+ end
144
+
145
+ def self.split_items_and_params(array)
146
+ items = Array(array).flatten.compact
147
+ params = items.last.kind_of?(Hash) ? items.pop : {}
148
+ [items, params]
149
+ end
150
+
151
+ # Generates a token in format of:
152
+ # 1. "1dd8d4e4-db6b-11df-b31d-0025b37efad0 (if UUID gem is loaded)
153
+ # 2. "1287483761-855215-zSv2z-bWGj2-31M5t-ags9m" (if UUID gem is not loaded)
154
+ TOKEN_GENERATOR_CHARSET = ('a'..'z').to_a + ('A'..'Z').to_a + ('0'..'9').to_a
155
+ def self.generate_unique_token
156
+ time = Time.now
157
+ token = "%d-%06d" % [time.to_i, time.usec]
158
+ 4.times do
159
+ token << "-"
160
+ 5.times { token << TOKEN_GENERATOR_CHARSET[rand(TOKEN_GENERATOR_CHARSET.size)] }
161
+ end
162
+ token
163
+ end
164
+ end
165
+
166
+ class AwsBenchmarkingBlock #:nodoc:
167
+ attr_accessor :xml, :service
168
+ def initialize
169
+ # Benchmark::Tms instance for service (Ec2, S3, or SQS) access benchmarking.
170
+ @service = Benchmark::Tms.new()
171
+ # Benchmark::Tms instance for XML parsing benchmarking.
172
+ @xml = Benchmark::Tms.new()
173
+ end
174
+ end
175
+
176
+ class AwsNoChange < RuntimeError
177
+ end
178
+
179
+ class RightAwsBase
180
+
181
+ # Amazon HTTP Error handling
182
+
183
+ # Text, if found in an error message returned by AWS, indicates that this may be a transient
184
+ # error. Transient errors are automatically retried with exponential back-off.
185
+ AMAZON_PROBLEMS = [ 'internal service error',
186
+ 'is currently unavailable',
187
+ 'no response from',
188
+ 'Please try again',
189
+ 'InternalError',
190
+ 'Internal Server Error',
191
+ 'ServiceUnavailable', #from SQS docs
192
+ 'Unavailable',
193
+ 'This application is not currently available',
194
+ 'InsufficientInstanceCapacity'
195
+ ]
196
+ @@amazon_problems = AMAZON_PROBLEMS
197
+ # Returns a list of Amazon service responses which are known to be transient problems.
198
+ # We have to re-request if we get any of them, because the problem will probably disappear.
199
+ # By default this method returns the same value as the AMAZON_PROBLEMS const.
200
+ def self.amazon_problems
201
+ @@amazon_problems
202
+ end
203
+
204
+ # Sets the list of Amazon side problems. Use in conjunction with the
205
+ # getter to append problems.
206
+ def self.amazon_problems=(problems_list)
207
+ @@amazon_problems = problems_list
208
+ end
209
+
210
+ # Raise an exception if a timeout occures while an API call is in progress.
211
+ # This helps to avoid a duplicate resources creation when Amazon hangs for some time and
212
+ # RightHttpConnection is forced to use retries to get a response from it.
213
+ #
214
+ # If an API call action is in the list then no attempts to retry are performed.
215
+ #
216
+ RAISE_ON_TIMEOUT_ON_ACTIONS = %w{
217
+ AllocateAddress
218
+ CreateSnapshot
219
+ CreateVolume
220
+ PurchaseReservedInstancesOffering
221
+ RequestSpotInstances
222
+ RunInstances
223
+ }
224
+ @@raise_on_timeout_on_actions = RAISE_ON_TIMEOUT_ON_ACTIONS.dup
225
+
226
+ def self.raise_on_timeout_on_actions
227
+ @@raise_on_timeout_on_actions
228
+ end
229
+
230
+ def self.raise_on_timeout_on_actions=(actions_list)
231
+ @@raise_on_timeout_on_actions = actions_list
232
+ end
233
+
234
+ end
235
+
236
+ module RightAwsBaseInterface
237
+ DEFAULT_SIGNATURE_VERSION = '2'
238
+
239
+ @@caching = false
240
+ def self.caching
241
+ @@caching
242
+ end
243
+ def self.caching=(caching)
244
+ @@caching = caching
245
+ end
246
+
247
+ # Current aws_access_key_id
248
+ attr_reader :aws_access_key_id
249
+ # Current aws_secret_access_key
250
+ attr_reader :aws_secret_access_key
251
+ # Last HTTP request object
252
+ attr_reader :last_request
253
+ # Last HTTP response object
254
+ attr_reader :last_response
255
+ # Last AWS errors list (used by AWSErrorHandler)
256
+ attr_accessor :last_errors
257
+ # Last AWS request id (used by AWSErrorHandler)
258
+ attr_accessor :last_request_id
259
+ # Logger object
260
+ attr_accessor :logger
261
+ # Initial params hash
262
+ attr_accessor :params
263
+ # RightHttpConnection instance
264
+ attr_reader :connection
265
+ # Cache
266
+ attr_reader :cache
267
+ # Signature version (all services except s3)
268
+ attr_reader :signature_version
269
+
270
+ def init(service_info, aws_access_key_id, aws_secret_access_key, params={}) #:nodoc:
271
+ @params = params
272
+ # If one defines EC2_URL he may forget to use a single slash as an "empty service" path.
273
+ # Amazon does not like this therefore add this bad boy if he is missing...
274
+ service_info[:default_service] = '/' if service_info[:default_service].right_blank?
275
+ raise AwsError.new("AWS access keys are required to operate on #{service_info[:name]}") \
276
+ if aws_access_key_id.right_blank? || aws_secret_access_key.right_blank?
277
+ @aws_access_key_id = aws_access_key_id
278
+ @aws_secret_access_key = aws_secret_access_key
279
+ # if the endpoint was explicitly defined - then use it
280
+ if @params[:endpoint_url]
281
+ uri = URI.parse(@params[:endpoint_url])
282
+ @params[:server] = uri.host
283
+ @params[:port] = uri.port
284
+ @params[:service] = uri.path
285
+ @params[:protocol] = uri.scheme
286
+ # make sure the 'service' path is not empty
287
+ @params[:service] = service_info[:default_service] if @params[:service].right_blank?
288
+ @params[:region] = nil
289
+ default_port = uri.default_port
290
+ else
291
+ @params[:server] ||= service_info[:default_host]
292
+ @params[:server] = "#{@params[:region]}.#{@params[:server]}" if @params[:region]
293
+ @params[:port] ||= service_info[:default_port]
294
+ @params[:service] ||= service_info[:default_service]
295
+ @params[:protocol] ||= service_info[:default_protocol]
296
+ default_port = @params[:protocol] == 'https' ? 443 : 80
297
+ end
298
+ # build a host name to sign
299
+ @params[:host_to_sign] = @params[:server].dup
300
+ @params[:host_to_sign] << ":#{@params[:port]}" unless default_port == @params[:port].to_i
301
+ # a set of options to be passed to RightHttpConnection object
302
+ @params[:connection_options] = {} unless @params[:connection_options].is_a?(Hash)
303
+ @with_connection_options = {}
304
+ @params[:connections] ||= :shared # || :dedicated
305
+ @params[:max_connections] ||= 10
306
+ @params[:connection_lifetime] ||= 20*60
307
+ @params[:api_version] ||= service_info[:default_api_version]
308
+ @logger = @params[:logger]
309
+ @logger = ::Rails.logger if !@logger && defined?(::Rails) && ::Rails.respond_to?(:logger)
310
+ @logger = RAILS_DEFAULT_LOGGER if !@logger && defined?(RAILS_DEFAULT_LOGGER)
311
+ @logger = Logger.new(STDOUT) if !@logger
312
+ @logger.info "New #{self.class.name} using #{@params[:connections]} connections mode"
313
+ @error_handler = nil
314
+ @cache = {}
315
+ @signature_version = (params[:signature_version] || DEFAULT_SIGNATURE_VERSION).to_s
316
+ end
317
+
318
+ def signed_service_params(aws_secret_access_key, service_hash, http_verb=nil, host=nil, service=nil )
319
+ case signature_version.to_s
320
+ when '0' then AwsUtils::sign_request_v0(aws_secret_access_key, service_hash)
321
+ when '1' then AwsUtils::sign_request_v1(aws_secret_access_key, service_hash)
322
+ when '2' then AwsUtils::sign_request_v2(aws_secret_access_key, service_hash, http_verb, host, service)
323
+ else raise AwsError.new("Unknown signature version (#{signature_version.to_s}) requested")
324
+ end
325
+ end
326
+
327
+ # Returns +true+ if the describe_xxx responses are being cached
328
+ def caching?
329
+ @params.key?(:cache) ? @params[:cache] : @@caching
330
+ end
331
+
332
+ # Check if the aws function response hits the cache or not.
333
+ # If the cache hits:
334
+ # - raises an +AwsNoChange+ exception if +do_raise+ == +:raise+.
335
+ # - returnes parsed response from the cache if it exists or +true+ otherwise.
336
+ # If the cache miss or the caching is off then returns +false+.
337
+ def cache_hits?(function, response, do_raise=:raise)
338
+ result = false
339
+ if caching?
340
+ function = function.to_sym
341
+ # get rid of requestId (this bad boy was added for API 2008-08-08+ and it is uniq for every response)
342
+ # feb 04, 2009 (load balancer uses 'RequestId' hence use 'i' modifier to hit it also)
343
+ response = response.sub(%r{<requestId>.+?</requestId>}i, '')
344
+ # this should work for both ruby 1.8.x and 1.9.x
345
+ response_md5 = Digest::MD5::new.update(response).to_s
346
+ # check for changes
347
+ unless @cache[function] && @cache[function][:response_md5] == response_md5
348
+ # well, the response is new, reset cache data
349
+ update_cache(function, {:response_md5 => response_md5,
350
+ :timestamp => Time.now,
351
+ :hits => 0,
352
+ :parsed => nil})
353
+ else
354
+ # aha, cache hits, update the data and throw an exception if needed
355
+ @cache[function][:hits] += 1
356
+ if do_raise == :raise
357
+ raise(AwsNoChange, "Cache hit: #{function} response has not changed since "+
358
+ "#{@cache[function][:timestamp].strftime('%Y-%m-%d %H:%M:%S')}, "+
359
+ "hits: #{@cache[function][:hits]}.")
360
+ else
361
+ result = @cache[function][:parsed] || true
362
+ end
363
+ end
364
+ end
365
+ result
366
+ end
367
+
368
+ def update_cache(function, hash)
369
+ (@cache[function.to_sym] ||= {}).merge!(hash) if caching?
370
+ end
371
+
372
+ def on_exception(options={:raise=>true, :log=>true}) # :nodoc:
373
+ raise if $!.is_a?(AwsNoChange)
374
+ AwsError::on_aws_exception(self, options)
375
+ end
376
+
377
+ #----------------------------
378
+ # HTTP Connections handling
379
+ #----------------------------
380
+
381
+ def get_server_url(request) # :nodoc:
382
+ "#{request[:protocol]}://#{request[:server]}:#{request[:port]}"
383
+ end
384
+
385
+ def get_connections_storage(aws_service) # :nodoc:
386
+ case @params[:connections].to_s
387
+ when 'dedicated' then @connections_storage ||= {}
388
+ else Thread.current[aws_service] ||= {}
389
+ end
390
+ end
391
+
392
+ def destroy_connection(request, reason) # :nodoc:
393
+ connections = get_connections_storage(request[:aws_service])
394
+ server_url = get_server_url(request)
395
+ if connections[server_url]
396
+ connections[server_url][:connection].finish(reason)
397
+ connections.delete(server_url)
398
+ end
399
+ end
400
+
401
+ # Expire the connection if it has expired.
402
+ def get_connection(request) # :nodoc:
403
+ server_url = get_server_url(request)
404
+ connection_storage = get_connections_storage(request[:aws_service])
405
+ life_time_scratch = Time.now-@params[:connection_lifetime]
406
+ # Delete out-of-dated connections
407
+ connections_in_list = 0
408
+ connection_storage.to_a.sort{|conn1, conn2| conn2[1][:last_used_at] <=> conn1[1][:last_used_at]}.each do |serv_url, conn_opts|
409
+ if @params[:max_connections] <= connections_in_list
410
+ conn_opts[:connection].finish('out-of-limit')
411
+ connection_storage.delete(server_url)
412
+ elsif conn_opts[:last_used_at] < life_time_scratch
413
+ conn_opts[:connection].finish('out-of-date')
414
+ connection_storage.delete(server_url)
415
+ else
416
+ connections_in_list += 1
417
+ end
418
+ end
419
+ connection = (connection_storage[server_url] ||= {})
420
+ connection[:last_used_at] = Time.now
421
+ connection[:connection] ||= Rightscale::HttpConnection.new(:exception => RightAws::AwsError, :logger => @logger)
422
+ end
423
+
424
+ #----------------------------
425
+ # HTTP Requests handling
426
+ #----------------------------
427
+
428
+ # ACF, AMS, EC2, LBS and SDB uses this guy
429
+ # SQS and S3 use their own methods
430
+ def generate_request_impl(verb, action, options={}, custom_options={}) #:nodoc:
431
+ # Form a valid http verb: 'GET' or 'POST' (all the other are not supported now)
432
+ http_verb = verb.to_s.upcase
433
+ # remove empty keys from request options
434
+ options.delete_if { |key, value| value.nil? }
435
+ # prepare service data
436
+ service_hash = {"Action" => action,
437
+ "AWSAccessKeyId" => @aws_access_key_id,
438
+ "Version" => custom_options[:api_version] || @params[:api_version] }
439
+ service_hash.merge!(options)
440
+ # Sign request options
441
+ service_params = signed_service_params(@aws_secret_access_key, service_hash, http_verb, @params[:host_to_sign], @params[:service])
442
+ # Use POST if the length of the query string is too large
443
+ # see http://docs.amazonwebservices.com/AmazonSimpleDB/2007-11-07/DeveloperGuide/MakingRESTRequests.html
444
+ if http_verb != 'POST' && service_params.size > 2000
445
+ http_verb = 'POST'
446
+ if signature_version == '2'
447
+ service_params = signed_service_params(@aws_secret_access_key, service_hash, http_verb, @params[:host_to_sign], @params[:service])
448
+ end
449
+ end
450
+ # create a request
451
+ case http_verb
452
+ when 'GET'
453
+ request = Net::HTTP::Get.new("#{@params[:service]}?#{service_params}")
454
+ when 'POST'
455
+ request = Net::HTTP::Post.new(@params[:service])
456
+ request.body = service_params
457
+ request['Content-Type'] = 'application/x-www-form-urlencoded; charset=utf-8'
458
+ else
459
+ raise "Unsupported HTTP verb #{verb.inspect}!"
460
+ end
461
+ # prepare output hash
462
+ request_hash = { :request => request,
463
+ :server => @params[:server],
464
+ :port => @params[:port],
465
+ :protocol => @params[:protocol] }
466
+ request_hash.merge!(@params[:connection_options])
467
+ request_hash.merge!(@with_connection_options)
468
+
469
+ # If an action is marked as "non-retryable" and there was no :raise_on_timeout option set
470
+ # explicitly then do set that option
471
+ if Array(RightAwsBase::raise_on_timeout_on_actions).include?(action) && !request_hash.has_key?(:raise_on_timeout)
472
+ request_hash.merge!(:raise_on_timeout => true)
473
+ end
474
+
475
+ request_hash
476
+ end
477
+
478
+ # All services uses this guy.
479
+ def request_info_impl(aws_service, benchblock, request, parser, &block) #:nodoc:
480
+ request[:aws_service] = aws_service
481
+ @connection = get_connection(request)
482
+ @last_request = request[:request]
483
+ @last_response = nil
484
+ response = nil
485
+ blockexception = nil
486
+
487
+ if(block != nil)
488
+ # TRB 9/17/07 Careful - because we are passing in blocks, we get a situation where
489
+ # an exception may get thrown in the block body (which is high-level
490
+ # code either here or in the application) but gets caught in the
491
+ # low-level code of HttpConnection. The solution is not to let any
492
+ # exception escape the block that we pass to HttpConnection::request.
493
+ # Exceptions can originate from code directly in the block, or from user
494
+ # code called in the other block which is passed to response.read_body.
495
+ benchblock.service.add! do
496
+ begin
497
+ responsehdr = @connection.request(request) do |response|
498
+ #########
499
+ begin
500
+ @last_response = response
501
+ if response.is_a?(Net::HTTPSuccess)
502
+ @error_handler = nil
503
+ response.read_body(&block)
504
+ else
505
+ @error_handler = AWSErrorHandler.new(self, parser, :errors_list => self.class.amazon_problems) unless @error_handler
506
+ check_result = @error_handler.check(request)
507
+ if check_result
508
+ @error_handler = nil
509
+ return check_result
510
+ end
511
+ raise AwsError.new(@last_errors, @last_response.code, @last_request_id)
512
+ end
513
+ rescue Exception => e
514
+ blockexception = e
515
+ end
516
+ end
517
+ rescue Exception => e
518
+ # Kill a connection if we run into a low level connection error
519
+ destroy_connection(request, "error: #{e.message}")
520
+ raise e
521
+ end
522
+ #########
523
+
524
+ #OK, now we are out of the block passed to the lower level
525
+ if(blockexception)
526
+ raise blockexception
527
+ end
528
+ benchblock.xml.add! do
529
+ parser.parse(responsehdr)
530
+ end
531
+ return parser.result
532
+ end
533
+ else
534
+ benchblock.service.add! do
535
+ begin
536
+ response = @connection.request(request)
537
+ rescue Exception => e
538
+ # Kill a connection if we run into a low level connection error
539
+ destroy_connection(request, "error: #{e.message}")
540
+ raise e
541
+ end
542
+ end
543
+ # check response for errors...
544
+ @last_response = response
545
+ if response.is_a?(Net::HTTPSuccess)
546
+ @error_handler = nil
547
+ benchblock.xml.add! { parser.parse(response) }
548
+ return parser.result
549
+ else
550
+ @error_handler = AWSErrorHandler.new(self, parser, :errors_list => self.class.amazon_problems) unless @error_handler
551
+ check_result = @error_handler.check(request)
552
+ if check_result
553
+ @error_handler = nil
554
+ return check_result
555
+ end
556
+ raise AwsError.new(@last_errors, @last_response.code, @last_request_id)
557
+ end
558
+ end
559
+ rescue
560
+ @error_handler = nil
561
+ raise
562
+ end
563
+
564
+ def request_cache_or_info(method, link, parser_class, benchblock, use_cache=true, &block) #:nodoc:
565
+ # We do not want to break the logic of parsing hence will use a dummy parser to process all the standard
566
+ # steps (errors checking etc). The dummy parser does nothig - just returns back the params it received.
567
+ # If the caching is enabled and hit then throw AwsNoChange.
568
+ # P.S. caching works for the whole images list only! (when the list param is blank)
569
+ # check cache
570
+ response, params = request_info(link, RightDummyParser.new)
571
+ cache_hits?(method.to_sym, response.body) if use_cache
572
+ parser = parser_class.new(:logger => @logger)
573
+ benchblock.xml.add!{ parser.parse(response, params) }
574
+ result = block ? block.call(parser) : parser.result
575
+ # update parsed data
576
+ update_cache(method.to_sym, :parsed => result) if use_cache
577
+ result
578
+ end
579
+
580
+ # Returns Amazons request ID for the latest request
581
+ def last_request_id
582
+ @last_response && @last_response.body.to_s[%r{<requestId>(.+?)</requestId>}i] && $1
583
+ end
584
+
585
+ # Incrementally lists something.
586
+ def incrementally_list_items(action, parser_class, params={}, &block) # :nodoc:
587
+ params = params.dup
588
+ params['MaxItems'] = params.delete(:max_items) if params[:max_items]
589
+ params['Marker'] = params.delete(:marker) if params[:marker]
590
+ last_response = nil
591
+ loop do
592
+ last_response = request_info( generate_request(action, params), parser_class.new(:logger => @logger))
593
+ params['Marker'] = last_response[:marker]
594
+ break unless block && block.call(last_response) && !last_response[:marker].right_blank?
595
+ end
596
+ last_response
597
+ end
598
+
599
+ # Format array of items into Amazons handy hash ('?' is a place holder):
600
+ # Options:
601
+ # :default => "something" : Set a value to "something" when it is nil
602
+ # :default => :skip_nils : Skip nil values
603
+ #
604
+ # amazonize_list('Item', ['a', 'b', 'c']) =>
605
+ # { 'Item.1' => 'a', 'Item.2' => 'b', 'Item.3' => 'c' }
606
+ #
607
+ # amazonize_list('Item.?.instance', ['a', 'c']) #=>
608
+ # { 'Item.1.instance' => 'a', 'Item.2.instance' => 'c' }
609
+ #
610
+ # amazonize_list(['Item.?.Name', 'Item.?.Value'], {'A' => 'a', 'B' => 'b'}) #=>
611
+ # { 'Item.1.Name' => 'A', 'Item.1.Value' => 'a',
612
+ # 'Item.2.Name' => 'B', 'Item.2.Value' => 'b' }
613
+ #
614
+ # amazonize_list(['Item.?.Name', 'Item.?.Value'], [['A','a'], ['B','b']]) #=>
615
+ # { 'Item.1.Name' => 'A', 'Item.1.Value' => 'a',
616
+ # 'Item.2.Name' => 'B', 'Item.2.Value' => 'b' }
617
+ #
618
+ # amazonize_list(['Filter.?.Key', 'Filter.?.Value.?'], {'A' => ['aa','ab'], 'B' => ['ba','bb']}) #=>
619
+ # amazonize_list(['Filter.?.Key', 'Filter.?.Value.?'], [['A',['aa','ab']], ['B',['ba','bb']]]) #=>
620
+ # {"Filter.1.Key"=>"A",
621
+ # "Filter.1.Value.1"=>"aa",
622
+ # "Filter.1.Value.2"=>"ab",
623
+ # "Filter.2.Key"=>"B",
624
+ # "Filter.2.Value.1"=>"ba",
625
+ # "Filter.2.Value.2"=>"bb"}
626
+ def amazonize_list(masks, list, options={}) #:nodoc:
627
+ groups = {}
628
+ list_idx = options[:index] || 1
629
+ Array(list).each do |list_item|
630
+ Array(masks).each_with_index do |mask, mask_idx|
631
+ key = mask[/\?/] ? mask.dup : mask.dup + '.?'
632
+ key.sub!('?', list_idx.to_s)
633
+ value = Array(list_item)[mask_idx]
634
+ if value.is_a?(Array)
635
+ groups.merge!(amazonize_list(key, value, options))
636
+ else
637
+ if value.nil?
638
+ next if options[:default] == :skip_nils
639
+ value = options[:default]
640
+ end
641
+ # Hack to avoid having unhandled '?' in keys : do replace them all with '1':
642
+ # bad: ec2.amazonize_list(['Filter.?.Key', 'Filter.?.Value.?'], { a: => :b }) => {"Filter.1.Key"=>:a, "Filter.1.Value.?"=>1}
643
+ # good: ec2.amazonize_list(['Filter.?.Key', 'Filter.?.Value.?'], { a: => :b }) => {"Filter.1.Key"=>:a, "Filter.1.Value.1"=>1}
644
+ key.gsub!('?', '1')
645
+ groups[key] = value
646
+ end
647
+ end
648
+ list_idx += 1
649
+ end
650
+ groups
651
+ end
652
+
653
+ BLOCK_DEVICE_KEY_MAPPING = { # :nodoc:
654
+ :device_name => 'DeviceName',
655
+ :virtual_name => 'VirtualName',
656
+ :no_device => 'NoDevice',
657
+ :ebs_snapshot_id => 'Ebs.SnapshotId',
658
+ :ebs_volume_size => 'Ebs.VolumeSize',
659
+ :ebs_delete_on_termination => 'Ebs.DeleteOnTermination' }
660
+
661
+ def amazonize_block_device_mappings(block_device_mappings, key = 'BlockDeviceMapping') # :nodoc:
662
+ result = {}
663
+ unless block_device_mappings.right_blank?
664
+ block_device_mappings = [block_device_mappings] unless block_device_mappings.is_a?(Array)
665
+ block_device_mappings.each_with_index do |b, idx|
666
+ BLOCK_DEVICE_KEY_MAPPING.each do |local_name, remote_name|
667
+ value = b[local_name]
668
+ case local_name
669
+ when :no_device then value = value ? '' : nil # allow to pass :no_device as boolean
670
+ end
671
+ result["#{key}.#{idx+1}.#{remote_name}"] = value unless value.nil?
672
+ end
673
+ end
674
+ end
675
+ result
676
+ end
677
+
678
+ # Build API request keys set.
679
+ #
680
+ # Options is a hash, expectations is a set of keys [and rules] how to represent options.
681
+ # Mappings is an Array (may include hashes) or a Hash.
682
+ #
683
+ # Example:
684
+ #
685
+ # options = { :valid_from => Time.now - 10,
686
+ # :instance_count => 3,
687
+ # :image_id => 'ami-08f41161',
688
+ # :spot_price => 0.059,
689
+ # :instance_type => 'c1.medium',
690
+ # :instance_count => 1,
691
+ # :key_name => 'tim',
692
+ # :availability_zone => 'us-east-1a',
693
+ # :monitoring_enabled => true,
694
+ # :launch_group => 'lg1',
695
+ # :availability_zone_group => 'azg1',
696
+ # :groups => ['a', 'b', 'c'],
697
+ # :group_ids => 'sg-1',
698
+ # :user_data => 'konstantin',
699
+ # :block_device_mappings => [ { :device_name => '/dev/sdk',
700
+ # :ebs_snapshot_id => 'snap-145cbc7d',
701
+ # :ebs_delete_on_termination => true,
702
+ # :ebs_volume_size => 3,
703
+ # :virtual_name => 'ephemeral2' }]}
704
+ # mappings = { :spot_price,
705
+ # :availability_zone_group,
706
+ # :launch_group,
707
+ # :type,
708
+ # :instance_count,
709
+ # :image_id => 'LaunchSpecification.ImageId',
710
+ # :instance_type => 'LaunchSpecification.InstanceType',
711
+ # :key_name => 'LaunchSpecification.KeyName',
712
+ # :addressing_type => 'LaunchSpecification.AddressingType',
713
+ # :kernel_id => 'LaunchSpecification.KernelId',
714
+ # :ramdisk_id => 'LaunchSpecification.RamdiskId',
715
+ # :subnet_id => 'LaunchSpecification.SubnetId',
716
+ # :availability_zone => 'LaunchSpecification.Placement.AvailabilityZone',
717
+ # :monitoring_enabled => 'LaunchSpecification.Monitoring.Enabled',
718
+ # :valid_from => { :value => Proc.new { !options[:valid_from].right_blank? && AwsUtils::utc_iso8601(options[:valid_from]) }},
719
+ # :valid_until => { :value => Proc.new { !options[:valid_until].right_blank? && AwsUtils::utc_iso8601(options[:valid_until]) }},
720
+ # :user_data => { :name => 'LaunchSpecification.UserData',
721
+ # :value => Proc.new { !options[:user_data].right_blank? && Base64.encode64(options[:user_data]).delete("\n") }},
722
+ # :groups => { :amazonize_list => 'LaunchSpecification.SecurityGroup'},
723
+ # :group_ids => { :amazonize_list => 'LaunchSpecification.SecurityGroupId'},
724
+ # :block_device_mappings => { :amazonize_bdm => 'LaunchSpecification.BlockDeviceMapping'})
725
+ #
726
+ # map_api_keys_and_values( options, mappings) #=>
727
+ # {"LaunchSpecification.BlockDeviceMapping.1.Ebs.DeleteOnTermination" => true,
728
+ # "LaunchSpecification.BlockDeviceMapping.1.VirtualName" => "ephemeral2",
729
+ # "LaunchSpecification.BlockDeviceMapping.1.Ebs.VolumeSize" => 3,
730
+ # "LaunchSpecification.BlockDeviceMapping.1.Ebs.SnapshotId" => "snap-145cbc7d",
731
+ # "LaunchSpecification.BlockDeviceMapping.1.DeviceName" => "/dev/sdk",
732
+ # "LaunchSpecification.SecurityGroupId.1" => "sg-1",
733
+ # "LaunchSpecification.InstanceType" => "c1.medium",
734
+ # "LaunchSpecification.KeyName" => "tim",
735
+ # "LaunchSpecification.ImageId" => "ami-08f41161",
736
+ # "LaunchSpecification.SecurityGroup.1" => "a",
737
+ # "LaunchSpecification.SecurityGroup.2" => "b",
738
+ # "LaunchSpecification.SecurityGroup.3" => "c",
739
+ # "LaunchSpecification.Placement.AvailabilityZone" => "us-east-1a",
740
+ # "LaunchSpecification.Monitoring.Enabled" => true,
741
+ # "LaunchGroup" => "lg1",
742
+ # "InstanceCount" => 1,
743
+ # "SpotPrice" => 0.059,
744
+ # "AvailabilityZoneGroup" => "azg1",
745
+ # "ValidFrom" => "2011-06-30T08:06:30.000Z",
746
+ # "LaunchSpecification.UserData" => "a29uc3RhbnRpbg=="}
747
+ #
748
+ def map_api_keys_and_values(options, *mappings) # :nodoc:
749
+ result = {}
750
+ vars = {}
751
+ # Fix inputs and make them all to be hashes
752
+ mappings.flatten.each do |mapping|
753
+ unless mapping.is_a?(Hash)
754
+ # mapping is just a :key_name
755
+ mapping = { mapping => { :name => mapping.to_s.right_camelize, :value => options[mapping] }}
756
+ else
757
+ mapping.each do |local_key, api_opts|
758
+ unless api_opts.is_a?(Hash)
759
+ # mapping is a { :key_name => 'ApiKeyName' }
760
+ mapping[local_key] = { :name => api_opts.to_s, :value => options[local_key]}
761
+ else
762
+ # mapping is a { :key_name => { :name => 'ApiKeyName', :value => 'Value', ... etc} }
763
+ api_opts[:name] = local_key.to_s.right_camelize if (api_opts.keys & [:name, :amazonize_list, :amazonize_bdm]).right_blank?
764
+ api_opts[:value] = options[local_key] unless api_opts.has_key?(:value)
765
+ end
766
+ end
767
+ end
768
+ vars.merge! mapping
769
+ end
770
+ # Build API keys set
771
+ # vars now is a Hash:
772
+ # { :key1 => { :name => 'ApiKey1', :value => 'BlahBlah'},
773
+ # :key2 => { :amazonize_list => 'ApiKey2.?', :value => [1, ...] },
774
+ # :key3 => { :amazonize_bdm => 'BDM', :value => [{..}, ...] }, ... }
775
+ #
776
+ vars.each do |local_key, api_opts|
777
+ if api_opts[:amazonize_list]
778
+ result.merge!(amazonize_list( api_opts[:amazonize_list], api_opts[:value] )) unless api_opts[:value].right_blank?
779
+ elsif api_opts[:amazonize_bdm]
780
+ result.merge!(amazonize_block_device_mappings( api_opts[:value], api_opts[:amazonize_bdm] )) unless api_opts[:value].right_blank?
781
+ else
782
+ api_key = api_opts[:name]
783
+ value = api_opts[:value]
784
+ value = value.call if value.is_a?(Proc)
785
+ next if value.right_blank?
786
+ result[api_key] = value
787
+ end
788
+ end
789
+ #
790
+ result
791
+ end
792
+
793
+ # Transform a hash of parameters into a hash suitable for sending
794
+ # to Amazon using a key mapping.
795
+ #
796
+ # amazonize_hash_with_key_mapping('Group.Filter',
797
+ # {:some_param => 'SomeParam'},
798
+ # {:some_param => 'value'}) #=> {'Group.Filter.SomeParam' => 'value'}
799
+ #
800
+ def amazonize_hash_with_key_mapping(key, mapping, hash, options={})
801
+ result = {}
802
+ unless hash.right_blank?
803
+ mapping.each do |local_name, remote_name|
804
+ value = hash[local_name]
805
+ next if value.nil?
806
+ result["#{key}.#{remote_name}"] = value
807
+ end
808
+ end
809
+ result
810
+ end
811
+
812
+ # Transform a list of hashes of parameters into a hash suitable for sending
813
+ # to Amazon using a key mapping.
814
+ #
815
+ # amazonize_list_with_key_mapping('Group.Filter',
816
+ # [{:some_param => 'SomeParam'}, {:some_param => 'SomeParam'}],
817
+ # {:some_param => 'value'}) #=>
818
+ # {'Group.Filter.1.SomeParam' => 'value',
819
+ # 'Group.Filter.2.SomeParam' => 'value'}
820
+ #
821
+ def amazonize_list_with_key_mapping(key, mapping, list, options={})
822
+ result = {}
823
+ unless list.right_blank?
824
+ list.each_with_index do |item, index|
825
+ mapping.each do |local_name, remote_name|
826
+ value = item[local_name]
827
+ next if value.nil?
828
+ result["#{key}.#{index+1}.#{remote_name}"] = value
829
+ end
830
+ end
831
+ end
832
+ end
833
+
834
+ # Execute a block of code with custom set of settings for right_http_connection.
835
+ # Accepts next options (see Rightscale::HttpConnection for explanation):
836
+ # :raise_on_timeout
837
+ # :http_connection_retry_count
838
+ # :http_connection_open_timeout
839
+ # :http_connection_read_timeout
840
+ # :http_connection_retry_delay
841
+ # :user_agent
842
+ # :exception
843
+ #
844
+ # Example #1:
845
+ #
846
+ # # Try to create a snapshot but stop with exception if timeout is received
847
+ # # to avoid having a duplicate API calls that create duplicate snapshots.
848
+ # ec2 = Rightscale::Ec2::new(aws_access_key_id, aws_secret_access_key)
849
+ # ec2.with_connection_options(:raise_on_timeout => true) do
850
+ # ec2.create_snapshot('vol-898a6fe0', 'KD: WooHoo!!')
851
+ # end
852
+ #
853
+ # Example #2:
854
+ #
855
+ # # Opposite case when the setting is global:
856
+ # @ec2 = Rightscale::Ec2::new(aws_access_key_id, aws_secret_access_key,
857
+ # :connection_options => { :raise_on_timeout => true })
858
+ # # Create an SSHKey but do tries on timeout
859
+ # ec2.with_connection_options(:raise_on_timeout => false) do
860
+ # new_key = ec2.create_key_pair('my_test_key')
861
+ # end
862
+ #
863
+ # Example #3:
864
+ #
865
+ # # Global settings (HttpConnection level):
866
+ # Rightscale::HttpConnection::params[:http_connection_open_timeout] = 5
867
+ # Rightscale::HttpConnection::params[:http_connection_read_timeout] = 250
868
+ # Rightscale::HttpConnection::params[:http_connection_retry_count] = 2
869
+ #
870
+ # # Local setings (RightAws level)
871
+ # ec2 = Rightscale::Ec2::new(AWS_ID, AWS_KEY,
872
+ # :region => 'us-east-1',
873
+ # :connection_options => {
874
+ # :http_connection_read_timeout => 2,
875
+ # :http_connection_retry_count => 5,
876
+ # :user_agent => 'Mozilla 4.0'
877
+ # })
878
+ #
879
+ # # Custom settings (API call level)
880
+ # ec2.with_connection_options(:raise_on_timeout => true,
881
+ # :http_connection_read_timeout => 10,
882
+ # :user_agent => '') do
883
+ # pp ec2.describe_images
884
+ # end
885
+ #
886
+ def with_connection_options(options, &block)
887
+ @with_connection_options = options
888
+ block.call self
889
+ ensure
890
+ @with_connection_options = {}
891
+ end
892
+ end
893
+
894
+
895
+ # Exception class to signal any Amazon errors. All errors occuring during calls to Amazon's
896
+ # web services raise this type of error.
897
+ # Attribute inherited by RuntimeError:
898
+ # message - the text of the error, generally as returned by AWS in its XML response.
899
+ class AwsError < RuntimeError
900
+
901
+ # either an array of errors where each item is itself an array of [code, message]),
902
+ # or an error string if the error was raised manually, as in <tt>AwsError.new('err_text')</tt>
903
+ attr_reader :errors
904
+
905
+ # Request id (if exists)
906
+ attr_reader :request_id
907
+
908
+ # Response HTTP error code
909
+ attr_reader :http_code
910
+
911
+ def initialize(errors=nil, http_code=nil, request_id=nil)
912
+ @errors = errors
913
+ @request_id = request_id
914
+ @http_code = http_code
915
+ super(@errors.is_a?(Array) ? @errors.map{|code, msg| "#{code}: #{msg}"}.join("; ") : @errors.to_s)
916
+ end
917
+
918
+ # Does any of the error messages include the regexp +pattern+?
919
+ # Used to determine whether to retry request.
920
+ def include?(pattern)
921
+ if @errors.is_a?(Array)
922
+ @errors.each{ |code, msg| return true if code =~ pattern }
923
+ else
924
+ return true if @errors_str =~ pattern
925
+ end
926
+ false
927
+ end
928
+
929
+ # Generic handler for AwsErrors. +aws+ is the RightAws::S3, RightAws::EC2, or RightAws::SQS
930
+ # object that caused the exception (it must provide last_request and last_response). Supported
931
+ # boolean options are:
932
+ # * <tt>:log</tt> print a message into the log using aws.logger to access the Logger
933
+ # * <tt>:puts</tt> do a "puts" of the error
934
+ # * <tt>:raise</tt> re-raise the error after logging
935
+ def self.on_aws_exception(aws, options={:raise=>true, :log=>true})
936
+ # Only log & notify if not user error
937
+ if !options[:raise] || system_error?($!)
938
+ error_text = "#{$!.inspect}\n#{$@}.join('\n')}"
939
+ puts error_text if options[:puts]
940
+ # Log the error
941
+ if options[:log]
942
+ request = aws.last_request ? aws.last_request.path : '-none-'
943
+ response = aws.last_response ? "#{aws.last_response.code} -- #{aws.last_response.message} -- #{aws.last_response.body}" : '-none-'
944
+ aws.logger.error error_text
945
+ aws.logger.error "Request was: #{request}"
946
+ aws.logger.error "Response was: #{response}"
947
+ end
948
+ end
949
+ raise if options[:raise] # re-raise an exception
950
+ return nil
951
+ end
952
+
953
+ # True if e is an AWS system error, i.e. something that is for sure not the caller's fault.
954
+ # Used to force logging.
955
+ def self.system_error?(e)
956
+ !e.is_a?(self) || e.message =~ /InternalError|InsufficientInstanceCapacity|Unavailable/
957
+ end
958
+
959
+ end
960
+
961
+
962
+ class AWSErrorHandler
963
+ # 0-100 (%)
964
+ DEFAULT_CLOSE_ON_4XX_PROBABILITY = 10
965
+
966
+ @@reiteration_start_delay = 0.2
967
+ def self.reiteration_start_delay
968
+ @@reiteration_start_delay
969
+ end
970
+ def self.reiteration_start_delay=(reiteration_start_delay)
971
+ @@reiteration_start_delay = reiteration_start_delay
972
+ end
973
+
974
+ @@reiteration_time = 5
975
+ def self.reiteration_time
976
+ @@reiteration_time
977
+ end
978
+ def self.reiteration_time=(reiteration_time)
979
+ @@reiteration_time = reiteration_time
980
+ end
981
+
982
+ @@close_on_error = true
983
+ def self.close_on_error
984
+ @@close_on_error
985
+ end
986
+ def self.close_on_error=(close_on_error)
987
+ @@close_on_error = close_on_error
988
+ end
989
+
990
+ @@close_on_4xx_probability = DEFAULT_CLOSE_ON_4XX_PROBABILITY
991
+ def self.close_on_4xx_probability
992
+ @@close_on_4xx_probability
993
+ end
994
+ def self.close_on_4xx_probability=(close_on_4xx_probability)
995
+ @@close_on_4xx_probability = close_on_4xx_probability
996
+ end
997
+
998
+ # params:
999
+ # :reiteration_time
1000
+ # :errors_list
1001
+ # :close_on_error = true | false
1002
+ # :close_on_4xx_probability = 1-100
1003
+ def initialize(aws, parser, params={}) #:nodoc:
1004
+ @aws = aws # Link to RightEc2 | RightSqs | RightS3 instance
1005
+ @parser = parser # parser to parse Amazon response
1006
+ @started_at = Time.now
1007
+ @stop_at = @started_at + (params[:reiteration_time] || @@reiteration_time)
1008
+ @errors_list = params[:errors_list] || []
1009
+ @reiteration_delay = @@reiteration_start_delay
1010
+ @retries = 0
1011
+ # close current HTTP(S) connection on 5xx, errors from list and 4xx errors
1012
+ @close_on_error = params[:close_on_error].nil? ? @@close_on_error : params[:close_on_error]
1013
+ @close_on_4xx_probability = params[:close_on_4xx_probability] || @@close_on_4xx_probability
1014
+ end
1015
+
1016
+ # Returns false if
1017
+ def check(request) #:nodoc:
1018
+ result = false
1019
+ error_found = false
1020
+ redirect_detected= false
1021
+ error_match = nil
1022
+ last_errors_text = ''
1023
+ response = @aws.last_response
1024
+ # log error
1025
+ request_text_data = "#{request[:protocol]}://#{request[:server]}:#{request[:port]}#{request[:request].path}"
1026
+ # is this a redirect?
1027
+ # yes!
1028
+ if response.is_a?(Net::HTTPRedirection)
1029
+ redirect_detected = true
1030
+ else
1031
+ # no, it's an error ...
1032
+ @aws.logger.warn("##### #{@aws.class.name} returned an error: #{response.code} #{response.message}\n#{response.body} #####")
1033
+ @aws.logger.warn("##### #{@aws.class.name} request: #{request_text_data} ####")
1034
+ end
1035
+
1036
+ # Extract error/redirection message from the response body
1037
+ # Amazon claims that a redirection must have a body but somethimes it is nil....
1038
+ if response.body && response.body[/^(<\?xml|<ErrorResponse)/]
1039
+ error_parser = RightErrorResponseParser.new
1040
+ @aws.class.bench_xml.add! do
1041
+ error_parser.parse(response.body)
1042
+ end
1043
+ @aws.last_errors = error_parser.errors
1044
+ @aws.last_request_id = error_parser.requestID
1045
+ last_errors_text = @aws.last_errors.flatten.join("\n")
1046
+ else
1047
+ @aws.last_errors = [[response.code, "#{response.message} (#{request_text_data})"]]
1048
+ @aws.last_request_id = '-undefined-'
1049
+ last_errors_text = response.message
1050
+ end
1051
+
1052
+ # Ok, it is a redirect, find the new destination location
1053
+ if redirect_detected
1054
+ location = response['location']
1055
+ # As for 301 ( Moved Permanently) Amazon does not return a 'Location' header but
1056
+ # it is possible to extract a new endpoint from the response body
1057
+ if location.right_blank? && response.code=='301' && response.body
1058
+ new_endpoint = response.body[/<Endpoint>(.*?)<\/Endpoint>/] && $1
1059
+ location = "#{request[:protocol]}://#{new_endpoint}:#{request[:port]}#{request[:request].path}"
1060
+ end
1061
+ # ... log information and ...
1062
+ @aws.logger.info("##### #{@aws.class.name} redirect requested: #{response.code} #{response.message} #####")
1063
+ @aws.logger.info(" Old location: #{request_text_data}")
1064
+ @aws.logger.info(" New location: #{location}")
1065
+ @aws.logger.info(" Request Verb: #{request[:request].class.name}")
1066
+ # ... fix the connection data
1067
+ request[:server] = URI.parse(location).host
1068
+ request[:protocol] = URI.parse(location).scheme
1069
+ request[:port] = URI.parse(location).port
1070
+ else
1071
+ # Not a redirect but an error: try to find the error in our list
1072
+ @errors_list.each do |error_to_find|
1073
+ if last_errors_text[/#{error_to_find}/i]
1074
+ error_found = true
1075
+ error_match = error_to_find
1076
+ @aws.logger.warn("##### Retry is needed, error pattern match: #{error_to_find} #####")
1077
+ break
1078
+ end
1079
+ end
1080
+ end
1081
+
1082
+ # check the time has gone from the first error come
1083
+ if redirect_detected || error_found
1084
+ # Close the connection to the server and recreate a new one.
1085
+ # It may have a chance that one server is a semi-down and reconnection
1086
+ # will help us to connect to the other server
1087
+ if !redirect_detected && @close_on_error
1088
+ @aws.destroy_connection(request, "#{self.class.name}: error match to pattern '#{error_match}'")
1089
+ end
1090
+
1091
+ if (Time.now < @stop_at)
1092
+ @retries += 1
1093
+ unless redirect_detected
1094
+ @aws.logger.warn("##### Retry ##{@retries} is being performed. Sleeping for #{@reiteration_delay} sec. Whole time: #{Time.now-@started_at} sec ####")
1095
+ sleep @reiteration_delay
1096
+ @reiteration_delay *= 2
1097
+
1098
+ # Always make sure that the fp is set to point to the beginning(?)
1099
+ # of the File/IO. TODO: it assumes that offset is 0, which is bad.
1100
+ if(request[:request].body_stream && request[:request].body_stream.respond_to?(:pos))
1101
+ begin
1102
+ request[:request].body_stream.pos = 0
1103
+ rescue Exception => e
1104
+ @logger.warn("Retry may fail due to unable to reset the file pointer" +
1105
+ " -- #{self.class.name} : #{e.inspect}")
1106
+ end
1107
+ end
1108
+ else
1109
+ @aws.logger.info("##### Retry ##{@retries} is being performed due to a redirect. ####")
1110
+ end
1111
+ result = @aws.request_info(request, @parser)
1112
+ else
1113
+ @aws.logger.warn("##### Ooops, time is over... ####")
1114
+ end
1115
+ # aha, this is unhandled error:
1116
+ elsif @close_on_error
1117
+ # On 5xx(Server errors), 403(RequestTimeTooSkewed) and 408(Request Timeout) a conection has to be closed
1118
+ if @aws.last_response.code.to_s[/^(5\d\d|403|408)$/]
1119
+ @aws.destroy_connection(request, "#{self.class.name}: code: #{@aws.last_response.code}: '#{@aws.last_response.message}'")
1120
+ # Is this a 4xx error ?
1121
+ elsif @aws.last_response.code.to_s[/^4\d\d$/] && @close_on_4xx_probability > rand(100)
1122
+ @aws.destroy_connection(request, "#{self.class.name}: code: #{@aws.last_response.code}: '#{@aws.last_response.message}', " +
1123
+ "probability: #{@close_on_4xx_probability}%")
1124
+ end
1125
+ end
1126
+ result
1127
+ end
1128
+
1129
+ end
1130
+
1131
+
1132
+ #-----------------------------------------------------------------
1133
+
1134
+ class RightSaxParserCallbackTemplate #:nodoc:
1135
+ def initialize(right_aws_parser)
1136
+ @right_aws_parser = right_aws_parser
1137
+ end
1138
+ def on_characters(chars)
1139
+ @right_aws_parser.text(chars)
1140
+ end
1141
+ def on_start_document; end
1142
+ def on_comment(msg); end
1143
+ def on_processing_instruction(target, data); end
1144
+ def on_cdata_block(cdata); end
1145
+ def on_end_document; end
1146
+ end
1147
+
1148
+ class RightSaxParserCallback < RightSaxParserCallbackTemplate
1149
+ def self.include_callback
1150
+ include XML::SaxParser::Callbacks
1151
+ end
1152
+ def on_start_element(name, attr_hash)
1153
+ @right_aws_parser.tag_start(name, attr_hash)
1154
+ end
1155
+ def on_end_element(name)
1156
+ @right_aws_parser.tag_end(name)
1157
+ end
1158
+ end
1159
+
1160
+ class RightSaxParserCallbackNs < RightSaxParserCallbackTemplate
1161
+ def on_start_element_ns(name, attr_hash, prefix, uri, namespaces)
1162
+ @right_aws_parser.tag_start(name, attr_hash)
1163
+ end
1164
+ def on_end_element_ns(name, prefix, uri)
1165
+ @right_aws_parser.tag_end(name)
1166
+ end
1167
+ end
1168
+
1169
+ class RightAWSParser #:nodoc:
1170
+ # default parsing library
1171
+ DEFAULT_XML_LIBRARY = 'rexml'
1172
+ # a list of supported parsers
1173
+ @@supported_xml_libs = [DEFAULT_XML_LIBRARY, 'libxml']
1174
+
1175
+ @@xml_lib = DEFAULT_XML_LIBRARY # xml library name: 'rexml' | 'libxml'
1176
+ def self.xml_lib
1177
+ @@xml_lib
1178
+ end
1179
+ def self.xml_lib=(new_lib_name)
1180
+ @@xml_lib = new_lib_name
1181
+ end
1182
+
1183
+ attr_accessor :result
1184
+ attr_reader :xmlpath
1185
+ attr_accessor :xml_lib
1186
+ attr_reader :full_tag_name
1187
+ attr_reader :tag
1188
+
1189
+ def initialize(params={})
1190
+ @xmlpath = ''
1191
+ @full_tag_name = ''
1192
+ @result = false
1193
+ @text = ''
1194
+ @tag = ''
1195
+ @xml_lib = params[:xml_lib] || @@xml_lib
1196
+ @logger = params[:logger]
1197
+ reset
1198
+ end
1199
+ def tag_start(name, attributes)
1200
+ @text = ''
1201
+ @tag = name
1202
+ @full_tag_name += @full_tag_name.empty? ? name : "/#{name}"
1203
+ tagstart(name, attributes)
1204
+ @xmlpath = @full_tag_name
1205
+ end
1206
+ def tag_end(name)
1207
+ @xmlpath = @full_tag_name[/^(.*?)\/?#{name}$/] && $1
1208
+ tagend(name)
1209
+ @full_tag_name = @xmlpath
1210
+ end
1211
+ def text(text)
1212
+ @text += text
1213
+ tagtext(text)
1214
+ end
1215
+ # Parser method.
1216
+ # Params:
1217
+ # xml_text - xml message text(String) or Net:HTTPxxx instance (response)
1218
+ # params[:xml_lib] - library name: 'rexml' | 'libxml'
1219
+ def parse(xml_text, params={})
1220
+ # Get response body
1221
+ xml_text = xml_text.body unless xml_text.is_a?(String)
1222
+ @xml_lib = params[:xml_lib] || @xml_lib
1223
+ # check that we had no problems with this library otherwise use default
1224
+ @xml_lib = DEFAULT_XML_LIBRARY unless @@supported_xml_libs.include?(@xml_lib)
1225
+ # load xml library
1226
+ if @xml_lib=='libxml' && !defined?(XML::SaxParser)
1227
+ begin
1228
+ require 'xml/libxml'
1229
+ # Setup SaxParserCallback
1230
+ if XML::Parser::VERSION >= '0.5.1' &&
1231
+ XML::Parser::VERSION < '0.9.7'
1232
+ RightSaxParserCallback.include_callback
1233
+ end
1234
+ rescue LoadError => e
1235
+ @@supported_xml_libs.delete(@xml_lib)
1236
+ @xml_lib = DEFAULT_XML_LIBRARY
1237
+ if @logger
1238
+ @logger.error e.inspect
1239
+ @logger.error e.backtrace
1240
+ @logger.info "Can not load 'libxml' library. '#{DEFAULT_XML_LIBRARY}' is used for parsing."
1241
+ end
1242
+ end
1243
+ end
1244
+ # Parse the xml text
1245
+ case @xml_lib
1246
+ when 'libxml'
1247
+ if XML::Parser::VERSION >= '0.9.9'
1248
+ # avoid warning on every usage
1249
+ xml = XML::SaxParser.string(xml_text)
1250
+ else
1251
+ xml = XML::SaxParser.new
1252
+ xml.string = xml_text
1253
+ end
1254
+ # check libxml-ruby version
1255
+ if XML::Parser::VERSION >= '0.9.7'
1256
+ xml.callbacks = RightSaxParserCallbackNs.new(self)
1257
+ elsif XML::Parser::VERSION >= '0.5.1'
1258
+ xml.callbacks = RightSaxParserCallback.new(self)
1259
+ else
1260
+ xml.on_start_element{|name, attr_hash| self.tag_start(name, attr_hash)}
1261
+ xml.on_characters{ |text| self.text(text)}
1262
+ xml.on_end_element{ |name| self.tag_end(name)}
1263
+ end
1264
+ xml.parse
1265
+ else
1266
+ REXML::Document.parse_stream(xml_text, self)
1267
+ end
1268
+ end
1269
+ # Parser must have a lots of methods
1270
+ # (see /usr/lib/ruby/1.8/rexml/parsers/streamparser.rb)
1271
+ # We dont need most of them in RightAWSParser and method_missing helps us
1272
+ # to skip their definition
1273
+ def method_missing(method, *params)
1274
+ # if the method is one of known - just skip it ...
1275
+ return if [:comment, :attlistdecl, :notationdecl, :elementdecl,
1276
+ :entitydecl, :cdata, :xmldecl, :attlistdecl, :instruction,
1277
+ :doctype].include?(method)
1278
+ # ... else - call super to raise an exception
1279
+ super(method, params)
1280
+ end
1281
+ # the functions to be overriden by children (if nessesery)
1282
+ def reset ; end
1283
+ def tagstart(name, attributes); end
1284
+ def tagend(name) ; end
1285
+ def tagtext(text) ; end
1286
+ end
1287
+
1288
+ #-----------------------------------------------------------------
1289
+ # PARSERS: Errors
1290
+ #-----------------------------------------------------------------
1291
+
1292
+ #<Error>
1293
+ # <Code>TemporaryRedirect</Code>
1294
+ # <Message>Please re-send this request to the specified temporary endpoint. Continue to use the original request endpoint for future requests.</Message>
1295
+ # <RequestId>FD8D5026D1C5ABA3</RequestId>
1296
+ # <Endpoint>bucket-for-k.s3-external-3.amazonaws.com</Endpoint>
1297
+ # <HostId>ItJy8xPFPli1fq/JR3DzQd3iDvFCRqi1LTRmunEdM1Uf6ZtW2r2kfGPWhRE1vtaU</HostId>
1298
+ # <Bucket>bucket-for-k</Bucket>
1299
+ #</Error>
1300
+
1301
+ class RightErrorResponseParser < RightAWSParser #:nodoc:
1302
+ attr_accessor :errors # array of hashes: error/message
1303
+ attr_accessor :requestID
1304
+ # attr_accessor :endpoint, :host_id, :bucket
1305
+ def tagend(name)
1306
+ case name
1307
+ when 'RequestID' ; @requestID = @text
1308
+ when 'Code' ; @code = @text
1309
+ when 'Message' ; @message = @text
1310
+ # when 'Endpoint' ; @endpoint = @text
1311
+ # when 'HostId' ; @host_id = @text
1312
+ # when 'Bucket' ; @bucket = @text
1313
+ when 'Error' ; @errors << [ @code, @message ]
1314
+ end
1315
+ end
1316
+ def reset
1317
+ @errors = []
1318
+ end
1319
+ end
1320
+
1321
+ # Dummy parser - does nothing
1322
+ # Returns the original params back
1323
+ class RightDummyParser # :nodoc:
1324
+ attr_accessor :result
1325
+ def parse(response, params={})
1326
+ @result = [response, params]
1327
+ end
1328
+ end
1329
+
1330
+ class RightHttp2xxParser < RightAWSParser # :nodoc:
1331
+ def parse(response)
1332
+ @result = response.is_a?(Net::HTTPSuccess)
1333
+ end
1334
+ end
1335
+
1336
+ class RightBoolResponseParser < RightAWSParser #:nodoc:
1337
+ def tagend(name)
1338
+ @result = (@text=='true') if name == 'return'
1339
+ end
1340
+ end
1341
+
1342
+ end
1343
+