right_aws-yodal 1.10.5

Sign up to get free protection for your applications and to get access to all the features.
@@ -0,0 +1,933 @@
1
+ #
2
+ # Copyright (c) 2007-2008 RightScale Inc
3
+ #
4
+ # Permission is hereby granted, free of charge, to any person obtaining
5
+ # a copy of this software and associated documentation files (the
6
+ # "Software"), to deal in the Software without restriction, including
7
+ # without limitation the rights to use, copy, modify, merge, publish,
8
+ # distribute, sublicense, and/or sell copies of the Software, and to
9
+ # permit persons to whom the Software is furnished to do so, subject to
10
+ # the following conditions:
11
+ #
12
+ # The above copyright notice and this permission notice shall be
13
+ # included in all copies or substantial portions of the Software.
14
+ #
15
+ # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
16
+ # EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
17
+ # MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
18
+ # NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
19
+ # LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
20
+ # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
21
+ # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
22
+ #
23
+
24
+ # Test
25
+ module RightAws
26
+ require 'md5'
27
+ require 'pp'
28
+
29
+ class AwsUtils #:nodoc:
30
+ @@digest1 = OpenSSL::Digest::Digest.new("sha1")
31
+ @@digest256 = nil
32
+ if OpenSSL::OPENSSL_VERSION_NUMBER > 0x00908000
33
+ @@digest256 = OpenSSL::Digest::Digest.new("sha256") rescue nil # Some installation may not support sha256
34
+ end
35
+
36
+ def self.sign(aws_secret_access_key, auth_string)
37
+ Base64.encode64(OpenSSL::HMAC.digest(@@digest1, aws_secret_access_key, auth_string)).strip
38
+ end
39
+
40
+ # Escape a string accordingly Amazon rulles
41
+ # http://docs.amazonwebservices.com/AmazonSimpleDB/2007-11-07/DeveloperGuide/index.html?REST_RESTAuth.html
42
+ def self.amz_escape(param)
43
+ param.to_s.gsub(/([^a-zA-Z0-9._~-]+)/n) do
44
+ '%' + $1.unpack('H2' * $1.size).join('%').upcase
45
+ end
46
+ end
47
+
48
+ # Set a timestamp and a signature version
49
+ def self.fix_service_params(service_hash, signature)
50
+ service_hash["Timestamp"] ||= Time.now.utc.strftime("%Y-%m-%dT%H:%M:%S.000Z") unless service_hash["Expires"]
51
+ service_hash["SignatureVersion"] = signature
52
+ service_hash
53
+ end
54
+
55
+ # Signature Version 0
56
+ # A deprecated guy (should work till septemper 2009)
57
+ def self.sign_request_v0(aws_secret_access_key, service_hash)
58
+ fix_service_params(service_hash, '0')
59
+ string_to_sign = "#{service_hash['Action']}#{service_hash['Timestamp'] || service_hash['Expires']}"
60
+ service_hash['Signature'] = AwsUtils::sign(aws_secret_access_key, string_to_sign)
61
+ service_hash.to_a.collect{|key,val| "#{amz_escape(key)}=#{amz_escape(val.to_s)}" }.join("&")
62
+ end
63
+
64
+ # Signature Version 1
65
+ # Another deprecated guy (should work till septemper 2009)
66
+ def self.sign_request_v1(aws_secret_access_key, service_hash)
67
+ fix_service_params(service_hash, '1')
68
+ string_to_sign = service_hash.sort{|a,b| (a[0].to_s.downcase)<=>(b[0].to_s.downcase)}.to_s
69
+ service_hash['Signature'] = AwsUtils::sign(aws_secret_access_key, string_to_sign)
70
+ service_hash.to_a.collect{|key,val| "#{amz_escape(key)}=#{amz_escape(val.to_s)}" }.join("&")
71
+ end
72
+
73
+ # Signature Version 2
74
+ # EC2, SQS and SDB requests must be signed by this guy.
75
+ # See: http://docs.amazonwebservices.com/AmazonSimpleDB/2007-11-07/DeveloperGuide/index.html?REST_RESTAuth.html
76
+ # http://developer.amazonwebservices.com/connect/entry.jspa?externalID=1928
77
+ def self.sign_request_v2(aws_secret_access_key, service_hash, http_verb, host, uri)
78
+ fix_service_params(service_hash, '2')
79
+ # select a signing method (make an old openssl working with sha1)
80
+ # make 'HmacSHA256' to be a default one
81
+ service_hash['SignatureMethod'] = 'HmacSHA256' unless ['HmacSHA256', 'HmacSHA1'].include?(service_hash['SignatureMethod'])
82
+ service_hash['SignatureMethod'] = 'HmacSHA1' unless @@digest256
83
+ # select a digest
84
+ digest = (service_hash['SignatureMethod'] == 'HmacSHA256' ? @@digest256 : @@digest1)
85
+ # form string to sign
86
+ canonical_string = service_hash.keys.sort.map do |key|
87
+ "#{amz_escape(key)}=#{amz_escape(service_hash[key])}"
88
+ end.join('&')
89
+ string_to_sign = "#{http_verb.to_s.upcase}\n#{host.downcase}\n#{uri}\n#{canonical_string}"
90
+ # sign the string
91
+ signature = amz_escape(Base64.encode64(OpenSSL::HMAC.digest(digest, aws_secret_access_key, string_to_sign)).strip)
92
+ "#{canonical_string}&Signature=#{signature}"
93
+ end
94
+
95
+ # From Amazon's SQS Dev Guide, a brief description of how to escape:
96
+ # "URL encode the computed signature and other query parameters as specified in
97
+ # RFC1738, section 2.2. In addition, because the + character is interpreted as a blank space
98
+ # by Sun Java classes that perform URL decoding, make sure to encode the + character
99
+ # although it is not required by RFC1738."
100
+ # Avoid using CGI::escape to escape URIs.
101
+ # CGI::escape will escape characters in the protocol, host, and port
102
+ # sections of the URI. Only target chars in the query
103
+ # string should be escaped.
104
+ def self.URLencode(raw)
105
+ e = URI.escape(raw)
106
+ e.gsub(/\+/, "%2b")
107
+ end
108
+
109
+ def self.allow_only(allowed_keys, params)
110
+ bogus_args = []
111
+ params.keys.each {|p| bogus_args.push(p) unless allowed_keys.include?(p) }
112
+ raise AwsError.new("The following arguments were given but are not legal for the function call #{caller_method}: #{bogus_args.inspect}") if bogus_args.length > 0
113
+ end
114
+
115
+ def self.mandatory_arguments(required_args, params)
116
+ rargs = required_args.dup
117
+ params.keys.each {|p| rargs.delete(p)}
118
+ raise AwsError.new("The following mandatory arguments were not provided to #{caller_method}: #{rargs.inspect}") if rargs.length > 0
119
+ end
120
+
121
+ def self.caller_method
122
+ caller[1]=~/`(.*?)'/
123
+ $1
124
+ end
125
+
126
+ def self.split_items_and_params(array)
127
+ items = array.to_a.flatten.compact
128
+ params = items.last.kind_of?(Hash) ? items.pop : {}
129
+ [items, params]
130
+ end
131
+ end
132
+
133
+ class AwsBenchmarkingBlock #:nodoc:
134
+ attr_accessor :xml, :service
135
+ def initialize
136
+ # Benchmark::Tms instance for service (Ec2, S3, or SQS) access benchmarking.
137
+ @service = Benchmark::Tms.new()
138
+ # Benchmark::Tms instance for XML parsing benchmarking.
139
+ @xml = Benchmark::Tms.new()
140
+ end
141
+ end
142
+
143
+ class AwsNoChange < RuntimeError
144
+ end
145
+
146
+ class RightAwsBase
147
+
148
+ # Amazon HTTP Error handling
149
+
150
+ # Text, if found in an error message returned by AWS, indicates that this may be a transient
151
+ # error. Transient errors are automatically retried with exponential back-off.
152
+ AMAZON_PROBLEMS = [ 'internal service error',
153
+ 'is currently unavailable',
154
+ 'no response from',
155
+ 'Please try again',
156
+ 'InternalError',
157
+ 'Internal Server Error',
158
+ 'ServiceUnavailable', #from SQS docs
159
+ 'Unavailable',
160
+ 'This application is not currently available',
161
+ 'InsufficientInstanceCapacity'
162
+ ]
163
+ @@amazon_problems = AMAZON_PROBLEMS
164
+ # Returns a list of Amazon service responses which are known to be transient problems.
165
+ # We have to re-request if we get any of them, because the problem will probably disappear.
166
+ # By default this method returns the same value as the AMAZON_PROBLEMS const.
167
+ def self.amazon_problems
168
+ @@amazon_problems
169
+ end
170
+
171
+ # Sets the list of Amazon side problems. Use in conjunction with the
172
+ # getter to append problems.
173
+ def self.amazon_problems=(problems_list)
174
+ @@amazon_problems = problems_list
175
+ end
176
+
177
+ end
178
+
179
+ module RightAwsBaseInterface
180
+ DEFAULT_SIGNATURE_VERSION = '2'
181
+
182
+ @@caching = false
183
+ def self.caching
184
+ @@caching
185
+ end
186
+ def self.caching=(caching)
187
+ @@caching = caching
188
+ end
189
+
190
+ # Current aws_access_key_id
191
+ attr_reader :aws_access_key_id
192
+ # Current aws_secret_access_key
193
+ attr_reader :aws_secret_access_key
194
+ # Last HTTP request object
195
+ attr_reader :last_request
196
+ # Last HTTP response object
197
+ attr_reader :last_response
198
+ # Last AWS errors list (used by AWSErrorHandler)
199
+ attr_accessor :last_errors
200
+ # Last AWS request id (used by AWSErrorHandler)
201
+ attr_accessor :last_request_id
202
+ # Logger object
203
+ attr_accessor :logger
204
+ # Initial params hash
205
+ attr_accessor :params
206
+ # RightHttpConnection instance
207
+ attr_reader :connection
208
+ # Cache
209
+ attr_reader :cache
210
+ # Signature version (all services except s3)
211
+ attr_reader :signature_version
212
+
213
+ def init(service_info, aws_access_key_id, aws_secret_access_key, params={}) #:nodoc:
214
+ @params = params
215
+ # If one defines EC2_URL he may forget to use a single slash as an "empty service" path.
216
+ # Amazon does not like this therefore add this bad boy if he is missing...
217
+ service_info[:default_service] = '/' if service_info[:default_service].blank?
218
+ raise AwsError.new("AWS access keys are required to operate on #{service_info[:name]}") \
219
+ if aws_access_key_id.blank? || aws_secret_access_key.blank?
220
+ @aws_access_key_id = aws_access_key_id
221
+ @aws_secret_access_key = aws_secret_access_key
222
+ # if the endpoint was explicitly defined - then use it
223
+ if @params[:endpoint_url]
224
+ @params[:server] = URI.parse(@params[:endpoint_url]).host
225
+ @params[:port] = URI.parse(@params[:endpoint_url]).port
226
+ @params[:service] = URI.parse(@params[:endpoint_url]).path
227
+ # make sure the 'service' path is not empty
228
+ @params[:service] = service_info[:default_service] if @params[:service].blank?
229
+ @params[:protocol] = URI.parse(@params[:endpoint_url]).scheme
230
+ @params[:region] = nil
231
+ else
232
+ @params[:server] ||= service_info[:default_host]
233
+ @params[:server] = "#{@params[:region]}.#{@params[:server]}" if @params[:region]
234
+ @params[:port] ||= service_info[:default_port]
235
+ @params[:service] ||= service_info[:default_service]
236
+ @params[:protocol] ||= service_info[:default_protocol]
237
+ end
238
+ # @params[:multi_thread] ||= defined?(AWS_DAEMON)
239
+ @params[:connections] ||= :shared # || :dedicated
240
+ @params[:max_connections] ||= 10
241
+ @params[:connection_lifetime] ||= 20*60
242
+ @params[:api_version] ||= service_info[:default_api_version]
243
+ @logger = @params[:logger]
244
+ @logger = RAILS_DEFAULT_LOGGER if !@logger && defined?(RAILS_DEFAULT_LOGGER)
245
+ @logger = Logger.new(STDOUT) if !@logger
246
+ @logger.info "New #{self.class.name} using #{@params[:connections]} connections mode"
247
+ @error_handler = nil
248
+ @cache = {}
249
+ @signature_version = (params[:signature_version] || DEFAULT_SIGNATURE_VERSION).to_s
250
+ end
251
+
252
+ def signed_service_params(aws_secret_access_key, service_hash, http_verb=nil, host=nil, service=nil )
253
+ case signature_version.to_s
254
+ when '0' then AwsUtils::sign_request_v0(aws_secret_access_key, service_hash)
255
+ when '1' then AwsUtils::sign_request_v1(aws_secret_access_key, service_hash)
256
+ when '2' then AwsUtils::sign_request_v2(aws_secret_access_key, service_hash, http_verb, host, service)
257
+ else raise AwsError.new("Unknown signature version (#{signature_version.to_s}) requested")
258
+ end
259
+ end
260
+
261
+ # Returns +true+ if the describe_xxx responses are being cached
262
+ def caching?
263
+ @params.key?(:cache) ? @params[:cache] : @@caching
264
+ end
265
+
266
+ # Check if the aws function response hits the cache or not.
267
+ # If the cache hits:
268
+ # - raises an +AwsNoChange+ exception if +do_raise+ == +:raise+.
269
+ # - returnes parsed response from the cache if it exists or +true+ otherwise.
270
+ # If the cache miss or the caching is off then returns +false+.
271
+ def cache_hits?(function, response, do_raise=:raise)
272
+ result = false
273
+ if caching?
274
+ function = function.to_sym
275
+ # get rid of requestId (this bad boy was added for API 2008-08-08+ and it is uniq for every response)
276
+ # feb 04, 2009 (load balancer uses 'RequestId' hence use 'i' modifier to hit it also)
277
+ response = response.sub(%r{<requestId>.+?</requestId>}i, '')
278
+ response_md5 = MD5.md5(response).to_s
279
+ # check for changes
280
+ unless @cache[function] && @cache[function][:response_md5] == response_md5
281
+ # well, the response is new, reset cache data
282
+ update_cache(function, {:response_md5 => response_md5,
283
+ :timestamp => Time.now,
284
+ :hits => 0,
285
+ :parsed => nil})
286
+ else
287
+ # aha, cache hits, update the data and throw an exception if needed
288
+ @cache[function][:hits] += 1
289
+ if do_raise == :raise
290
+ raise(AwsNoChange, "Cache hit: #{function} response has not changed since "+
291
+ "#{@cache[function][:timestamp].strftime('%Y-%m-%d %H:%M:%S')}, "+
292
+ "hits: #{@cache[function][:hits]}.")
293
+ else
294
+ result = @cache[function][:parsed] || true
295
+ end
296
+ end
297
+ end
298
+ result
299
+ end
300
+
301
+ def update_cache(function, hash)
302
+ (@cache[function.to_sym] ||= {}).merge!(hash) if caching?
303
+ end
304
+
305
+ def on_exception(options={:raise=>true, :log=>true}) # :nodoc:
306
+ raise if $!.is_a?(AwsNoChange)
307
+ AwsError::on_aws_exception(self, options)
308
+ end
309
+
310
+ # # Return +true+ if this instance works in multi_thread mode and +false+ otherwise.
311
+ # def multi_thread
312
+ # @params[:multi_thread]
313
+ # end
314
+
315
+ # ACF, AMS, EC2, LBS and SDB uses this guy
316
+ # SQS and S3 use their own methods
317
+ def generate_request_impl(verb, action, options={}) #:nodoc:
318
+ # Form a valid http verb: 'GET' or 'POST' (all the other are not supported now)
319
+ http_verb = verb.to_s.upcase
320
+ # remove empty keys from request options
321
+ options.delete_if { |key, value| value.nil? }
322
+ # prepare service data
323
+ service_hash = {"Action" => action,
324
+ "AWSAccessKeyId" => @aws_access_key_id,
325
+ "Version" => @params[:api_version] }
326
+ service_hash.merge!(options)
327
+ # Sign request options
328
+ service_params = signed_service_params(@aws_secret_access_key, service_hash, http_verb, @params[:server], @params[:service])
329
+ # Use POST if the length of the query string is too large
330
+ # see http://docs.amazonwebservices.com/AmazonSimpleDB/2007-11-07/DeveloperGuide/MakingRESTRequests.html
331
+ if http_verb != 'POST' && service_params.size > 2000
332
+ http_verb = 'POST'
333
+ if signature_version == '2'
334
+ service_params = signed_service_params(@aws_secret_access_key, service_hash, http_verb, @params[:server], @params[:service])
335
+ end
336
+ end
337
+ # create a request
338
+ case http_verb
339
+ when 'GET'
340
+ request = Net::HTTP::Get.new("#{@params[:service]}?#{service_params}")
341
+ when 'POST'
342
+ request = Net::HTTP::Post.new(@params[:service])
343
+ request.body = service_params
344
+ request['Content-Type'] = 'application/x-www-form-urlencoded'
345
+ else
346
+ raise "Unsupported HTTP verb #{verb.inspect}!"
347
+ end
348
+ # prepare output hash
349
+ { :request => request,
350
+ :server => @params[:server],
351
+ :port => @params[:port],
352
+ :protocol => @params[:protocol] }
353
+ end
354
+
355
+ def get_connection(aws_service, request) #:nodoc
356
+ server_url = "#{request[:protocol]}://#{request[:server]}:#{request[:port]}}"
357
+ #
358
+ case @params[:connections].to_s
359
+ when 'dedicated'
360
+ @connections_storage ||= {}
361
+ else # 'dedicated'
362
+ @connections_storage = (Thread.current[aws_service] ||= {})
363
+ end
364
+ #
365
+ @connections_storage[server_url] ||= {}
366
+ @connections_storage[server_url][:last_used_at] = Time.now
367
+ @connections_storage[server_url][:connection] ||= Rightscale::HttpConnection.new(:exception => RightAws::AwsError, :logger => @logger)
368
+ # keep X most recent connections (but were used not far than Y minutes ago)
369
+ connections = 0
370
+ @connections_storage.to_a.sort{|i1, i2| i2[1][:last_used_at] <=> i1[1][:last_used_at]}.to_a.each do |i|
371
+ if i[0] != server_url && (@params[:max_connections] <= connections || i[1][:last_used_at] < Time.now - @params[:connection_lifetime])
372
+ # delete the connection from the list
373
+ @connections_storage.delete(i[0])
374
+ # then finish it
375
+ i[1][:connection].finish((@params[:max_connections] <= connections) ? "out-of-limit" : "out-of-date") rescue nil
376
+ else
377
+ connections += 1
378
+ end
379
+ end
380
+ @connections_storage[server_url][:connection]
381
+ end
382
+
383
+ # All services uses this guy.
384
+ def request_info_impl(aws_service, benchblock, request, parser, &block) #:nodoc:
385
+ @connection = get_connection(aws_service, request)
386
+ @last_request = request[:request]
387
+ @last_response = nil
388
+ response = nil
389
+ blockexception = nil
390
+
391
+ if(block != nil)
392
+ # TRB 9/17/07 Careful - because we are passing in blocks, we get a situation where
393
+ # an exception may get thrown in the block body (which is high-level
394
+ # code either here or in the application) but gets caught in the
395
+ # low-level code of HttpConnection. The solution is not to let any
396
+ # exception escape the block that we pass to HttpConnection::request.
397
+ # Exceptions can originate from code directly in the block, or from user
398
+ # code called in the other block which is passed to response.read_body.
399
+ benchblock.service.add! do
400
+ responsehdr = @connection.request(request) do |response|
401
+ #########
402
+ begin
403
+ @last_response = response
404
+ if response.is_a?(Net::HTTPSuccess)
405
+ @error_handler = nil
406
+ response.read_body(&block)
407
+ else
408
+ @error_handler = AWSErrorHandler.new(self, parser, :errors_list => self.class.amazon_problems) unless @error_handler
409
+ check_result = @error_handler.check(request)
410
+ if check_result
411
+ @error_handler = nil
412
+ return check_result
413
+ end
414
+ raise AwsError.new(@last_errors, @last_response.code, @last_request_id)
415
+ end
416
+ rescue Exception => e
417
+ blockexception = e
418
+ end
419
+ end
420
+ #########
421
+
422
+ #OK, now we are out of the block passed to the lower level
423
+ if(blockexception)
424
+ raise blockexception
425
+ end
426
+ benchblock.xml.add! do
427
+ parser.parse(responsehdr)
428
+ end
429
+ return parser.result
430
+ end
431
+ else
432
+ benchblock.service.add!{ response = @connection.request(request) }
433
+ # check response for errors...
434
+ @last_response = response
435
+ if response.is_a?(Net::HTTPSuccess)
436
+ @error_handler = nil
437
+ benchblock.xml.add! { parser.parse(response) }
438
+ return parser.result
439
+ else
440
+ @error_handler = AWSErrorHandler.new(self, parser, :errors_list => self.class.amazon_problems) unless @error_handler
441
+ check_result = @error_handler.check(request)
442
+ if check_result
443
+ @error_handler = nil
444
+ return check_result
445
+ end
446
+ raise AwsError.new(@last_errors, @last_response.code, @last_request_id)
447
+ end
448
+ end
449
+ rescue
450
+ @error_handler = nil
451
+ raise
452
+ end
453
+
454
+ def request_cache_or_info(method, link, parser_class, benchblock, use_cache=true) #:nodoc:
455
+ # We do not want to break the logic of parsing hence will use a dummy parser to process all the standard
456
+ # steps (errors checking etc). The dummy parser does nothig - just returns back the params it received.
457
+ # If the caching is enabled and hit then throw AwsNoChange.
458
+ # P.S. caching works for the whole images list only! (when the list param is blank)
459
+ # check cache
460
+ response, params = request_info(link, RightDummyParser.new)
461
+ cache_hits?(method.to_sym, response.body) if use_cache
462
+ parser = parser_class.new(:logger => @logger)
463
+ benchblock.xml.add!{ parser.parse(response, params) }
464
+ result = block_given? ? yield(parser) : parser.result
465
+ # update parsed data
466
+ update_cache(method.to_sym, :parsed => result) if use_cache
467
+ result
468
+ end
469
+
470
+ # Returns Amazons request ID for the latest request
471
+ def last_request_id
472
+ @last_response && @last_response.body.to_s[%r{<requestId>(.+?)</requestId>}i] && $1
473
+ end
474
+
475
+ # Format array of items into Amazons handy hash ('?' is a place holder):
476
+ #
477
+ # amazonize_list('Item', ['a', 'b', 'c']) =>
478
+ # { 'Item.1' => 'a', 'Item.2' => 'b', 'Item.3' => 'c' }
479
+ #
480
+ # amazonize_list('Item.?.instance', ['a', 'c']) #=>
481
+ # { 'Item.1.instance' => 'a', 'Item.2.instance' => 'c' }
482
+ #
483
+ # amazonize_list(['Item.?.Name', 'Item.?.Value'], {'A' => 'a', 'B' => 'b'}) #=>
484
+ # { 'Item.1.Name' => 'A', 'Item.1.Value' => 'a',
485
+ # 'Item.2.Name' => 'B', 'Item.2.Value' => 'b' }
486
+ #
487
+ # amazonize_list(['Item.?.Name', 'Item.?.Value'], [['A','a'], ['B','b']]) #=>
488
+ # { 'Item.1.Name' => 'A', 'Item.1.Value' => 'a',
489
+ # 'Item.2.Name' => 'B', 'Item.2.Value' => 'b' }
490
+ #
491
+ # amazonize_list(['Filter.?.Key', 'Filter.?.Value.?'], {'A' => ['aa','ab'], 'B' => ['ba','bb']}) #=>
492
+ # amazonize_list(['Filter.?.Key', 'Filter.?.Value.?'], [['A',['aa','ab']], ['B',['ba','bb']]]) #=>
493
+ # {"Filter.1.Key"=>"A",
494
+ # "Filter.1.Value.1"=>"aa",
495
+ # "Filter.1.Value.2"=>"ab",
496
+ # "Filter.2.Key"=>"B",
497
+ # "Filter.2.Value.1"=>"ba",
498
+ # "Filter.2.Value.2"=>"bb"}
499
+ def amazonize_list(masks, list) #:nodoc:
500
+ groups = {}
501
+ list.to_a.each_with_index do |list_item, i|
502
+ masks.to_a.each_with_index do |mask, mask_idx|
503
+ key = mask[/\?/] ? mask.dup : mask.dup + '.?'
504
+ key.sub!('?', (i+1).to_s)
505
+ value = list_item.to_a[mask_idx]
506
+ if value.is_a?(Array)
507
+ groups.merge!(amazonize_list(key, value))
508
+ else
509
+ groups[key] = value
510
+ end
511
+ end
512
+ end
513
+ groups
514
+ end
515
+ end
516
+
517
+
518
+ # Exception class to signal any Amazon errors. All errors occuring during calls to Amazon's
519
+ # web services raise this type of error.
520
+ # Attribute inherited by RuntimeError:
521
+ # message - the text of the error, generally as returned by AWS in its XML response.
522
+ class AwsError < RuntimeError
523
+
524
+ # either an array of errors where each item is itself an array of [code, message]),
525
+ # or an error string if the error was raised manually, as in <tt>AwsError.new('err_text')</tt>
526
+ attr_reader :errors
527
+
528
+ # Request id (if exists)
529
+ attr_reader :request_id
530
+
531
+ # Response HTTP error code
532
+ attr_reader :http_code
533
+
534
+ def initialize(errors=nil, http_code=nil, request_id=nil)
535
+ @errors = errors
536
+ @request_id = request_id
537
+ @http_code = http_code
538
+ super(@errors.is_a?(Array) ? @errors.map{|code, msg| "#{code}: #{msg}"}.join("; ") : @errors.to_s)
539
+ end
540
+
541
+ # Does any of the error messages include the regexp +pattern+?
542
+ # Used to determine whether to retry request.
543
+ def include?(pattern)
544
+ if @errors.is_a?(Array)
545
+ @errors.each{ |code, msg| return true if code =~ pattern }
546
+ else
547
+ return true if @errors_str =~ pattern
548
+ end
549
+ false
550
+ end
551
+
552
+ # Generic handler for AwsErrors. +aws+ is the RightAws::S3, RightAws::EC2, or RightAws::SQS
553
+ # object that caused the exception (it must provide last_request and last_response). Supported
554
+ # boolean options are:
555
+ # * <tt>:log</tt> print a message into the log using aws.logger to access the Logger
556
+ # * <tt>:puts</tt> do a "puts" of the error
557
+ # * <tt>:raise</tt> re-raise the error after logging
558
+ def self.on_aws_exception(aws, options={:raise=>true, :log=>true})
559
+ # Only log & notify if not user error
560
+ if !options[:raise] || system_error?($!)
561
+ error_text = "#{$!.inspect}\n#{$@}.join('\n')}"
562
+ puts error_text if options[:puts]
563
+ # Log the error
564
+ if options[:log]
565
+ request = aws.last_request ? aws.last_request.path : '-none-'
566
+ response = aws.last_response ? "#{aws.last_response.code} -- #{aws.last_response.message} -- #{aws.last_response.body}" : '-none-'
567
+ aws.logger.error error_text
568
+ aws.logger.error "Request was: #{request}"
569
+ aws.logger.error "Response was: #{response}"
570
+ end
571
+ end
572
+ raise if options[:raise] # re-raise an exception
573
+ return nil
574
+ end
575
+
576
+ # True if e is an AWS system error, i.e. something that is for sure not the caller's fault.
577
+ # Used to force logging.
578
+ def self.system_error?(e)
579
+ !e.is_a?(self) || e.message =~ /InternalError|InsufficientInstanceCapacity|Unavailable/
580
+ end
581
+
582
+ end
583
+
584
+
585
+ class AWSErrorHandler
586
+ # 0-100 (%)
587
+ DEFAULT_CLOSE_ON_4XX_PROBABILITY = 10
588
+
589
+ @@reiteration_start_delay = 0.2
590
+ def self.reiteration_start_delay
591
+ @@reiteration_start_delay
592
+ end
593
+ def self.reiteration_start_delay=(reiteration_start_delay)
594
+ @@reiteration_start_delay = reiteration_start_delay
595
+ end
596
+
597
+ @@reiteration_time = 5
598
+ def self.reiteration_time
599
+ @@reiteration_time
600
+ end
601
+ def self.reiteration_time=(reiteration_time)
602
+ @@reiteration_time = reiteration_time
603
+ end
604
+
605
+ @@close_on_error = true
606
+ def self.close_on_error
607
+ @@close_on_error
608
+ end
609
+ def self.close_on_error=(close_on_error)
610
+ @@close_on_error = close_on_error
611
+ end
612
+
613
+ @@close_on_4xx_probability = DEFAULT_CLOSE_ON_4XX_PROBABILITY
614
+ def self.close_on_4xx_probability
615
+ @@close_on_4xx_probability
616
+ end
617
+ def self.close_on_4xx_probability=(close_on_4xx_probability)
618
+ @@close_on_4xx_probability = close_on_4xx_probability
619
+ end
620
+
621
+ # params:
622
+ # :reiteration_time
623
+ # :errors_list
624
+ # :close_on_error = true | false
625
+ # :close_on_4xx_probability = 1-100
626
+ def initialize(aws, parser, params={}) #:nodoc:
627
+ @aws = aws # Link to RightEc2 | RightSqs | RightS3 instance
628
+ @parser = parser # parser to parse Amazon response
629
+ @started_at = Time.now
630
+ @stop_at = @started_at + (params[:reiteration_time] || @@reiteration_time)
631
+ @errors_list = params[:errors_list] || []
632
+ @reiteration_delay = @@reiteration_start_delay
633
+ @retries = 0
634
+ # close current HTTP(S) connection on 5xx, errors from list and 4xx errors
635
+ @close_on_error = params[:close_on_error].nil? ? @@close_on_error : params[:close_on_error]
636
+ @close_on_4xx_probability = params[:close_on_4xx_probability] || @@close_on_4xx_probability
637
+ end
638
+
639
+ # Returns false if
640
+ def check(request) #:nodoc:
641
+ result = false
642
+ error_found = false
643
+ redirect_detected= false
644
+ error_match = nil
645
+ last_errors_text = ''
646
+ response = @aws.last_response
647
+ # log error
648
+ request_text_data = "#{request[:protocol]}://#{request[:server]}:#{request[:port]}#{request[:request].path}"
649
+ # is this a redirect?
650
+ # yes!
651
+ if response.is_a?(Net::HTTPRedirection)
652
+ redirect_detected = true
653
+ else
654
+ # no, it's an error ...
655
+ @aws.logger.warn("##### #{@aws.class.name} returned an error: #{response.code} #{response.message}\n#{response.body} #####")
656
+ @aws.logger.warn("##### #{@aws.class.name} request: #{request_text_data} ####")
657
+ end
658
+
659
+ # Extract error/redirection message from the response body
660
+ # Amazon claims that a redirection must have a body but somethimes it is nil....
661
+ if response.body && response.body[/^(<\?xml|<ErrorResponse)/]
662
+ error_parser = RightErrorResponseParser.new
663
+ @aws.class.bench_xml.add! do
664
+ error_parser.parse(response.body)
665
+ end
666
+ @aws.last_errors = error_parser.errors
667
+ @aws.last_request_id = error_parser.requestID
668
+ last_errors_text = @aws.last_errors.flatten.join("\n")
669
+ else
670
+ @aws.last_errors = [[response.code, "#{response.message} (#{request_text_data})"]]
671
+ @aws.last_request_id = '-undefined-'
672
+ last_errors_text = response.message
673
+ end
674
+
675
+ # Ok, it is a redirect, find the new destination location
676
+ if redirect_detected
677
+ location = response['location']
678
+ # ... log information and ...
679
+ @aws.logger.info("##### #{@aws.class.name} redirect requested: #{response.code} #{response.message} #####")
680
+ @aws.logger.info(" Old location: #{request_text_data}")
681
+ @aws.logger.info(" New location: #{location}")
682
+ # ... fix the connection data
683
+ request[:server] = URI.parse(location).host
684
+ request[:protocol] = URI.parse(location).scheme
685
+ request[:port] = URI.parse(location).port
686
+ else
687
+ # Not a redirect but an error: try to find the error in our list
688
+ @errors_list.each do |error_to_find|
689
+ if last_errors_text[/#{error_to_find}/i]
690
+ error_found = true
691
+ error_match = error_to_find
692
+ @aws.logger.warn("##### Retry is needed, error pattern match: #{error_to_find} #####")
693
+ break
694
+ end
695
+ end
696
+ end
697
+
698
+ # check the time has gone from the first error come
699
+ if redirect_detected || error_found
700
+ # Close the connection to the server and recreate a new one.
701
+ # It may have a chance that one server is a semi-down and reconnection
702
+ # will help us to connect to the other server
703
+ if !redirect_detected && @close_on_error
704
+ @aws.connection.finish "#{self.class.name}: error match to pattern '#{error_match}'"
705
+ end
706
+
707
+ if (Time.now < @stop_at)
708
+ @retries += 1
709
+ unless redirect_detected
710
+ @aws.logger.warn("##### Retry ##{@retries} is being performed. Sleeping for #{@reiteration_delay} sec. Whole time: #{Time.now-@started_at} sec ####")
711
+ sleep @reiteration_delay
712
+ @reiteration_delay *= 2
713
+
714
+ # Always make sure that the fp is set to point to the beginning(?)
715
+ # of the File/IO. TODO: it assumes that offset is 0, which is bad.
716
+ if(request[:request].body_stream && request[:request].body_stream.respond_to?(:pos))
717
+ begin
718
+ request[:request].body_stream.pos = 0
719
+ rescue Exception => e
720
+ @logger.warn("Retry may fail due to unable to reset the file pointer" +
721
+ " -- #{self.class.name} : #{e.inspect}")
722
+ end
723
+ end
724
+ else
725
+ @aws.logger.info("##### Retry ##{@retries} is being performed due to a redirect. ####")
726
+ end
727
+ result = @aws.request_info(request, @parser)
728
+ else
729
+ @aws.logger.warn("##### Ooops, time is over... ####")
730
+ end
731
+ # aha, this is unhandled error:
732
+ elsif @close_on_error
733
+ # Is this a 5xx error ?
734
+ if @aws.last_response.code.to_s[/^5\d\d$/]
735
+ @aws.connection.finish "#{self.class.name}: code: #{@aws.last_response.code}: '#{@aws.last_response.message}'"
736
+ # Is this a 4xx error ?
737
+ elsif @aws.last_response.code.to_s[/^4\d\d$/] && @close_on_4xx_probability > rand(100)
738
+ @aws.connection.finish "#{self.class.name}: code: #{@aws.last_response.code}: '#{@aws.last_response.message}', " +
739
+ "probability: #{@close_on_4xx_probability}%"
740
+ end
741
+ end
742
+ result
743
+ end
744
+
745
+ end
746
+
747
+
748
+ #-----------------------------------------------------------------
749
+
750
+ class RightSaxParserCallback #:nodoc:
751
+ def self.include_callback
752
+ include XML::SaxParser::Callbacks
753
+ end
754
+ def initialize(right_aws_parser)
755
+ @right_aws_parser = right_aws_parser
756
+ end
757
+ def on_start_element(name, attr_hash)
758
+ @right_aws_parser.tag_start(name, attr_hash)
759
+ end
760
+ def on_characters(chars)
761
+ @right_aws_parser.text(chars)
762
+ end
763
+ def on_end_element(name)
764
+ @right_aws_parser.tag_end(name)
765
+ end
766
+ def on_start_document; end
767
+ def on_comment(msg); end
768
+ def on_processing_instruction(target, data); end
769
+ def on_cdata_block(cdata); end
770
+ def on_end_document; end
771
+ end
772
+
773
+ class RightAWSParser #:nodoc:
774
+ # default parsing library
775
+ DEFAULT_XML_LIBRARY = 'rexml'
776
+ # a list of supported parsers
777
+ @@supported_xml_libs = [DEFAULT_XML_LIBRARY, 'libxml']
778
+
779
+ @@xml_lib = DEFAULT_XML_LIBRARY # xml library name: 'rexml' | 'libxml'
780
+ def self.xml_lib
781
+ @@xml_lib
782
+ end
783
+ def self.xml_lib=(new_lib_name)
784
+ @@xml_lib = new_lib_name
785
+ end
786
+
787
+ attr_accessor :result
788
+ attr_reader :xmlpath
789
+ attr_accessor :xml_lib
790
+ attr_reader :full_tag_name
791
+ attr_reader :tag
792
+
793
+ def initialize(params={})
794
+ @xmlpath = ''
795
+ @full_tag_name = ''
796
+ @result = false
797
+ @text = ''
798
+ @tag = ''
799
+ @xml_lib = params[:xml_lib] || @@xml_lib
800
+ @logger = params[:logger]
801
+ reset
802
+ end
803
+ def tag_start(name, attributes)
804
+ @text = ''
805
+ @tag = name
806
+ @full_tag_name += @full_tag_name.empty? ? name : "/#{name}"
807
+ tagstart(name, attributes)
808
+ @xmlpath = @full_tag_name
809
+ end
810
+ def tag_end(name)
811
+ @xmlpath = @full_tag_name[/^(.*?)\/?#{name}$/] && $1
812
+ tagend(name)
813
+ @full_tag_name = @xmlpath
814
+ end
815
+ def text(text)
816
+ @text += text
817
+ tagtext(text)
818
+ end
819
+ # Parser method.
820
+ # Params:
821
+ # xml_text - xml message text(String) or Net:HTTPxxx instance (response)
822
+ # params[:xml_lib] - library name: 'rexml' | 'libxml'
823
+ def parse(xml_text, params={})
824
+ # Get response body
825
+ xml_text = xml_text.body unless xml_text.is_a?(String)
826
+ @xml_lib = params[:xml_lib] || @xml_lib
827
+ # check that we had no problems with this library otherwise use default
828
+ @xml_lib = DEFAULT_XML_LIBRARY unless @@supported_xml_libs.include?(@xml_lib)
829
+ # load xml library
830
+ if @xml_lib=='libxml' && !defined?(XML::SaxParser)
831
+ begin
832
+ require 'xml/libxml'
833
+ # is it new ? - Setup SaxParserCallback
834
+ if XML::Parser::VERSION >= '0.5.1.0'
835
+ RightSaxParserCallback.include_callback
836
+ end
837
+ rescue LoadError => e
838
+ @@supported_xml_libs.delete(@xml_lib)
839
+ @xml_lib = DEFAULT_XML_LIBRARY
840
+ if @logger
841
+ @logger.error e.inspect
842
+ @logger.error e.backtrace
843
+ @logger.info "Can not load 'libxml' library. '#{DEFAULT_XML_LIBRARY}' is used for parsing."
844
+ end
845
+ end
846
+ end
847
+ # Parse the xml text
848
+ case @xml_lib
849
+ when 'libxml'
850
+ xml = XML::SaxParser.new
851
+ xml.string = xml_text
852
+ # check libxml-ruby version
853
+ if XML::Parser::VERSION >= '0.5.1.0'
854
+ xml.callbacks = RightSaxParserCallback.new(self)
855
+ else
856
+ xml.on_start_element{|name, attr_hash| self.tag_start(name, attr_hash)}
857
+ xml.on_characters{ |text| self.text(text)}
858
+ xml.on_end_element{ |name| self.tag_end(name)}
859
+ end
860
+ xml.parse
861
+ else
862
+ REXML::Document.parse_stream(xml_text, self)
863
+ end
864
+ end
865
+ # Parser must have a lots of methods
866
+ # (see /usr/lib/ruby/1.8/rexml/parsers/streamparser.rb)
867
+ # We dont need most of them in RightAWSParser and method_missing helps us
868
+ # to skip their definition
869
+ def method_missing(method, *params)
870
+ # if the method is one of known - just skip it ...
871
+ return if [:comment, :attlistdecl, :notationdecl, :elementdecl,
872
+ :entitydecl, :cdata, :xmldecl, :attlistdecl, :instruction,
873
+ :doctype].include?(method)
874
+ # ... else - call super to raise an exception
875
+ super(method, params)
876
+ end
877
+ # the functions to be overriden by children (if nessesery)
878
+ def reset ; end
879
+ def tagstart(name, attributes); end
880
+ def tagend(name) ; end
881
+ def tagtext(text) ; end
882
+ end
883
+
884
+ #-----------------------------------------------------------------
885
+ # PARSERS: Errors
886
+ #-----------------------------------------------------------------
887
+
888
+ #<Error>
889
+ # <Code>TemporaryRedirect</Code>
890
+ # <Message>Please re-send this request to the specified temporary endpoint. Continue to use the original request endpoint for future requests.</Message>
891
+ # <RequestId>FD8D5026D1C5ABA3</RequestId>
892
+ # <Endpoint>bucket-for-k.s3-external-3.amazonaws.com</Endpoint>
893
+ # <HostId>ItJy8xPFPli1fq/JR3DzQd3iDvFCRqi1LTRmunEdM1Uf6ZtW2r2kfGPWhRE1vtaU</HostId>
894
+ # <Bucket>bucket-for-k</Bucket>
895
+ #</Error>
896
+
897
+ class RightErrorResponseParser < RightAWSParser #:nodoc:
898
+ attr_accessor :errors # array of hashes: error/message
899
+ attr_accessor :requestID
900
+ # attr_accessor :endpoint, :host_id, :bucket
901
+ def tagend(name)
902
+ case name
903
+ when 'RequestID' ; @requestID = @text
904
+ when 'Code' ; @code = @text
905
+ when 'Message' ; @message = @text
906
+ # when 'Endpoint' ; @endpoint = @text
907
+ # when 'HostId' ; @host_id = @text
908
+ # when 'Bucket' ; @bucket = @text
909
+ when 'Error' ; @errors << [ @code, @message ]
910
+ end
911
+ end
912
+ def reset
913
+ @errors = []
914
+ end
915
+ end
916
+
917
+ # Dummy parser - does nothing
918
+ # Returns the original params back
919
+ class RightDummyParser # :nodoc:
920
+ attr_accessor :result
921
+ def parse(response, params={})
922
+ @result = [response, params]
923
+ end
924
+ end
925
+
926
+ class RightHttp2xxParser < RightAWSParser # :nodoc:
927
+ def parse(response)
928
+ @result = response.is_a?(Net::HTTPSuccess)
929
+ end
930
+ end
931
+
932
+ end
933
+