ahoward-helene 0.0.3

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (72) hide show
  1. data/Rakefile +274 -0
  2. data/helene.gemspec +26 -0
  3. data/lib/helene.rb +113 -0
  4. data/lib/helene/attempt.rb +46 -0
  5. data/lib/helene/aws.rb +50 -0
  6. data/lib/helene/config.rb +147 -0
  7. data/lib/helene/content_type.rb +15 -0
  8. data/lib/helene/content_type.yml +661 -0
  9. data/lib/helene/error.rb +12 -0
  10. data/lib/helene/logging.rb +55 -0
  11. data/lib/helene/objectpool.rb +220 -0
  12. data/lib/helene/rails.rb +21 -0
  13. data/lib/helene/rightscale/acf/right_acf_interface.rb +379 -0
  14. data/lib/helene/rightscale/awsbase/benchmark_fix.rb +39 -0
  15. data/lib/helene/rightscale/awsbase/right_awsbase.rb +803 -0
  16. data/lib/helene/rightscale/awsbase/support.rb +111 -0
  17. data/lib/helene/rightscale/ec2/right_ec2.rb +1737 -0
  18. data/lib/helene/rightscale/net_fix.rb +160 -0
  19. data/lib/helene/rightscale/right_aws.rb +71 -0
  20. data/lib/helene/rightscale/right_http_connection.rb +507 -0
  21. data/lib/helene/rightscale/s3/right_s3.rb +1094 -0
  22. data/lib/helene/rightscale/s3/right_s3_interface.rb +1180 -0
  23. data/lib/helene/rightscale/sdb/active_sdb.rb +930 -0
  24. data/lib/helene/rightscale/sdb/right_sdb_interface.rb +696 -0
  25. data/lib/helene/rightscale/sqs/right_sqs.rb +388 -0
  26. data/lib/helene/rightscale/sqs/right_sqs_gen2.rb +286 -0
  27. data/lib/helene/rightscale/sqs/right_sqs_gen2_interface.rb +444 -0
  28. data/lib/helene/rightscale/sqs/right_sqs_interface.rb +596 -0
  29. data/lib/helene/s3.rb +34 -0
  30. data/lib/helene/s3/bucket.rb +379 -0
  31. data/lib/helene/s3/grantee.rb +134 -0
  32. data/lib/helene/s3/key.rb +162 -0
  33. data/lib/helene/s3/owner.rb +16 -0
  34. data/lib/helene/sdb.rb +9 -0
  35. data/lib/helene/sdb/base.rb +1204 -0
  36. data/lib/helene/sdb/base/associations.rb +481 -0
  37. data/lib/helene/sdb/base/attributes.rb +90 -0
  38. data/lib/helene/sdb/base/connection.rb +20 -0
  39. data/lib/helene/sdb/base/error.rb +20 -0
  40. data/lib/helene/sdb/base/hooks.rb +82 -0
  41. data/lib/helene/sdb/base/literal.rb +52 -0
  42. data/lib/helene/sdb/base/logging.rb +23 -0
  43. data/lib/helene/sdb/base/transactions.rb +53 -0
  44. data/lib/helene/sdb/base/type.rb +137 -0
  45. data/lib/helene/sdb/base/types.rb +123 -0
  46. data/lib/helene/sdb/base/validations.rb +256 -0
  47. data/lib/helene/sdb/cast.rb +114 -0
  48. data/lib/helene/sdb/connection.rb +36 -0
  49. data/lib/helene/sdb/error.rb +5 -0
  50. data/lib/helene/sdb/interface.rb +412 -0
  51. data/lib/helene/sdb/sentinel.rb +15 -0
  52. data/lib/helene/sleepcycle.rb +29 -0
  53. data/lib/helene/superhash.rb +297 -0
  54. data/lib/helene/util.rb +132 -0
  55. data/test/auth.rb +31 -0
  56. data/test/helper.rb +98 -0
  57. data/test/integration/begin.rb +0 -0
  58. data/test/integration/ensure.rb +8 -0
  59. data/test/integration/s3/bucket.rb +106 -0
  60. data/test/integration/sdb/associations.rb +45 -0
  61. data/test/integration/sdb/creating.rb +13 -0
  62. data/test/integration/sdb/emptiness.rb +56 -0
  63. data/test/integration/sdb/hooks.rb +19 -0
  64. data/test/integration/sdb/limits.rb +27 -0
  65. data/test/integration/sdb/saving.rb +21 -0
  66. data/test/integration/sdb/selecting.rb +39 -0
  67. data/test/integration/sdb/types.rb +31 -0
  68. data/test/integration/sdb/validations.rb +60 -0
  69. data/test/integration/setup.rb +27 -0
  70. data/test/integration/teardown.rb +21 -0
  71. data/test/loader.rb +39 -0
  72. metadata +139 -0
@@ -0,0 +1,39 @@
1
+ #
2
+ # Copyright (c) 2007-2008 RightScale Inc
3
+ #
4
+ # Permission is hereby granted, free of charge, to any person obtaining
5
+ # a copy of this software and associated documentation files (the
6
+ # "Software"), to deal in the Software without restriction, including
7
+ # without limitation the rights to use, copy, modify, merge, publish,
8
+ # distribute, sublicense, and/or sell copies of the Software, and to
9
+ # permit persons to whom the Software is furnished to do so, subject to
10
+ # the following conditions:
11
+ #
12
+ # The above copyright notice and this permission notice shall be
13
+ # included in all copies or substantial portions of the Software.
14
+ #
15
+ # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
16
+ # EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
17
+ # MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
18
+ # NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
19
+ # LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
20
+ # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
21
+ # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
22
+ #
23
+ #
24
+
25
+
26
+ # A hack because there's a bug in add! in Benchmark::Tms
27
+ module Benchmark #:nodoc:
28
+ class Tms #:nodoc:
29
+ def add!(&blk)
30
+ t = Benchmark::measure(&blk)
31
+ @utime = utime + t.utime
32
+ @stime = stime + t.stime
33
+ @cutime = cutime + t.cutime
34
+ @cstime = cstime + t.cstime
35
+ @real = real + t.real
36
+ self
37
+ end
38
+ end
39
+ end
@@ -0,0 +1,803 @@
1
+ #
2
+ # Copyright (c) 2007-2008 RightScale Inc
3
+ #
4
+ # Permission is hereby granted, free of charge, to any person obtaining
5
+ # a copy of this software and associated documentation files (the
6
+ # "Software"), to deal in the Software without restriction, including
7
+ # without limitation the rights to use, copy, modify, merge, publish,
8
+ # distribute, sublicense, and/or sell copies of the Software, and to
9
+ # permit persons to whom the Software is furnished to do so, subject to
10
+ # the following conditions:
11
+ #
12
+ # The above copyright notice and this permission notice shall be
13
+ # included in all copies or substantial portions of the Software.
14
+ #
15
+ # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
16
+ # EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
17
+ # MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
18
+ # NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
19
+ # LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
20
+ # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
21
+ # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
22
+ #
23
+
24
+ # Test
25
+ module RightAws
26
+ require 'md5'
27
+ require 'pp'
28
+
29
+ class AwsUtils #:nodoc:
30
+ @@digest1 = OpenSSL::Digest::Digest.new("sha1")
31
+ @@digest256 = nil
32
+ if OpenSSL::OPENSSL_VERSION_NUMBER > 0x00908000
33
+ @@digest256 = OpenSSL::Digest::Digest.new("sha256") rescue nil # Some installation may not support sha256
34
+ end
35
+
36
+ def self.sign(aws_secret_access_key, auth_string)
37
+ Base64.encode64(OpenSSL::HMAC.digest(@@digest1, aws_secret_access_key, auth_string)).strip
38
+ end
39
+
40
+ # Escape a string accordingly Amazon rulles
41
+ # http://docs.amazonwebservices.com/AmazonSimpleDB/2007-11-07/DeveloperGuide/index.html?REST_RESTAuth.html
42
+ def self.amz_escape(param)
43
+ param.to_s.gsub(/([^a-zA-Z0-9._~-]+)/n) do
44
+ '%' + $1.unpack('H2' * $1.size).join('%').upcase
45
+ end
46
+ end
47
+
48
+ # Set a timestamp and a signature version
49
+ def self.fix_service_params(service_hash, signature)
50
+ service_hash["Timestamp"] ||= Time.now.utc.strftime("%Y-%m-%dT%H:%M:%S.000Z") unless service_hash["Expires"]
51
+ service_hash["SignatureVersion"] = signature
52
+ service_hash
53
+ end
54
+
55
+ # Signature Version 0
56
+ # A deprecated guy (should work till septemper 2009)
57
+ def self.sign_request_v0(aws_secret_access_key, service_hash)
58
+ fix_service_params(service_hash, '0')
59
+ string_to_sign = "#{service_hash['Action']}#{service_hash['Timestamp'] || service_hash['Expires']}"
60
+ service_hash['Signature'] = AwsUtils::sign(aws_secret_access_key, string_to_sign)
61
+ service_hash.to_a.collect{|key,val| "#{amz_escape(key)}=#{amz_escape(val.to_s)}" }.join("&")
62
+ end
63
+
64
+ # Signature Version 1
65
+ # Another deprecated guy (should work till septemper 2009)
66
+ def self.sign_request_v1(aws_secret_access_key, service_hash)
67
+ fix_service_params(service_hash, '1')
68
+ string_to_sign = service_hash.sort{|a,b| (a[0].to_s.downcase)<=>(b[0].to_s.downcase)}.to_s
69
+ service_hash['Signature'] = AwsUtils::sign(aws_secret_access_key, string_to_sign)
70
+ service_hash.to_a.collect{|key,val| "#{amz_escape(key)}=#{amz_escape(val.to_s)}" }.join("&")
71
+ end
72
+
73
+ # Signature Version 2
74
+ # EC2, SQS and SDB requests must be signed by this guy.
75
+ # See: http://docs.amazonwebservices.com/AmazonSimpleDB/2007-11-07/DeveloperGuide/index.html?REST_RESTAuth.html
76
+ # http://developer.amazonwebservices.com/connect/entry.jspa?externalID=1928
77
+ def self.sign_request_v2(aws_secret_access_key, service_hash, http_verb, host, uri)
78
+ fix_service_params(service_hash, '2')
79
+ # select a signing method (make an old openssl working with sha1)
80
+ # make 'HmacSHA256' to be a default one
81
+ service_hash['SignatureMethod'] = 'HmacSHA256' unless ['HmacSHA256', 'HmacSHA1'].include?(service_hash['SignatureMethod'])
82
+ service_hash['SignatureMethod'] = 'HmacSHA1' unless @@digest256
83
+ # select a digest
84
+ digest = (service_hash['SignatureMethod'] == 'HmacSHA256' ? @@digest256 : @@digest1)
85
+ # form string to sign
86
+ canonical_string = service_hash.keys.sort.map do |key|
87
+ "#{amz_escape(key)}=#{amz_escape(service_hash[key])}"
88
+ end.join('&')
89
+ string_to_sign = "#{http_verb.to_s.upcase}\n#{host.downcase}\n#{uri}\n#{canonical_string}"
90
+ # sign the string
91
+ signature = amz_escape(Base64.encode64(OpenSSL::HMAC.digest(digest, aws_secret_access_key, string_to_sign)).strip)
92
+ "#{canonical_string}&Signature=#{signature}"
93
+ end
94
+
95
+ # From Amazon's SQS Dev Guide, a brief description of how to escape:
96
+ # "URL encode the computed signature and other query parameters as specified in
97
+ # RFC1738, section 2.2. In addition, because the + character is interpreted as a blank space
98
+ # by Sun Java classes that perform URL decoding, make sure to encode the + character
99
+ # although it is not required by RFC1738."
100
+ # Avoid using CGI::escape to escape URIs.
101
+ # CGI::escape will escape characters in the protocol, host, and port
102
+ # sections of the URI. Only target chars in the query
103
+ # string should be escaped.
104
+ def self.URLencode(raw)
105
+ e = URI.escape(raw)
106
+ e.gsub(/\+/, "%2b")
107
+ end
108
+
109
+ def self.allow_only(allowed_keys, params)
110
+ bogus_args = []
111
+ params.keys.each {|p| bogus_args.push(p) unless allowed_keys.include?(p) }
112
+ raise AwsError.new("The following arguments were given but are not legal for the function call #{caller_method}: #{bogus_args.inspect}") if bogus_args.length > 0
113
+ end
114
+
115
+ def self.mandatory_arguments(required_args, params)
116
+ rargs = required_args.dup
117
+ params.keys.each {|p| rargs.delete(p)}
118
+ raise AwsError.new("The following mandatory arguments were not provided to #{caller_method}: #{rargs.inspect}") if rargs.length > 0
119
+ end
120
+
121
+ def self.caller_method
122
+ caller[1]=~/`(.*?)'/
123
+ $1
124
+ end
125
+
126
+ end
127
+
128
+ class AwsBenchmarkingBlock #:nodoc:
129
+ attr_accessor :xml, :service
130
+ def initialize
131
+ # Benchmark::Tms instance for service (Ec2, S3, or SQS) access benchmarking.
132
+ @service = Benchmark::Tms.new()
133
+ # Benchmark::Tms instance for XML parsing benchmarking.
134
+ @xml = Benchmark::Tms.new()
135
+ end
136
+ end
137
+
138
+ class AwsNoChange < RuntimeError
139
+ end
140
+
141
+ class RightAwsBase
142
+
143
+ # Amazon HTTP Error handling
144
+
145
+ # Text, if found in an error message returned by AWS, indicates that this may be a transient
146
+ # error. Transient errors are automatically retried with exponential back-off.
147
+ AMAZON_PROBLEMS = [ 'internal service error',
148
+ 'is currently unavailable',
149
+ 'no response from',
150
+ 'Please try again',
151
+ 'InternalError',
152
+ 'ServiceUnavailable', #from SQS docs
153
+ 'Unavailable',
154
+ 'This application is not currently available',
155
+ 'InsufficientInstanceCapacity'
156
+ ]
157
+ @@amazon_problems = AMAZON_PROBLEMS
158
+ # Returns a list of Amazon service responses which are known to be transient problems.
159
+ # We have to re-request if we get any of them, because the problem will probably disappear.
160
+ # By default this method returns the same value as the AMAZON_PROBLEMS const.
161
+ def self.amazon_problems
162
+ @@amazon_problems
163
+ end
164
+
165
+ # Sets the list of Amazon side problems. Use in conjunction with the
166
+ # getter to append problems.
167
+ def self.amazon_problems=(problems_list)
168
+ @@amazon_problems = problems_list
169
+ end
170
+
171
+ end
172
+
173
+ module RightAwsBaseInterface
174
+ module NullLogger
175
+ def respond_to?(*a, &b) true end
176
+ def method_missing(m, *a, &b) end
177
+ extend self
178
+ end
179
+
180
+ DEFAULT_SIGNATURE_VERSION = '2'
181
+
182
+ @@caching = false
183
+ def self.caching
184
+ @@caching
185
+ end
186
+ def self.caching=(caching)
187
+ @@caching = caching
188
+ end
189
+
190
+ # Current aws_access_key_id
191
+ attr_reader :aws_access_key_id
192
+ # Last HTTP request object
193
+ attr_reader :last_request
194
+ # Last HTTP response object
195
+ attr_reader :last_response
196
+ # Last AWS errors list (used by AWSErrorHandler)
197
+ attr_accessor :last_errors
198
+ # Last AWS request id (used by AWSErrorHandler)
199
+ attr_accessor :last_request_id
200
+ # Logger object
201
+ attr_accessor :logger
202
+ # Initial params hash
203
+ attr_accessor :params
204
+ # RightHttpConnection instance
205
+ attr_reader :connection
206
+ # Cache
207
+ attr_reader :cache
208
+ # Signature version (all services except s3)
209
+ attr_reader :signature_version
210
+
211
+ def init(service_info, aws_access_key_id, aws_secret_access_key, params={}) #:nodoc:
212
+ @params = params
213
+ raise AwsError.new("AWS access keys are required to operate on #{service_info[:name]}") \
214
+ if aws_access_key_id.blank? || aws_secret_access_key.blank?
215
+ @aws_access_key_id = aws_access_key_id
216
+ @aws_secret_access_key = aws_secret_access_key
217
+ # if the endpoint was explicitly defined - then use it
218
+ if @params[:endpoint_url]
219
+ @params[:server] = URI.parse(@params[:endpoint_url]).host
220
+ @params[:port] = URI.parse(@params[:endpoint_url]).port
221
+ @params[:service] = URI.parse(@params[:endpoint_url]).path
222
+ @params[:protocol] = URI.parse(@params[:endpoint_url]).scheme
223
+ @params[:region] = nil
224
+ else
225
+ @params[:server] ||= service_info[:default_host]
226
+ @params[:server] = "#{@params[:region]}.#{@params[:server]}" if @params[:region]
227
+ @params[:port] ||= service_info[:default_port]
228
+ @params[:service] ||= service_info[:default_service]
229
+ @params[:protocol] ||= service_info[:default_protocol]
230
+ end
231
+ @params[:multi_thread] ||= defined?(AWS_DAEMON)
232
+ @logger = @params[:logger]
233
+ @logger ||= RAILS_DEFAULT_LOGGER if defined?(RAILS_DEFAULT_LOGGER)
234
+ @logger ||= NullLogger
235
+ @logger.info "New #{self.class.name} using #{@params[:multi_thread] ? 'multi' : 'single'}-threaded mode"
236
+ @error_handler = nil
237
+ @cache = {}
238
+ @signature_version = (params[:signature_version] || DEFAULT_SIGNATURE_VERSION).to_s
239
+ end
240
+
241
+ def signed_service_params(aws_secret_access_key, service_hash, http_verb=nil, host=nil, service=nil )
242
+ case signature_version.to_s
243
+ when '0' then AwsUtils::sign_request_v0(aws_secret_access_key, service_hash)
244
+ when '1' then AwsUtils::sign_request_v1(aws_secret_access_key, service_hash)
245
+ when '2' then AwsUtils::sign_request_v2(aws_secret_access_key, service_hash, http_verb, host, service)
246
+ else raise AwsError.new("Unknown signature version (#{signature_version.to_s}) requested")
247
+ end
248
+ end
249
+
250
+ # Returns +true+ if the describe_xxx responses are being cached
251
+ def caching?
252
+ @params.key?(:cache) ? @params[:cache] : @@caching
253
+ end
254
+
255
+ # Check if the aws function response hits the cache or not.
256
+ # If the cache hits:
257
+ # - raises an +AwsNoChange+ exception if +do_raise+ == +:raise+.
258
+ # - returnes parsed response from the cache if it exists or +true+ otherwise.
259
+ # If the cache miss or the caching is off then returns +false+.
260
+ def cache_hits?(function, response, do_raise=:raise)
261
+ result = false
262
+ if caching?
263
+ function = function.to_sym
264
+ # get rid of requestId (this bad boy was added for API 2008-08-08+ and it is uniq for every response)
265
+ response = response.sub(%r{<requestId>.+?</requestId>}, '')
266
+ response_md5 = MD5.md5(response).to_s
267
+ # check for changes
268
+ unless @cache[function] && @cache[function][:response_md5] == response_md5
269
+ # well, the response is new, reset cache data
270
+ update_cache(function, {:response_md5 => response_md5,
271
+ :timestamp => Time.now,
272
+ :hits => 0,
273
+ :parsed => nil})
274
+ else
275
+ # aha, cache hits, update the data and throw an exception if needed
276
+ @cache[function][:hits] += 1
277
+ if do_raise == :raise
278
+ raise(AwsNoChange, "Cache hit: #{function} response has not changed since "+
279
+ "#{@cache[function][:timestamp].strftime('%Y-%m-%d %H:%M:%S')}, "+
280
+ "hits: #{@cache[function][:hits]}.")
281
+ else
282
+ result = @cache[function][:parsed] || true
283
+ end
284
+ end
285
+ end
286
+ result
287
+ end
288
+
289
+ def update_cache(function, hash)
290
+ (@cache[function.to_sym] ||= {}).merge!(hash) if caching?
291
+ end
292
+
293
+ def on_exception(options={:raise=>true, :log=>true}) # :nodoc:
294
+ raise if $!.is_a?(AwsNoChange)
295
+ AwsError::on_aws_exception(self, options)
296
+ end
297
+
298
+ # Return +true+ if this instance works in multi_thread mode and +false+ otherwise.
299
+ def multi_thread
300
+ @params[:multi_thread]
301
+ end
302
+
303
+ def request_info_impl(connection, benchblock, request, parser, &block) #:nodoc:
304
+ @connection = connection
305
+ @last_request = request[:request]
306
+ @last_response = nil
307
+ response=nil
308
+ blockexception = nil
309
+
310
+ if(block != nil)
311
+ # TRB 9/17/07 Careful - because we are passing in blocks, we get a situation where
312
+ # an exception may get thrown in the block body (which is high-level
313
+ # code either here or in the application) but gets caught in the
314
+ # low-level code of HttpConnection. The solution is not to let any
315
+ # exception escape the block that we pass to HttpConnection::request.
316
+ # Exceptions can originate from code directly in the block, or from user
317
+ # code called in the other block which is passed to response.read_body.
318
+ benchblock.service.add! do
319
+ responsehdr = @connection.request(request) do |response|
320
+ #########
321
+ begin
322
+ @last_response = response
323
+ if response.is_a?(Net::HTTPSuccess)
324
+ @error_handler = nil
325
+ response.read_body(&block)
326
+ else
327
+ @error_handler = AWSErrorHandler.new(self, parser, :errors_list => self.class.amazon_problems) unless @error_handler
328
+ check_result = @error_handler.check(request)
329
+ if check_result
330
+ @error_handler = nil
331
+ return check_result
332
+ end
333
+ raise AwsError.new(@last_errors, @last_response.code, @last_request_id)
334
+ end
335
+ rescue Exception => e
336
+ blockexception = e
337
+ end
338
+ end
339
+ #########
340
+
341
+ #OK, now we are out of the block passed to the lower level
342
+ if(blockexception)
343
+ raise blockexception
344
+ end
345
+ benchblock.xml.add! do
346
+ parser.parse(responsehdr)
347
+ end
348
+ return parser.result
349
+ end
350
+ else
351
+ benchblock.service.add!{ response = @connection.request(request) }
352
+ # check response for errors...
353
+ @last_response = response
354
+ if response.is_a?(Net::HTTPSuccess)
355
+ @error_handler = nil
356
+ benchblock.xml.add! { parser.parse(response) }
357
+ return parser.result
358
+ else
359
+ @error_handler = AWSErrorHandler.new(self, parser, :errors_list => self.class.amazon_problems) unless @error_handler
360
+ check_result = @error_handler.check(request)
361
+ if check_result
362
+ @error_handler = nil
363
+ return check_result
364
+ end
365
+ raise AwsError.new(@last_errors, @last_response.code, @last_request_id)
366
+ end
367
+ end
368
+ rescue
369
+ @error_handler = nil
370
+ raise
371
+ end
372
+
373
+ def request_cache_or_info(method, link, parser_class, benchblock, use_cache=true) #:nodoc:
374
+ # We do not want to break the logic of parsing hence will use a dummy parser to process all the standard
375
+ # steps (errors checking etc). The dummy parser does nothig - just returns back the params it received.
376
+ # If the caching is enabled and hit then throw AwsNoChange.
377
+ # P.S. caching works for the whole images list only! (when the list param is blank)
378
+ # check cache
379
+ response, params = request_info(link, RightDummyParser.new)
380
+ cache_hits?(method.to_sym, response.body) if use_cache
381
+ parser = parser_class.new(:logger => @logger)
382
+ benchblock.xml.add!{ parser.parse(response, params) }
383
+ result = block_given? ? yield(parser) : parser.result
384
+ # update parsed data
385
+ update_cache(method.to_sym, :parsed => result) if use_cache
386
+ result
387
+ end
388
+
389
+ # Returns Amazons request ID for the latest request
390
+ def last_request_id
391
+ @last_response && @last_response.body.to_s[%r{<requestId>(.+?)</requestId>}] && $1
392
+ end
393
+
394
+ end
395
+
396
+
397
+ # Exception class to signal any Amazon errors. All errors occuring during calls to Amazon's
398
+ # web services raise this type of error.
399
+ # Attribute inherited by RuntimeError:
400
+ # message - the text of the error, generally as returned by AWS in its XML response.
401
+ class AwsError < RuntimeError
402
+
403
+ # either an array of errors where each item is itself an array of [code, message]),
404
+ # or an error string if the error was raised manually, as in <tt>AwsError.new('err_text')</tt>
405
+ attr_reader :errors
406
+
407
+ # Request id (if exists)
408
+ attr_reader :request_id
409
+
410
+ # Response HTTP error code
411
+ attr_reader :http_code
412
+
413
+ def initialize(errors=nil, http_code=nil, request_id=nil)
414
+ @errors = errors
415
+ @request_id = request_id
416
+ @http_code = http_code
417
+ super(@errors.is_a?(Array) ? @errors.map{|code, msg| "#{code}: #{msg}"}.join("; ") : @errors.to_s)
418
+ end
419
+
420
+ # Does any of the error messages include the regexp +pattern+?
421
+ # Used to determine whether to retry request.
422
+ def include?(pattern)
423
+ if @errors.is_a?(Array)
424
+ @errors.each{ |code, msg| return true if code =~ pattern }
425
+ else
426
+ return true if @errors_str =~ pattern
427
+ end
428
+ false
429
+ end
430
+
431
+ # Generic handler for AwsErrors. +aws+ is the RightAws::S3, RightAws::EC2, or RightAws::SQS
432
+ # object that caused the exception (it must provide last_request and last_response). Supported
433
+ # boolean options are:
434
+ # * <tt>:log</tt> print a message into the log using aws.logger to access the Logger
435
+ # * <tt>:puts</tt> do a "puts" of the error
436
+ # * <tt>:raise</tt> re-raise the error after logging
437
+ def self.on_aws_exception(aws, options={:raise=>true, :log=>true})
438
+ # Only log & notify if not user error
439
+ if !options[:raise] || system_error?($!)
440
+ error_text = "#{$!.inspect}\n#{$@}.join('\n')}"
441
+ puts error_text if options[:puts]
442
+ # Log the error
443
+ if options[:log]
444
+ request = aws.last_request ? aws.last_request.path : '-none-'
445
+ response = aws.last_response ? "#{aws.last_response.code} -- #{aws.last_response.message} -- #{aws.last_response.body}" : '-none-'
446
+ aws.logger.error error_text
447
+ aws.logger.error "Request was: #{request}"
448
+ aws.logger.error "Response was: #{response}"
449
+ end
450
+ end
451
+ raise if options[:raise] # re-raise an exception
452
+ return nil
453
+ end
454
+
455
+ # True if e is an AWS system error, i.e. something that is for sure not the caller's fault.
456
+ # Used to force logging.
457
+ def self.system_error?(e)
458
+ !e.is_a?(self) || e.message =~ /InternalError|InsufficientInstanceCapacity|Unavailable/
459
+ end
460
+
461
+ end
462
+
463
+
464
+ class AWSErrorHandler
465
+ # 0-100 (%)
466
+ DEFAULT_CLOSE_ON_4XX_PROBABILITY = 10
467
+
468
+ @@reiteration_start_delay = 0.2
469
+ def self.reiteration_start_delay
470
+ @@reiteration_start_delay
471
+ end
472
+ def self.reiteration_start_delay=(reiteration_start_delay)
473
+ @@reiteration_start_delay = reiteration_start_delay
474
+ end
475
+
476
+ @@reiteration_time = 5
477
+ def self.reiteration_time
478
+ @@reiteration_time
479
+ end
480
+ def self.reiteration_time=(reiteration_time)
481
+ @@reiteration_time = reiteration_time
482
+ end
483
+
484
+ @@close_on_error = true
485
+ def self.close_on_error
486
+ @@close_on_error
487
+ end
488
+ def self.close_on_error=(close_on_error)
489
+ @@close_on_error = close_on_error
490
+ end
491
+
492
+ @@close_on_4xx_probability = DEFAULT_CLOSE_ON_4XX_PROBABILITY
493
+ def self.close_on_4xx_probability
494
+ @@close_on_4xx_probability
495
+ end
496
+ def self.close_on_4xx_probability=(close_on_4xx_probability)
497
+ @@close_on_4xx_probability = close_on_4xx_probability
498
+ end
499
+
500
+ # params:
501
+ # :reiteration_time
502
+ # :errors_list
503
+ # :close_on_error = true | false
504
+ # :close_on_4xx_probability = 1-100
505
+ def initialize(aws, parser, params={}) #:nodoc:
506
+ @aws = aws # Link to RightEc2 | RightSqs | RightS3 instance
507
+ @parser = parser # parser to parse Amazon response
508
+ @started_at = Time.now
509
+ @stop_at = @started_at + (params[:reiteration_time] || @@reiteration_time)
510
+ @errors_list = params[:errors_list] || []
511
+ @reiteration_delay = @@reiteration_start_delay
512
+ @retries = 0
513
+ # close current HTTP(S) connection on 5xx, errors from list and 4xx errors
514
+ @close_on_error = params[:close_on_error].nil? ? @@close_on_error : params[:close_on_error]
515
+ @close_on_4xx_probability = params[:close_on_4xx_probability] || @@close_on_4xx_probability
516
+ end
517
+
518
+ # Returns false if
519
+ def check(request) #:nodoc:
520
+ result = false
521
+ error_found = false
522
+ redirect_detected= false
523
+ error_match = nil
524
+ last_errors_text = ''
525
+ response = @aws.last_response
526
+ # log error
527
+ request_text_data = "#{request[:server]}:#{request[:port]}#{request[:request].path}"
528
+ # is this a redirect?
529
+ # yes!
530
+ if response.is_a?(Net::HTTPRedirection)
531
+ redirect_detected = true
532
+ else
533
+ # no, it's an error ...
534
+ @aws.logger.warn("##### #{@aws.class.name} returned an error: #{response.code} #{response.message}\n#{response.body} #####")
535
+ @aws.logger.warn("##### #{@aws.class.name} request: #{request_text_data} ####")
536
+ end
537
+ # Check response body: if it is an Amazon XML document or not:
538
+ if redirect_detected || (response.body && response.body[/<\?xml/]) # ... it is a xml document
539
+ @aws.class.bench_xml.add! do
540
+ error_parser = RightErrorResponseParser.new
541
+ error_parser.parse(response)
542
+ @aws.last_errors = error_parser.errors
543
+ @aws.last_request_id = error_parser.requestID
544
+ last_errors_text = @aws.last_errors.flatten.join("\n")
545
+ # on redirect :
546
+ if redirect_detected
547
+ location = response['location']
548
+ # ... log information and ...
549
+ @aws.logger.info("##### #{@aws.class.name} redirect requested: #{response.code} #{response.message} #####")
550
+ @aws.logger.info("##### New location: #{location} #####")
551
+ # ... fix the connection data
552
+ request[:server] = URI.parse(location).host
553
+ request[:protocol] = URI.parse(location).scheme
554
+ request[:port] = URI.parse(location).port
555
+ end
556
+ end
557
+ else # ... it is not a xml document(probably just a html page?)
558
+ @aws.last_errors = [[response.code, "#{response.message} (#{request_text_data})"]]
559
+ @aws.last_request_id = '-undefined-'
560
+ last_errors_text = response.message
561
+ end
562
+ # now - check the error
563
+ unless redirect_detected
564
+ @errors_list.each do |error_to_find|
565
+ if last_errors_text[/#{error_to_find}/i]
566
+ error_found = true
567
+ error_match = error_to_find
568
+ @aws.logger.warn("##### Retry is needed, error pattern match: #{error_to_find} #####")
569
+ break
570
+ end
571
+ end
572
+ end
573
+ # check the time has gone from the first error come
574
+ if redirect_detected || error_found
575
+ # Close the connection to the server and recreate a new one.
576
+ # It may have a chance that one server is a semi-down and reconnection
577
+ # will help us to connect to the other server
578
+ if !redirect_detected && @close_on_error
579
+ @aws.connection.finish "#{self.class.name}: error match to pattern '#{error_match}'"
580
+ end
581
+
582
+ if (Time.now < @stop_at)
583
+ @retries += 1
584
+ unless redirect_detected
585
+ @aws.logger.warn("##### Retry ##{@retries} is being performed. Sleeping for #{@reiteration_delay} sec. Whole time: #{Time.now-@started_at} sec ####")
586
+ sleep @reiteration_delay
587
+ @reiteration_delay *= 2
588
+
589
+ # Always make sure that the fp is set to point to the beginning(?)
590
+ # of the File/IO. TODO: it assumes that offset is 0, which is bad.
591
+ if(request[:request].body_stream && request[:request].body_stream.respond_to?(:pos))
592
+ begin
593
+ request[:request].body_stream.pos = 0
594
+ rescue Exception => e
595
+ @logger.warn("Retry may fail due to unable to reset the file pointer" +
596
+ " -- #{self.class.name} : #{e.inspect}")
597
+ end
598
+ end
599
+ else
600
+ @aws.logger.info("##### Retry ##{@retries} is being performed due to a redirect. ####")
601
+ end
602
+ result = @aws.request_info(request, @parser)
603
+ else
604
+ @aws.logger.warn("##### Ooops, time is over... ####")
605
+ end
606
+ # aha, this is unhandled error:
607
+ elsif @close_on_error
608
+ # Is this a 5xx error ?
609
+ if @aws.last_response.code.to_s[/^5\d\d$/]
610
+ @aws.connection.finish "#{self.class.name}: code: #{@aws.last_response.code}: '#{@aws.last_response.message}'"
611
+ # Is this a 4xx error ?
612
+ elsif @aws.last_response.code.to_s[/^4\d\d$/] && @close_on_4xx_probability > rand(100)
613
+ @aws.connection.finish "#{self.class.name}: code: #{@aws.last_response.code}: '#{@aws.last_response.message}', " +
614
+ "probability: #{@close_on_4xx_probability}%"
615
+ end
616
+ end
617
+ result
618
+ end
619
+
620
+ end
621
+
622
+
623
+ #-----------------------------------------------------------------
624
+
625
+ class RightSaxParserCallback #:nodoc:
626
+ def self.include_callback
627
+ include XML::SaxParser::Callbacks
628
+ end
629
+ def initialize(right_aws_parser)
630
+ @right_aws_parser = right_aws_parser
631
+ end
632
+ def on_start_element(name, attr_hash)
633
+ @right_aws_parser.tag_start(name, attr_hash)
634
+ end
635
+ def on_characters(chars)
636
+ @right_aws_parser.text(chars)
637
+ end
638
+ def on_end_element(name)
639
+ @right_aws_parser.tag_end(name)
640
+ end
641
+ def on_start_document; end
642
+ def on_comment(msg); end
643
+ def on_processing_instruction(target, data); end
644
+ def on_cdata_block(cdata); end
645
+ def on_end_document; end
646
+ end
647
+
648
+ class RightAWSParser #:nodoc:
649
+ # default parsing library
650
+ DEFAULT_XML_LIBRARY = 'rexml'
651
+ # a list of supported parsers
652
+ @@supported_xml_libs = [DEFAULT_XML_LIBRARY, 'libxml']
653
+
654
+ @@xml_lib = DEFAULT_XML_LIBRARY # xml library name: 'rexml' | 'libxml'
655
+ def self.xml_lib
656
+ @@xml_lib
657
+ end
658
+ def self.xml_lib=(new_lib_name)
659
+ @@xml_lib = new_lib_name
660
+ end
661
+
662
+ attr_accessor :result
663
+ attr_reader :xmlpath
664
+ attr_accessor :xml_lib
665
+
666
+ def initialize(params={})
667
+ @xmlpath = ''
668
+ @result = false
669
+ @text = ''
670
+ @xml_lib = params[:xml_lib] || @@xml_lib
671
+ @logger = params[:logger]
672
+ reset
673
+ end
674
+ def tag_start(name, attributes)
675
+ @text = ''
676
+ tagstart(name, attributes)
677
+ @xmlpath += @xmlpath.empty? ? name : "/#{name}"
678
+ end
679
+ def tag_end(name)
680
+ if @xmlpath =~ /^(.*?)\/?#{name}$/
681
+ @xmlpath = $1
682
+ end
683
+ tagend(name)
684
+ end
685
+ def text(text)
686
+ @text += text
687
+ tagtext(text)
688
+ end
689
+ # Parser method.
690
+ # Params:
691
+ # xml_text - xml message text(String) or Net:HTTPxxx instance (response)
692
+ # params[:xml_lib] - library name: 'rexml' | 'libxml'
693
+ def parse(xml_text, params={})
694
+ # Get response body
695
+ xml_text = xml_text.body unless xml_text.is_a?(String)
696
+ @xml_lib = params[:xml_lib] || @xml_lib
697
+ # check that we had no problems with this library otherwise use default
698
+ @xml_lib = DEFAULT_XML_LIBRARY unless @@supported_xml_libs.include?(@xml_lib)
699
+ # load xml library
700
+ if @xml_lib=='libxml' && !defined?(XML::SaxParser)
701
+ begin
702
+ require 'xml/libxml'
703
+ # is it new ? - Setup SaxParserCallback
704
+ if XML::Parser::VERSION >= '0.5.1.0'
705
+ RightSaxParserCallback.include_callback
706
+ end
707
+ rescue LoadError => e
708
+ @@supported_xml_libs.delete(@xml_lib)
709
+ @xml_lib = DEFAULT_XML_LIBRARY
710
+ if @logger
711
+ @logger.error e.inspect
712
+ @logger.error e.backtrace
713
+ @logger.info "Can not load 'libxml' library. '#{DEFAULT_XML_LIBRARY}' is used for parsing."
714
+ end
715
+ end
716
+ end
717
+ # Parse the xml text
718
+ case @xml_lib
719
+ when 'libxml'
720
+ xml = XML::SaxParser.new
721
+ xml.string = xml_text
722
+ # check libxml-ruby version
723
+ if XML::Parser::VERSION >= '0.5.1.0'
724
+ xml.callbacks = RightSaxParserCallback.new(self)
725
+ else
726
+ xml.on_start_element{|name, attr_hash| self.tag_start(name, attr_hash)}
727
+ xml.on_characters{ |text| self.text(text)}
728
+ xml.on_end_element{ |name| self.tag_end(name)}
729
+ end
730
+ xml.parse
731
+ else
732
+ REXML::Document.parse_stream(xml_text, self)
733
+ end
734
+ end
735
+ # Parser must have a lots of methods
736
+ # (see /usr/lib/ruby/1.8/rexml/parsers/streamparser.rb)
737
+ # We dont need most of them in RightAWSParser and method_missing helps us
738
+ # to skip their definition
739
+ def method_missing(method, *params)
740
+ # if the method is one of known - just skip it ...
741
+ return if [:comment, :attlistdecl, :notationdecl, :elementdecl,
742
+ :entitydecl, :cdata, :xmldecl, :attlistdecl, :instruction,
743
+ :doctype].include?(method)
744
+ # ... else - call super to raise an exception
745
+ super(method, params)
746
+ end
747
+ # the functions to be overriden by children (if nessesery)
748
+ def reset ; end
749
+ def tagstart(name, attributes); end
750
+ def tagend(name) ; end
751
+ def tagtext(text) ; end
752
+ end
753
+
754
+ #-----------------------------------------------------------------
755
+ # PARSERS: Errors
756
+ #-----------------------------------------------------------------
757
+
758
+ #<Error>
759
+ # <Code>TemporaryRedirect</Code>
760
+ # <Message>Please re-send this request to the specified temporary endpoint. Continue to use the original request endpoint for future requests.</Message>
761
+ # <RequestId>FD8D5026D1C5ABA3</RequestId>
762
+ # <Endpoint>bucket-for-k.s3-external-3.amazonaws.com</Endpoint>
763
+ # <HostId>ItJy8xPFPli1fq/JR3DzQd3iDvFCRqi1LTRmunEdM1Uf6ZtW2r2kfGPWhRE1vtaU</HostId>
764
+ # <Bucket>bucket-for-k</Bucket>
765
+ #</Error>
766
+
767
+ class RightErrorResponseParser < RightAWSParser #:nodoc:
768
+ attr_accessor :errors # array of hashes: error/message
769
+ attr_accessor :requestID
770
+ # attr_accessor :endpoint, :host_id, :bucket
771
+ def tagend(name)
772
+ case name
773
+ when 'RequestID' ; @requestID = @text
774
+ when 'Code' ; @code = @text
775
+ when 'Message' ; @message = @text
776
+ # when 'Endpoint' ; @endpoint = @text
777
+ # when 'HostId' ; @host_id = @text
778
+ # when 'Bucket' ; @bucket = @text
779
+ when 'Error' ; @errors << [ @code, @message ]
780
+ end
781
+ end
782
+ def reset
783
+ @errors = []
784
+ end
785
+ end
786
+
787
+ # Dummy parser - does nothing
788
+ # Returns the original params back
789
+ class RightDummyParser # :nodoc:
790
+ attr_accessor :result
791
+ def parse(response, params={})
792
+ @result = [response, params]
793
+ end
794
+ end
795
+
796
+ class RightHttp2xxParser < RightAWSParser # :nodoc:
797
+ def parse(response)
798
+ @result = response.is_a?(Net::HTTPSuccess)
799
+ end
800
+ end
801
+
802
+ end
803
+