appoxy-aws 1.11.34 → 1.11.35

Sign up to get free protection for your applications and to get access to all the features.
Files changed (5) hide show
  1. data/lib/awsbase/right_awsbase.rb +759 -702
  2. data/lib/ec2/right_mon_interface.rb +222 -0
  3. data/lib/sdb/active_sdb.rb +1 -0
  4. data/lib/sdb/right_sdb_interface.rb +5 -2
  5. metadata +3 -2
data/lib/awsbase/right_awsbase.rb CHANGED
@@ -23,749 +23,800 @@
23
23
 
24
24
  # Test
25
25
  module RightAws
26
- require 'digest/md5'
27
- require 'pp'
28
- require 'cgi'
29
-
30
- class AwsUtils #:nodoc:
31
- @@digest1 = OpenSSL::Digest::Digest.new("sha1")
32
- @@digest256 = nil
33
- if OpenSSL::OPENSSL_VERSION_NUMBER > 0x00908000
34
- @@digest256 = OpenSSL::Digest::Digest.new("sha256") rescue nil # Some installation may not support sha256
35
- end
26
+ require 'digest/md5'
27
+ require 'pp'
28
+ require 'cgi'
29
+ require 'uri'
30
+
31
+ class AwsUtils #:nodoc:
32
+ @@digest1 = OpenSSL::Digest::Digest.new("sha1")
33
+ @@digest256 = nil
34
+ if OpenSSL::OPENSSL_VERSION_NUMBER > 0x00908000
35
+ @@digest256 = OpenSSL::Digest::Digest.new("sha256") rescue nil # Some installation may not support sha256
36
+ end
36
37
 
37
- def self.sign(aws_secret_access_key, auth_string)
38
- Base64.encode64(OpenSSL::HMAC.digest(@@digest1, aws_secret_access_key, auth_string)).strip
39
- end
38
+ def self.sign(aws_secret_access_key, auth_string)
39
+ Base64.encode64(OpenSSL::HMAC.digest(@@digest1, aws_secret_access_key, auth_string)).strip
40
+ end
40
41
 
41
- # Escape a string accordingly Amazon rulles
42
- # http://docs.amazonwebservices.com/AmazonSimpleDB/2007-11-07/DeveloperGuide/index.html?REST_RESTAuth.html
43
- def self.amz_escape(param)
44
- return CGI.escape(param.to_s).gsub("%7E", "~").gsub("+", "%20") # from: http://umlaut.rubyforge.org/svn/trunk/lib/aws_product_sign.rb
45
- #param.to_s.gsub(/([^a-zA-Z0-9._~-]+)/n) do
46
- # '%' + $1.unpack('H2' * $1.size).join('%').upcase
47
- #end
48
- end
49
42
 
50
- # Set a timestamp and a signature version
51
- def self.fix_service_params(service_hash, signature)
52
- service_hash["Timestamp"] ||= Time.now.utc.strftime("%Y-%m-%dT%H:%M:%S.000Z") unless service_hash["Expires"]
53
- service_hash["SignatureVersion"] = signature
54
- service_hash
55
- end
43
+ # Set a timestamp and a signature version
44
+ def self.fix_service_params(service_hash, signature)
45
+ service_hash["Timestamp"] ||= Time.now.utc.strftime("%Y-%m-%dT%H:%M:%S.000Z") unless service_hash["Expires"]
46
+ service_hash["SignatureVersion"] = signature
47
+ service_hash
48
+ end
56
49
 
57
- # Signature Version 0
58
- # A deprecated guy (should work till septemper 2009)
59
- def self.sign_request_v0(aws_secret_access_key, service_hash)
60
- fix_service_params(service_hash, '0')
61
- string_to_sign = "#{service_hash['Action']}#{service_hash['Timestamp'] || service_hash['Expires']}"
62
- service_hash['Signature'] = AwsUtils::sign(aws_secret_access_key, string_to_sign)
63
- service_hash.to_a.collect{|key,val| "#{amz_escape(key)}=#{amz_escape(val.to_s)}" }.join("&")
64
- end
50
+ # Signature Version 0
51
+ # A deprecated guy (should work till septemper 2009)
52
+ def self.sign_request_v0(aws_secret_access_key, service_hash)
53
+ fix_service_params(service_hash, '0')
54
+ string_to_sign = "#{service_hash['Action']}#{service_hash['Timestamp'] || service_hash['Expires']}"
55
+ service_hash['Signature'] = AwsUtils::sign(aws_secret_access_key, string_to_sign)
56
+ service_hash.to_a.collect{|key, val| "#{amz_escape(key)}=#{amz_escape(val.to_s)}" }.join("&")
57
+ end
65
58
 
66
- # Signature Version 1
67
- # Another deprecated guy (should work till septemper 2009)
68
- def self.sign_request_v1(aws_secret_access_key, service_hash)
69
- fix_service_params(service_hash, '1')
70
- string_to_sign = service_hash.sort{|a,b| (a[0].to_s.downcase)<=>(b[0].to_s.downcase)}.to_s
71
- service_hash['Signature'] = AwsUtils::sign(aws_secret_access_key, string_to_sign)
72
- service_hash.to_a.collect{|key,val| "#{amz_escape(key)}=#{amz_escape(val.to_s)}" }.join("&")
73
- end
59
+ # Signature Version 1
60
+ # Another deprecated guy (should work till septemper 2009)
61
+ def self.sign_request_v1(aws_secret_access_key, service_hash)
62
+ fix_service_params(service_hash, '1')
63
+ string_to_sign = service_hash.sort{|a, b| (a[0].to_s.downcase)<=>(b[0].to_s.downcase)}.to_s
64
+ service_hash['Signature'] = AwsUtils::sign(aws_secret_access_key, string_to_sign)
65
+ service_hash.to_a.collect{|key, val| "#{amz_escape(key)}=#{amz_escape(val.to_s)}" }.join("&")
66
+ end
74
67
 
75
- # Signature Version 2
76
- # EC2, SQS and SDB requests must be signed by this guy.
77
- # See: http://docs.amazonwebservices.com/AmazonSimpleDB/2007-11-07/DeveloperGuide/index.html?REST_RESTAuth.html
78
- # http://developer.amazonwebservices.com/connect/entry.jspa?externalID=1928
79
- def self.sign_request_v2(aws_secret_access_key, service_hash, http_verb, host, uri)
80
- fix_service_params(service_hash, '2')
81
- # select a signing method (make an old openssl working with sha1)
82
- # make 'HmacSHA256' to be a default one
83
- service_hash['SignatureMethod'] = 'HmacSHA256' unless ['HmacSHA256', 'HmacSHA1'].include?(service_hash['SignatureMethod'])
84
- service_hash['SignatureMethod'] = 'HmacSHA1' unless @@digest256
85
- # select a digest
86
- digest = (service_hash['SignatureMethod'] == 'HmacSHA256' ? @@digest256 : @@digest1)
87
- # form string to sign
88
- canonical_string = service_hash.keys.sort.map do |key|
89
- "#{amz_escape(key)}=#{amz_escape(service_hash[key])}"
90
- end.join('&')
91
- string_to_sign = "#{http_verb.to_s.upcase}\n#{host.downcase}\n#{uri}\n#{canonical_string}"
92
- # sign the string
93
- signature = amz_escape(Base64.encode64(OpenSSL::HMAC.digest(digest, aws_secret_access_key, string_to_sign)).strip)
94
- "#{canonical_string}&Signature=#{signature}"
95
- end
68
+ # Signature Version 2
69
+ # EC2, SQS and SDB requests must be signed by this guy.
70
+ # See: http://docs.amazonwebservices.com/AmazonSimpleDB/2007-11-07/DeveloperGuide/index.html?REST_RESTAuth.html
71
+ # http://developer.amazonwebservices.com/connect/entry.jspa?externalID=1928
72
+ def self.sign_request_v2(aws_secret_access_key, service_hash, http_verb, host, uri)
73
+ fix_service_params(service_hash, '2')
74
+ # select a signing method (make an old openssl working with sha1)
75
+ # make 'HmacSHA256' to be a default one
76
+ service_hash['SignatureMethod'] = 'HmacSHA256' unless ['HmacSHA256', 'HmacSHA1'].include?(service_hash['SignatureMethod'])
77
+ service_hash['SignatureMethod'] = 'HmacSHA1' unless @@digest256
78
+ # select a digest
79
+ digest = (service_hash['SignatureMethod'] == 'HmacSHA256' ? @@digest256 : @@digest1)
80
+ # form string to sign
81
+ canonical_string = service_hash.keys.sort.map do |key|
82
+ "#{amz_escape(key)}=#{amz_escape(service_hash[key])}"
83
+ end.join('&')
84
+ string_to_sign = "#{http_verb.to_s.upcase}\n#{host.downcase}\n#{uri}\n#{canonical_string}"
85
+ # sign the string
86
+ signature = escape_sig(Base64.encode64(OpenSSL::HMAC.digest(digest, aws_secret_access_key, string_to_sign)).strip)
87
+ "#{canonical_string}&Signature=#{signature}"
88
+ end
96
89
 
97
- # From Amazon's SQS Dev Guide, a brief description of how to escape:
98
- # "URL encode the computed signature and other query parameters as specified in
99
- # RFC1738, section 2.2. In addition, because the + character is interpreted as a blank space
100
- # by Sun Java classes that perform URL decoding, make sure to encode the + character
101
- # although it is not required by RFC1738."
102
- # Avoid using CGI::escape to escape URIs.
103
- # CGI::escape will escape characters in the protocol, host, and port
104
- # sections of the URI. Only target chars in the query
105
- # string should be escaped.
106
- def self.URLencode(raw)
107
- e = URI.escape(raw)
108
- e.gsub(/\+/, "%2b")
109
- end
90
+ # Escape a string accordingly Amazon rulles
91
+ # http://docs.amazonwebservices.com/AmazonSimpleDB/2007-11-07/DeveloperGuide/index.html?REST_RESTAuth.html
92
+ def self.amz_escape(param)
93
+ #return CGI.escape(param.to_s).gsub("%7E", "~").gsub("+", "%20") # from: http://umlaut.rubyforge.org/svn/trunk/lib/aws_product_sign.rb
94
+
95
+ #param.to_s.gsub(/([^a-zA-Z0-9._~-]+)/n) do
96
+ # '%' + $1.unpack('H2' * $1.size).join('%').upcase
97
+ #end
98
+ e = CGI.escape(param.to_s)
99
+ e = e.gsub("+", "%20")
100
+ e = e.gsub("*", "%2A")
101
+ e = e.gsub("%7E", "~")
102
+ end
110
103
 
111
- def self.allow_only(allowed_keys, params)
112
- bogus_args = []
113
- params.keys.each {|p| bogus_args.push(p) unless allowed_keys.include?(p) }
114
- raise AwsError.new("The following arguments were given but are not legal for the function call #{caller_method}: #{bogus_args.inspect}") if bogus_args.length > 0
115
- end
116
104
 
117
- def self.mandatory_arguments(required_args, params)
118
- rargs = required_args.dup
119
- params.keys.each {|p| rargs.delete(p)}
120
- raise AwsError.new("The following mandatory arguments were not provided to #{caller_method}: #{rargs.inspect}") if rargs.length > 0
121
- end
105
+ def self.escape_sig(raw)
106
+ e = CGI.escape(raw)
107
+ end
122
108
 
123
- def self.caller_method
124
- caller[1]=~/`(.*?)'/
125
- $1
126
- end
109
+ # From Amazon's SQS Dev Guide, a brief description of how to escape:
110
+ # "URL encode the computed signature and other query parameters as specified in
111
+ # RFC1738, section 2.2. In addition, because the + character is interpreted as a blank space
112
+ # by Sun Java classes that perform URL decoding, make sure to encode the + character
113
+ # although it is not required by RFC1738."
114
+ # Avoid using CGI::escape to escape URIs.
115
+ # CGI::escape will escape characters in the protocol, host, and port
116
+ # sections of the URI. Only target chars in the query
117
+ # string should be escaped.
118
+ def self.URLencode(raw)
119
+ e = URI.escape(raw)
120
+ e.gsub(/\+/, "%2b")
121
+ end
127
122
 
128
- end
129
123
 
130
- class AwsBenchmarkingBlock #:nodoc:
131
- attr_accessor :xml, :service
132
- def initialize
133
- # Benchmark::Tms instance for service (Ec2, S3, or SQS) access benchmarking.
134
- @service = Benchmark::Tms.new()
135
- # Benchmark::Tms instance for XML parsing benchmarking.
136
- @xml = Benchmark::Tms.new()
124
+ def self.allow_only(allowed_keys, params)
125
+ bogus_args = []
126
+ params.keys.each {|p| bogus_args.push(p) unless allowed_keys.include?(p) }
127
+ raise AwsError.new("The following arguments were given but are not legal for the function call #{caller_method}: #{bogus_args.inspect}") if bogus_args.length > 0
128
+ end
129
+
130
+ def self.mandatory_arguments(required_args, params)
131
+ rargs = required_args.dup
132
+ params.keys.each {|p| rargs.delete(p)}
133
+ raise AwsError.new("The following mandatory arguments were not provided to #{caller_method}: #{rargs.inspect}") if rargs.length > 0
134
+ end
135
+
136
+ def self.caller_method
137
+ caller[1]=~/`(.*?)'/
138
+ $1
139
+ end
140
+
137
141
  end
138
- end
139
-
140
- class AwsNoChange < RuntimeError
141
- end
142
-
143
- class RightAwsBase
144
-
145
- # Amazon HTTP Error handling
146
-
147
- # Text, if found in an error message returned by AWS, indicates that this may be a transient
148
- # error. Transient errors are automatically retried with exponential back-off.
149
- AMAZON_PROBLEMS = [ 'internal service error',
150
- 'is currently unavailable',
151
- 'no response from',
152
- 'Please try again',
153
- 'InternalError',
154
- 'ServiceUnavailable', #from SQS docs
155
- 'Unavailable',
156
- 'This application is not currently available',
157
- 'InsufficientInstanceCapacity'
158
- ]
159
- @@amazon_problems = AMAZON_PROBLEMS
160
- # Returns a list of Amazon service responses which are known to be transient problems.
161
- # We have to re-request if we get any of them, because the problem will probably disappear.
162
- # By default this method returns the same value as the AMAZON_PROBLEMS const.
163
- def self.amazon_problems
164
- @@amazon_problems
142
+
143
+ class AwsBenchmarkingBlock #:nodoc:
144
+ attr_accessor :xml, :service
145
+
146
+ def initialize
147
+ # Benchmark::Tms instance for service (Ec2, S3, or SQS) access benchmarking.
148
+ @service = Benchmark::Tms.new()
149
+ # Benchmark::Tms instance for XML parsing benchmarking.
150
+ @xml = Benchmark::Tms.new()
151
+ end
165
152
  end
166
153
 
167
- # Sets the list of Amazon side problems. Use in conjunction with the
168
- # getter to append problems.
169
- def self.amazon_problems=(problems_list)
170
- @@amazon_problems = problems_list
154
+ class AwsNoChange < RuntimeError
171
155
  end
172
156
 
173
- end
157
+ class RightAwsBase
174
158
 
175
- module RightAwsBaseInterface
176
- DEFAULT_SIGNATURE_VERSION = '2'
159
+ # Amazon HTTP Error handling
160
+
161
+ # Text, if found in an error message returned by AWS, indicates that this may be a transient
162
+ # error. Transient errors are automatically retried with exponential back-off.
163
+ AMAZON_PROBLEMS = [ 'internal service error',
164
+ 'is currently unavailable',
165
+ 'no response from',
166
+ 'Please try again',
167
+ 'InternalError',
168
+ 'ServiceUnavailable', #from SQS docs
169
+ 'Unavailable',
170
+ 'This application is not currently available',
171
+ 'InsufficientInstanceCapacity'
172
+ ]
173
+ @@amazon_problems = AMAZON_PROBLEMS
174
+ # Returns a list of Amazon service responses which are known to be transient problems.
175
+ # We have to re-request if we get any of them, because the problem will probably disappear.
176
+ # By default this method returns the same value as the AMAZON_PROBLEMS const.
177
+ def self.amazon_problems
178
+ @@amazon_problems
179
+ end
180
+
181
+ # Sets the list of Amazon side problems. Use in conjunction with the
182
+ # getter to append problems.
183
+ def self.amazon_problems=(problems_list)
184
+ @@amazon_problems = problems_list
185
+ end
177
186
 
178
- @@caching = false
179
- def self.caching
180
- @@caching
181
- end
182
- def self.caching=(caching)
183
- @@caching = caching
184
187
  end
185
188
 
186
- # Current aws_access_key_id
187
- attr_reader :aws_access_key_id
188
- # Last HTTP request object
189
- attr_reader :last_request
190
- # Last HTTP response object
191
- attr_reader :last_response
192
- # Last AWS errors list (used by AWSErrorHandler)
193
- attr_accessor :last_errors
194
- # Last AWS request id (used by AWSErrorHandler)
195
- attr_accessor :last_request_id
196
- # Logger object
197
- attr_accessor :logger
198
- # Initial params hash
199
- attr_accessor :params
200
- # RightHttpConnection instance
201
- attr_reader :connection
202
- # Cache
203
- attr_reader :cache
204
- # Signature version (all services except s3)
205
- attr_reader :signature_version
206
-
207
- def init(service_info, aws_access_key_id, aws_secret_access_key, params={}) #:nodoc:
208
- @params = params
209
- raise AwsError.new("AWS access keys are required to operate on #{service_info[:name]}") \
189
+ module RightAwsBaseInterface
190
+ DEFAULT_SIGNATURE_VERSION = '2'
191
+
192
+ @@caching = false
193
+ def self.caching
194
+ @@caching
195
+ end
196
+ def self.caching=(caching)
197
+ @@caching = caching
198
+ end
199
+
200
+ # Current aws_access_key_id
201
+ attr_reader :aws_access_key_id
202
+ # Last HTTP request object
203
+ attr_reader :last_request
204
+ # Last HTTP response object
205
+ attr_reader :last_response
206
+ # Last AWS errors list (used by AWSErrorHandler)
207
+ attr_accessor :last_errors
208
+ # Last AWS request id (used by AWSErrorHandler)
209
+ attr_accessor :last_request_id
210
+ # Logger object
211
+ attr_accessor :logger
212
+ # Initial params hash
213
+ attr_accessor :params
214
+ # RightHttpConnection instance
215
+ attr_reader :connection
216
+ # Cache
217
+ attr_reader :cache
218
+ # Signature version (all services except s3)
219
+ attr_reader :signature_version
220
+
221
+ def init(service_info, aws_access_key_id, aws_secret_access_key, params={}) #:nodoc:
222
+ @params = params
223
+ raise AwsError.new("AWS access keys are required to operate on #{service_info[:name]}") \
210
224
  if aws_access_key_id.blank? || aws_secret_access_key.blank?
211
- @aws_access_key_id = aws_access_key_id
212
- @aws_secret_access_key = aws_secret_access_key
213
- # if the endpoint was explicitly defined - then use it
214
- if @params[:endpoint_url]
215
- @params[:server] = URI.parse(@params[:endpoint_url]).host
216
- @params[:port] = URI.parse(@params[:endpoint_url]).port
217
- @params[:service] = URI.parse(@params[:endpoint_url]).path
218
- @params[:protocol] = URI.parse(@params[:endpoint_url]).scheme
219
- @params[:region] = nil
220
- else
221
- @params[:server] ||= service_info[:default_host]
222
- @params[:server] = "#{@params[:region]}.#{@params[:server]}" if @params[:region]
223
- @params[:port] ||= service_info[:default_port]
224
- @params[:service] ||= service_info[:default_service]
225
- @params[:protocol] ||= service_info[:default_protocol]
226
- end
227
- if !@params[:multi_thread].nil? && @params[:connection_mode].nil? # user defined this
228
- @params[:connection_mode] = @params[:multi_thread] ? :per_thread : :single
229
- end
225
+ @aws_access_key_id = aws_access_key_id
226
+ @aws_secret_access_key = aws_secret_access_key
227
+ # if the endpoint was explicitly defined - then use it
228
+ if @params[:endpoint_url]
229
+ @params[:server] = URI.parse(@params[:endpoint_url]).host
230
+ @params[:port] = URI.parse(@params[:endpoint_url]).port
231
+ @params[:service] = URI.parse(@params[:endpoint_url]).path
232
+ @params[:protocol] = URI.parse(@params[:endpoint_url]).scheme
233
+ @params[:region] = nil
234
+ else
235
+ @params[:server] ||= service_info[:default_host]
236
+ @params[:server] = "#{@params[:region]}.#{@params[:server]}" if @params[:region]
237
+ @params[:port] ||= service_info[:default_port]
238
+ @params[:service] ||= service_info[:default_service]
239
+ @params[:protocol] ||= service_info[:default_protocol]
240
+ end
241
+ if !@params[:multi_thread].nil? && @params[:connection_mode].nil? # user defined this
242
+ @params[:connection_mode] = @params[:multi_thread] ? :per_thread : :single
243
+ end
230
244
  # @params[:multi_thread] ||= defined?(AWS_DAEMON)
231
- @params[:connection_mode] ||= :default
232
- @params[:connection_mode] = :per_request if @params[:connection_mode] == :default
233
- @logger = @params[:logger]
234
- @logger = RAILS_DEFAULT_LOGGER if !@logger && defined?(RAILS_DEFAULT_LOGGER)
235
- @logger = Logger.new(STDOUT) if !@logger
236
- @logger.info "New #{self.class.name} using #{@params[:connection_mode].to_s}-connection mode"
237
- @error_handler = nil
238
- @cache = {}
239
- @signature_version = (params[:signature_version] || DEFAULT_SIGNATURE_VERSION).to_s
240
- end
245
+ @params[:connection_mode] ||= :default
246
+ @params[:connection_mode] = :per_request if @params[:connection_mode] == :default
247
+ @logger = @params[:logger]
248
+ @logger = RAILS_DEFAULT_LOGGER if !@logger && defined?(RAILS_DEFAULT_LOGGER)
249
+ @logger = Logger.new(STDOUT) if !@logger
250
+ @logger.info "New #{self.class.name} using #{@params[:connection_mode].to_s}-connection mode"
251
+ @error_handler = nil
252
+ @cache = {}
253
+ @signature_version = (params[:signature_version] || DEFAULT_SIGNATURE_VERSION).to_s
254
+ end
241
255
 
242
- def signed_service_params(aws_secret_access_key, service_hash, http_verb=nil, host=nil, service=nil )
243
- case signature_version.to_s
244
- when '0' then AwsUtils::sign_request_v0(aws_secret_access_key, service_hash)
245
- when '1' then AwsUtils::sign_request_v1(aws_secret_access_key, service_hash)
246
- when '2' then AwsUtils::sign_request_v2(aws_secret_access_key, service_hash, http_verb, host, service)
247
- else raise AwsError.new("Unknown signature version (#{signature_version.to_s}) requested")
248
- end
249
- end
256
+ def signed_service_params(aws_secret_access_key, service_hash, http_verb=nil, host=nil, service=nil )
257
+ case signature_version.to_s
258
+ when '0' then
259
+ AwsUtils::sign_request_v0(aws_secret_access_key, service_hash)
260
+ when '1' then
261
+ AwsUtils::sign_request_v1(aws_secret_access_key, service_hash)
262
+ when '2' then
263
+ AwsUtils::sign_request_v2(aws_secret_access_key, service_hash, http_verb, host, service)
264
+ else
265
+ raise AwsError.new("Unknown signature version (#{signature_version.to_s}) requested")
266
+ end
267
+ end
250
268
 
251
- # Returns +true+ if the describe_xxx responses are being cached
252
- def caching?
253
- @params.key?(:cache) ? @params[:cache] : @@caching
254
- end
269
+ # Returns +true+ if the describe_xxx responses are being cached
270
+ def caching?
271
+ @params.key?(:cache) ? @params[:cache] : @@caching
272
+ end
255
273
 
256
- # Check if the aws function response hits the cache or not.
257
- # If the cache hits:
258
- # - raises an +AwsNoChange+ exception if +do_raise+ == +:raise+.
259
- # - returnes parsed response from the cache if it exists or +true+ otherwise.
260
- # If the cache miss or the caching is off then returns +false+.
261
- def cache_hits?(function, response, do_raise=:raise)
262
- result = false
263
- if caching?
264
- function = function.to_sym
265
- # get rid of requestId (this bad boy was added for API 2008-08-08+ and it is uniq for every response)
266
- response = response.sub(%r{<requestId>.+?</requestId>}, '')
267
- response_md5 =Digest::MD5.hexdigest(response).to_s
268
- # check for changes
269
- unless @cache[function] && @cache[function][:response_md5] == response_md5
270
- # well, the response is new, reset cache data
271
- update_cache(function, {:response_md5 => response_md5,
272
- :timestamp => Time.now,
273
- :hits => 0,
274
- :parsed => nil})
275
- else
276
- # aha, cache hits, update the data and throw an exception if needed
277
- @cache[function][:hits] += 1
278
- if do_raise == :raise
279
- raise(AwsNoChange, "Cache hit: #{function} response has not changed since "+
280
- "#{@cache[function][:timestamp].strftime('%Y-%m-%d %H:%M:%S')}, "+
281
- "hits: #{@cache[function][:hits]}.")
282
- else
283
- result = @cache[function][:parsed] || true
284
- end
285
- end
286
- end
287
- result
288
- end
274
+ # Check if the aws function response hits the cache or not.
275
+ # If the cache hits:
276
+ # - raises an +AwsNoChange+ exception if +do_raise+ == +:raise+.
277
+ # - returnes parsed response from the cache if it exists or +true+ otherwise.
278
+ # If the cache miss or the caching is off then returns +false+.
279
+ def cache_hits?(function, response, do_raise=:raise)
280
+ result = false
281
+ if caching?
282
+ function = function.to_sym
283
+ # get rid of requestId (this bad boy was added for API 2008-08-08+ and it is uniq for every response)
284
+ response = response.sub(%r{<requestId>.+?</requestId>}, '')
285
+ response_md5 =Digest::MD5.hexdigest(response).to_s
286
+ # check for changes
287
+ unless @cache[function] && @cache[function][:response_md5] == response_md5
288
+ # well, the response is new, reset cache data
289
+ update_cache(function, {:response_md5 => response_md5,
290
+ :timestamp => Time.now,
291
+ :hits => 0,
292
+ :parsed => nil})
293
+ else
294
+ # aha, cache hits, update the data and throw an exception if needed
295
+ @cache[function][:hits] += 1
296
+ if do_raise == :raise
297
+ raise(AwsNoChange, "Cache hit: #{function} response has not changed since "+
298
+ "#{@cache[function][:timestamp].strftime('%Y-%m-%d %H:%M:%S')}, "+
299
+ "hits: #{@cache[function][:hits]}.")
300
+ else
301
+ result = @cache[function][:parsed] || true
302
+ end
303
+ end
304
+ end
305
+ result
306
+ end
289
307
 
290
- def update_cache(function, hash)
291
- (@cache[function.to_sym] ||= {}).merge!(hash) if caching?
292
- end
308
+ def update_cache(function, hash)
309
+ (@cache[function.to_sym] ||= {}).merge!(hash) if caching?
310
+ end
293
311
 
294
- def on_exception(options={:raise=>true, :log=>true}) # :nodoc:
295
- raise if $!.is_a?(AwsNoChange)
296
- AwsError::on_aws_exception(self, options)
297
- end
312
+ def on_exception(options={:raise=>true, :log=>true}) # :nodoc:
313
+ raise if $!.is_a?(AwsNoChange)
314
+ AwsError::on_aws_exception(self, options)
315
+ end
298
316
 
299
- # Return +true+ if this instance works in multi_thread mode and +false+ otherwise.
300
- def multi_thread
301
- @params[:multi_thread]
302
- end
317
+ # Return +true+ if this instance works in multi_thread mode and +false+ otherwise.
318
+ def multi_thread
319
+ @params[:multi_thread]
320
+ end
303
321
 
304
- def request_info_impl(connection, benchblock, request, parser, &block) #:nodoc:
305
- @connection = connection
306
- @last_request = request[:request]
307
- @last_response = nil
308
- response=nil
309
- blockexception = nil
310
-
311
- if(block != nil)
312
- # TRB 9/17/07 Careful - because we are passing in blocks, we get a situation where
313
- # an exception may get thrown in the block body (which is high-level
314
- # code either here or in the application) but gets caught in the
315
- # low-level code of HttpConnection. The solution is not to let any
316
- # exception escape the block that we pass to HttpConnection::request.
317
- # Exceptions can originate from code directly in the block, or from user
318
- # code called in the other block which is passed to response.read_body.
319
- benchblock.service.add! do
320
- responsehdr = @connection.request(request) do |response|
321
- #########
322
- begin
323
- @last_response = response
324
- if response.is_a?(Net::HTTPSuccess)
325
- @error_handler = nil
326
- response.read_body(&block)
327
- else
328
- @error_handler = AWSErrorHandler.new(self, parser, :errors_list => self.class.amazon_problems) unless @error_handler
329
- check_result = @error_handler.check(request)
330
- if check_result
331
- @error_handler = nil
332
- return check_result
322
+ def request_info_impl(connection, benchblock, request, parser, &block) #:nodoc:
323
+ @connection = connection
324
+ @last_request = request[:request]
325
+ @last_response = nil
326
+ response=nil
327
+ blockexception = nil
328
+
329
+ if (block != nil)
330
+ # TRB 9/17/07 Careful - because we are passing in blocks, we get a situation where
331
+ # an exception may get thrown in the block body (which is high-level
332
+ # code either here or in the application) but gets caught in the
333
+ # low-level code of HttpConnection. The solution is not to let any
334
+ # exception escape the block that we pass to HttpConnection::request.
335
+ # Exceptions can originate from code directly in the block, or from user
336
+ # code called in the other block which is passed to response.read_body.
337
+ benchblock.service.add! do
338
+ responsehdr = @connection.request(request) do |response|
339
+ #########
340
+ begin
341
+ @last_response = response
342
+ if response.is_a?(Net::HTTPSuccess)
343
+ @error_handler = nil
344
+ response.read_body(&block)
345
+ else
346
+ @error_handler = AWSErrorHandler.new(self, parser, :errors_list => self.class.amazon_problems) unless @error_handler
347
+ check_result = @error_handler.check(request)
348
+ if check_result
349
+ @error_handler = nil
350
+ return check_result
351
+ end
352
+ request_text_data = "#{request[:server]}:#{request[:port]}#{request[:request].path}"
353
+ raise AwsError.new(@last_errors, @last_response.code, @last_request_id, request_text_data)
354
+ end
355
+ rescue Exception => e
356
+ blockexception = e
357
+ end
358
+ end
359
+ #########
360
+
361
+ #OK, now we are out of the block passed to the lower level
362
+ if (blockexception)
363
+ raise blockexception
364
+ end
365
+ benchblock.xml.add! do
366
+ parser.parse(responsehdr)
367
+ end
368
+ return parser.result
369
+ end
370
+ else
371
+ benchblock.service.add!{ response = @connection.request(request) }
372
+ # check response for errors...
373
+ @last_response = response
374
+ if response.is_a?(Net::HTTPSuccess)
375
+ @error_handler = nil
376
+ benchblock.xml.add! { parser.parse(response) }
377
+ return parser.result
378
+ else
379
+ @error_handler = AWSErrorHandler.new(self, parser, :errors_list => self.class.amazon_problems) unless @error_handler
380
+ check_result = @error_handler.check(request)
381
+ if check_result
382
+ @error_handler = nil
383
+ return check_result
384
+ end
385
+ request_text_data = "#{request[:server]}:#{request[:port]}#{request[:request].path}"
386
+ raise AwsError.new(@last_errors, @last_response.code, @last_request_id, request_text_data)
333
387
  end
334
- request_text_data = "#{request[:server]}:#{request[:port]}#{request[:request].path}"
335
- raise AwsError.new(@last_errors, @last_response.code, @last_request_id, request_text_data)
336
- end
337
- rescue Exception => e
338
- blockexception = e
339
388
  end
340
- end
341
- #########
342
-
343
- #OK, now we are out of the block passed to the lower level
344
- if(blockexception)
345
- raise blockexception
346
- end
347
- benchblock.xml.add! do
348
- parser.parse(responsehdr)
349
- end
350
- return parser.result
351
- end
352
- else
353
- benchblock.service.add!{ response = @connection.request(request) }
354
- # check response for errors...
355
- @last_response = response
356
- if response.is_a?(Net::HTTPSuccess)
357
- @error_handler = nil
358
- benchblock.xml.add! { parser.parse(response) }
359
- return parser.result
360
- else
361
- @error_handler = AWSErrorHandler.new(self, parser, :errors_list => self.class.amazon_problems) unless @error_handler
362
- check_result = @error_handler.check(request)
363
- if check_result
389
+ rescue
364
390
  @error_handler = nil
365
- return check_result
366
- end
367
- request_text_data = "#{request[:server]}:#{request[:port]}#{request[:request].path}"
368
- raise AwsError.new(@last_errors, @last_response.code, @last_request_id, request_text_data)
369
- end
370
- end
371
- rescue
372
- @error_handler = nil
373
- raise
374
- end
391
+ raise
392
+ end
375
393
 
376
- def request_cache_or_info(method, link, parser_class, benchblock, use_cache=true) #:nodoc:
377
- # We do not want to break the logic of parsing hence will use a dummy parser to process all the standard
378
- # steps (errors checking etc). The dummy parser does nothig - just returns back the params it received.
379
- # If the caching is enabled and hit then throw AwsNoChange.
380
- # P.S. caching works for the whole images list only! (when the list param is blank)
381
- # check cache
382
- response, params = request_info(link, RightDummyParser.new)
383
- cache_hits?(method.to_sym, response.body) if use_cache
384
- parser = parser_class.new(:logger => @logger)
385
- benchblock.xml.add!{ parser.parse(response, params) }
386
- result = block_given? ? yield(parser) : parser.result
387
- # update parsed data
388
- update_cache(method.to_sym, :parsed => result) if use_cache
389
- result
390
- end
394
+ def request_cache_or_info(method, link, parser_class, benchblock, use_cache=true) #:nodoc:
395
+ # We do not want to break the logic of parsing hence will use a dummy parser to process all the standard
396
+ # steps (errors checking etc). The dummy parser does nothig - just returns back the params it received.
397
+ # If the caching is enabled and hit then throw AwsNoChange.
398
+ # P.S. caching works for the whole images list only! (when the list param is blank)
399
+ # check cache
400
+ response, params = request_info(link, RightDummyParser.new)
401
+ cache_hits?(method.to_sym, response.body) if use_cache
402
+ parser = parser_class.new(:logger => @logger)
403
+ benchblock.xml.add!{ parser.parse(response, params) }
404
+ result = block_given? ? yield(parser) : parser.result
405
+ # update parsed data
406
+ update_cache(method.to_sym, :parsed => result) if use_cache
407
+ result
408
+ end
409
+
410
+ # Returns Amazons request ID for the latest request
411
+ def last_request_id
412
+ @last_response && @last_response.body.to_s[%r{<requestId>(.+?)</requestId>}] && $1
413
+ end
391
414
 
392
- # Returns Amazons request ID for the latest request
393
- def last_request_id
394
- @last_response && @last_response.body.to_s[%r{<requestId>(.+?)</requestId>}] && $1
395
415
  end
396
416
 
397
- end
398
417
 
418
+ # Exception class to signal any Amazon errors. All errors occuring during calls to Amazon's
419
+ # web services raise this type of error.
420
+ # Attribute inherited by RuntimeError:
421
+ # message - the text of the error, generally as returned by AWS in its XML response.
422
+ class AwsError < RuntimeError
399
423
 
400
- # Exception class to signal any Amazon errors. All errors occuring during calls to Amazon's
401
- # web services raise this type of error.
402
- # Attribute inherited by RuntimeError:
403
- # message - the text of the error, generally as returned by AWS in its XML response.
404
- class AwsError < RuntimeError
424
+ # either an array of errors where each item is itself an array of [code, message]),
425
+ # or an error string if the error was raised manually, as in <tt>AwsError.new('err_text')</tt>
426
+ attr_reader :errors
405
427
 
406
- # either an array of errors where each item is itself an array of [code, message]),
407
- # or an error string if the error was raised manually, as in <tt>AwsError.new('err_text')</tt>
408
- attr_reader :errors
428
+ # Request id (if exists)
429
+ attr_reader :request_id
409
430
 
410
- # Request id (if exists)
411
- attr_reader :request_id
431
+ # Response HTTP error code
432
+ attr_reader :http_code
412
433
 
413
- # Response HTTP error code
414
- attr_reader :http_code
434
+ # Raw request text data to AWS
435
+ attr_reader :request_data
415
436
 
416
- # Raw request text data to AWS
417
- attr_reader :request_data
437
+ def initialize(errors=nil, http_code=nil, request_id=nil, request_data=nil)
438
+ @errors = errors
439
+ @request_id = request_id
440
+ @http_code = http_code
441
+ @request_data = request_data
442
+ msg = @errors.is_a?(Array) ? @errors.map{|code, msg| "#{code}: #{msg}"}.join("; ") : @errors.to_s
443
+ msg += "\nREQUEST(#{@request_data})" unless @request_data.nil?
444
+ super(msg)
445
+ end
418
446
 
419
- def initialize(errors=nil, http_code=nil, request_id=nil, request_data=nil)
420
- @errors = errors
421
- @request_id = request_id
422
- @http_code = http_code
423
- @request_data = request_data
424
- msg = @errors.is_a?(Array) ? @errors.map{|code, msg| "#{code}: #{msg}"}.join("; ") : @errors.to_s
425
- msg += "\nREQUEST(#{@request_data})" unless @request_data.nil?
426
- super(msg)
427
- end
447
+ # Does any of the error messages include the regexp +pattern+?
448
+ # Used to determine whether to retry request.
449
+ def include?(pattern)
450
+ if @errors.is_a?(Array)
451
+ @errors.each{ |code, msg| return true if code =~ pattern }
452
+ else
453
+ return true if @errors_str =~ pattern
454
+ end
455
+ false
456
+ end
428
457
 
429
- # Does any of the error messages include the regexp +pattern+?
430
- # Used to determine whether to retry request.
431
- def include?(pattern)
432
- if @errors.is_a?(Array)
433
- @errors.each{ |code, msg| return true if code =~ pattern }
434
- else
435
- return true if @errors_str =~ pattern
436
- end
437
- false
438
- end
458
+ # Generic handler for AwsErrors. +aws+ is the RightAws::S3, RightAws::EC2, or RightAws::SQS
459
+ # object that caused the exception (it must provide last_request and last_response). Supported
460
+ # boolean options are:
461
+ # * <tt>:log</tt> print a message into the log using aws.logger to access the Logger
462
+ # * <tt>:puts</tt> do a "puts" of the error
463
+ # * <tt>:raise</tt> re-raise the error after logging
464
+ def self.on_aws_exception(aws, options={:raise=>true, :log=>true})
465
+ # Only log & notify if not user error
466
+ if !options[:raise] || system_error?($!)
467
+ error_text = "#{$!.inspect}\n#{$@}.join('\n')}"
468
+ puts error_text if options[:puts]
469
+ # Log the error
470
+ if options[:log]
471
+ request = aws.last_request ? aws.last_request.path : '-none-'
472
+ response = aws.last_response ? "#{aws.last_response.code} -- #{aws.last_response.message} -- #{aws.last_response.body}" : '-none-'
473
+ aws.logger.error error_text
474
+ aws.logger.error "Request was: #{request}"
475
+ aws.logger.error "Response was: #{response}"
476
+ end
477
+ end
478
+ raise if options[:raise] # re-raise an exception
479
+ return nil
480
+ end
439
481
 
440
- # Generic handler for AwsErrors. +aws+ is the RightAws::S3, RightAws::EC2, or RightAws::SQS
441
- # object that caused the exception (it must provide last_request and last_response). Supported
442
- # boolean options are:
443
- # * <tt>:log</tt> print a message into the log using aws.logger to access the Logger
444
- # * <tt>:puts</tt> do a "puts" of the error
445
- # * <tt>:raise</tt> re-raise the error after logging
446
- def self.on_aws_exception(aws, options={:raise=>true, :log=>true})
447
- # Only log & notify if not user error
448
- if !options[:raise] || system_error?($!)
449
- error_text = "#{$!.inspect}\n#{$@}.join('\n')}"
450
- puts error_text if options[:puts]
451
- # Log the error
452
- if options[:log]
453
- request = aws.last_request ? aws.last_request.path : '-none-'
454
- response = aws.last_response ? "#{aws.last_response.code} -- #{aws.last_response.message} -- #{aws.last_response.body}" : '-none-'
455
- aws.logger.error error_text
456
- aws.logger.error "Request was: #{request}"
457
- aws.logger.error "Response was: #{response}"
458
- end
459
- end
460
- raise if options[:raise] # re-raise an exception
461
- return nil
462
- end
482
+ # True if e is an AWS system error, i.e. something that is for sure not the caller's fault.
483
+ # Used to force logging.
484
+ def self.system_error?(e)
485
+ !e.is_a?(self) || e.message =~ /InternalError|InsufficientInstanceCapacity|Unavailable/
486
+ end
463
487
 
464
- # True if e is an AWS system error, i.e. something that is for sure not the caller's fault.
465
- # Used to force logging.
466
- def self.system_error?(e)
467
- !e.is_a?(self) || e.message =~ /InternalError|InsufficientInstanceCapacity|Unavailable/
468
488
  end
469
489
 
470
- end
471
490
 
491
+ class AWSErrorHandler
492
+ # 0-100 (%)
493
+ DEFAULT_CLOSE_ON_4XX_PROBABILITY = 10
472
494
 
473
- class AWSErrorHandler
474
- # 0-100 (%)
475
- DEFAULT_CLOSE_ON_4XX_PROBABILITY = 10
495
+ @@reiteration_start_delay = 0.2
496
+ def self.reiteration_start_delay
497
+ @@reiteration_start_delay
498
+ end
499
+ def self.reiteration_start_delay=(reiteration_start_delay)
500
+ @@reiteration_start_delay = reiteration_start_delay
501
+ end
476
502
 
477
- @@reiteration_start_delay = 0.2
478
- def self.reiteration_start_delay
479
- @@reiteration_start_delay
480
- end
481
- def self.reiteration_start_delay=(reiteration_start_delay)
482
- @@reiteration_start_delay = reiteration_start_delay
483
- end
503
+ @@reiteration_time = 5
504
+ def self.reiteration_time
505
+ @@reiteration_time
506
+ end
507
+ def self.reiteration_time=(reiteration_time)
508
+ @@reiteration_time = reiteration_time
509
+ end
484
510
 
485
- @@reiteration_time = 5
486
- def self.reiteration_time
487
- @@reiteration_time
488
- end
489
- def self.reiteration_time=(reiteration_time)
490
- @@reiteration_time = reiteration_time
491
- end
511
+ @@close_on_error = true
512
+ def self.close_on_error
513
+ @@close_on_error
514
+ end
515
+ def self.close_on_error=(close_on_error)
516
+ @@close_on_error = close_on_error
517
+ end
492
518
 
493
- @@close_on_error = true
494
- def self.close_on_error
495
- @@close_on_error
496
- end
497
- def self.close_on_error=(close_on_error)
498
- @@close_on_error = close_on_error
499
- end
519
+ @@close_on_4xx_probability = DEFAULT_CLOSE_ON_4XX_PROBABILITY
520
+ def self.close_on_4xx_probability
521
+ @@close_on_4xx_probability
522
+ end
523
+ def self.close_on_4xx_probability=(close_on_4xx_probability)
524
+ @@close_on_4xx_probability = close_on_4xx_probability
525
+ end
500
526
 
501
- @@close_on_4xx_probability = DEFAULT_CLOSE_ON_4XX_PROBABILITY
502
- def self.close_on_4xx_probability
503
- @@close_on_4xx_probability
504
- end
505
- def self.close_on_4xx_probability=(close_on_4xx_probability)
506
- @@close_on_4xx_probability = close_on_4xx_probability
507
- end
527
+ # params:
528
+ # :reiteration_time
529
+ # :errors_list
530
+ # :close_on_error = true | false
531
+ # :close_on_4xx_probability = 1-100
532
+ def initialize(aws, parser, params={}) #:nodoc:
533
+ @aws = aws # Link to RightEc2 | RightSqs | RightS3 instance
534
+ @parser = parser # parser to parse Amazon response
535
+ @started_at = Time.now
536
+ @stop_at = @started_at + (params[:reiteration_time] || @@reiteration_time)
537
+ @errors_list = params[:errors_list] || []
538
+ @reiteration_delay = @@reiteration_start_delay
539
+ @retries = 0
540
+ # close current HTTP(S) connection on 5xx, errors from list and 4xx errors
541
+ @close_on_error = params[:close_on_error].nil? ? @@close_on_error : params[:close_on_error]
542
+ @close_on_4xx_probability = params[:close_on_4xx_probability] || @@close_on_4xx_probability
543
+ end
508
544
 
509
- # params:
510
- # :reiteration_time
511
- # :errors_list
512
- # :close_on_error = true | false
513
- # :close_on_4xx_probability = 1-100
514
- def initialize(aws, parser, params={}) #:nodoc:
515
- @aws = aws # Link to RightEc2 | RightSqs | RightS3 instance
516
- @parser = parser # parser to parse Amazon response
517
- @started_at = Time.now
518
- @stop_at = @started_at + (params[:reiteration_time] || @@reiteration_time)
519
- @errors_list = params[:errors_list] || []
520
- @reiteration_delay = @@reiteration_start_delay
521
- @retries = 0
522
- # close current HTTP(S) connection on 5xx, errors from list and 4xx errors
523
- @close_on_error = params[:close_on_error].nil? ? @@close_on_error : params[:close_on_error]
524
- @close_on_4xx_probability = params[:close_on_4xx_probability] || @@close_on_4xx_probability
525
- end
545
+ # Returns false if
546
+ def check(request) #:nodoc:
547
+ result = false
548
+ error_found = false
549
+ redirect_detected= false
550
+ error_match = nil
551
+ last_errors_text = ''
552
+ response = @aws.last_response
553
+ # log error
554
+ request_text_data = "#{request[:server]}:#{request[:port]}#{request[:request].path}"
555
+ # is this a redirect?
556
+ # yes!
557
+ if response.is_a?(Net::HTTPRedirection)
558
+ redirect_detected = true
559
+ else
560
+ # no, it's an error ...
561
+ @aws.logger.warn("##### #{@aws.class.name} returned an error: #{response.code} #{response.message}\n#{response.body} #####")
562
+ @aws.logger.warn("##### #{@aws.class.name} request: #{request_text_data} ####")
563
+ end
564
+ # Check response body: if it is an Amazon XML document or not:
565
+ if redirect_detected || (response.body && response.body[/<\?xml/]) # ... it is a xml document
566
+ @aws.class.bench_xml.add! do
567
+ error_parser = RightErrorResponseParser.new
568
+ error_parser.parse(response)
569
+ @aws.last_errors = error_parser.errors
570
+ @aws.last_request_id = error_parser.requestID
571
+ last_errors_text = @aws.last_errors.flatten.join("\n")
572
+ # on redirect :
573
+ if redirect_detected
574
+ location = response['location']
575
+ # ... log information and ...
576
+ @aws.logger.info("##### #{@aws.class.name} redirect requested: #{response.code} #{response.message} #####")
577
+ @aws.logger.info("##### New location: #{location} #####")
578
+ # ... fix the connection data
579
+ request[:server] = URI.parse(location).host
580
+ request[:protocol] = URI.parse(location).scheme
581
+ request[:port] = URI.parse(location).port
582
+ end
583
+ end
584
+ else # ... it is not a xml document(probably just a html page?)
585
+ @aws.last_errors = [[response.code, "#{response.message} (#{request_text_data})"]]
586
+ @aws.last_request_id = '-undefined-'
587
+ last_errors_text = response.message
588
+ end
589
+ # now - check the error
590
+ unless redirect_detected
591
+ @errors_list.each do |error_to_find|
592
+ if last_errors_text[/#{error_to_find}/i]
593
+ error_found = true
594
+ error_match = error_to_find
595
+ @aws.logger.warn("##### Retry is needed, error pattern match: #{error_to_find} #####")
596
+ break
597
+ end
598
+ end
599
+ end
600
+ # check the time has gone from the first error come
601
+ if redirect_detected || error_found
602
+ # Close the connection to the server and recreate a new one.
603
+ # It may have a chance that one server is a semi-down and reconnection
604
+ # will help us to connect to the other server
605
+ if !redirect_detected && @close_on_error
606
+ @aws.connection.finish "#{self.class.name}: error match to pattern '#{error_match}'"
607
+ end
526
608
 
527
- # Returns false if
528
- def check(request) #:nodoc:
529
- result = false
530
- error_found = false
531
- redirect_detected= false
532
- error_match = nil
533
- last_errors_text = ''
534
- response = @aws.last_response
535
- # log error
536
- request_text_data = "#{request[:server]}:#{request[:port]}#{request[:request].path}"
537
- # is this a redirect?
538
- # yes!
539
- if response.is_a?(Net::HTTPRedirection)
540
- redirect_detected = true
541
- else
542
- # no, it's an error ...
543
- @aws.logger.warn("##### #{@aws.class.name} returned an error: #{response.code} #{response.message}\n#{response.body} #####")
544
- @aws.logger.warn("##### #{@aws.class.name} request: #{request_text_data} ####")
545
- end
546
- # Check response body: if it is an Amazon XML document or not:
547
- if redirect_detected || (response.body && response.body[/<\?xml/]) # ... it is a xml document
548
- @aws.class.bench_xml.add! do
549
- error_parser = RightErrorResponseParser.new
550
- error_parser.parse(response)
551
- @aws.last_errors = error_parser.errors
552
- @aws.last_request_id = error_parser.requestID
553
- last_errors_text = @aws.last_errors.flatten.join("\n")
554
- # on redirect :
555
- if redirect_detected
556
- location = response['location']
557
- # ... log information and ...
558
- @aws.logger.info("##### #{@aws.class.name} redirect requested: #{response.code} #{response.message} #####")
559
- @aws.logger.info("##### New location: #{location} #####")
560
- # ... fix the connection data
561
- request[:server] = URI.parse(location).host
562
- request[:protocol] = URI.parse(location).scheme
563
- request[:port] = URI.parse(location).port
564
- end
565
- end
566
- else # ... it is not a xml document(probably just a html page?)
567
- @aws.last_errors = [[response.code, "#{response.message} (#{request_text_data})"]]
568
- @aws.last_request_id = '-undefined-'
569
- last_errors_text = response.message
570
- end
571
- # now - check the error
572
- unless redirect_detected
573
- @errors_list.each do |error_to_find|
574
- if last_errors_text[/#{error_to_find}/i]
575
- error_found = true
576
- error_match = error_to_find
577
- @aws.logger.warn("##### Retry is needed, error pattern match: #{error_to_find} #####")
578
- break
579
- end
580
- end
581
- end
582
- # check the time has gone from the first error come
583
- if redirect_detected || error_found
584
- # Close the connection to the server and recreate a new one.
585
- # It may have a chance that one server is a semi-down and reconnection
586
- # will help us to connect to the other server
587
- if !redirect_detected && @close_on_error
588
- @aws.connection.finish "#{self.class.name}: error match to pattern '#{error_match}'"
589
- end
590
-
591
- if (Time.now < @stop_at)
592
- @retries += 1
593
- unless redirect_detected
594
- @aws.logger.warn("##### Retry ##{@retries} is being performed. Sleeping for #{@reiteration_delay} sec. Whole time: #{Time.now-@started_at} sec ####")
595
- sleep @reiteration_delay
596
- @reiteration_delay *= 2
597
-
598
- # Always make sure that the fp is set to point to the beginning(?)
599
- # of the File/IO. TODO: it assumes that offset is 0, which is bad.
600
- if(request[:request].body_stream && request[:request].body_stream.respond_to?(:pos))
601
- begin
602
- request[:request].body_stream.pos = 0
603
- rescue Exception => e
604
- @logger.warn("Retry may fail due to unable to reset the file pointer" +
605
- " -- #{self.class.name} : #{e.inspect}")
606
- end
609
+ if (Time.now < @stop_at)
610
+ @retries += 1
611
+ unless redirect_detected
612
+ @aws.logger.warn("##### Retry ##{@retries} is being performed. Sleeping for #{@reiteration_delay} sec. Whole time: #{Time.now-@started_at} sec ####")
613
+ sleep @reiteration_delay
614
+ @reiteration_delay *= 2
615
+
616
+ # Always make sure that the fp is set to point to the beginning(?)
617
+ # of the File/IO. TODO: it assumes that offset is 0, which is bad.
618
+ if (request[:request].body_stream && request[:request].body_stream.respond_to?(:pos))
619
+ begin
620
+ request[:request].body_stream.pos = 0
621
+ rescue Exception => e
622
+ @logger.warn("Retry may fail due to unable to reset the file pointer" +
623
+ " -- #{self.class.name} : #{e.inspect}")
624
+ end
625
+ end
626
+ else
627
+ @aws.logger.info("##### Retry ##{@retries} is being performed due to a redirect. ####")
628
+ end
629
+ result = @aws.request_info(request, @parser)
630
+ else
631
+ @aws.logger.warn("##### Ooops, time is over... ####")
632
+ end
633
+ # aha, this is unhandled error:
634
+ elsif @close_on_error
635
+ # Is this a 5xx error ?
636
+ if @aws.last_response.code.to_s[/^5\d\d$/]
637
+ @aws.connection.finish "#{self.class.name}: code: #{@aws.last_response.code}: '#{@aws.last_response.message}'"
638
+ # Is this a 4xx error ?
639
+ elsif @aws.last_response.code.to_s[/^4\d\d$/] && @close_on_4xx_probability > rand(100)
640
+ @aws.connection.finish "#{self.class.name}: code: #{@aws.last_response.code}: '#{@aws.last_response.message}', " +
641
+ "probability: #{@close_on_4xx_probability}%"
642
+ end
607
643
  end
608
- else
609
- @aws.logger.info("##### Retry ##{@retries} is being performed due to a redirect. ####")
610
- end
611
- result = @aws.request_info(request, @parser)
612
- else
613
- @aws.logger.warn("##### Ooops, time is over... ####")
614
- end
615
- # aha, this is unhandled error:
616
- elsif @close_on_error
617
- # Is this a 5xx error ?
618
- if @aws.last_response.code.to_s[/^5\d\d$/]
619
- @aws.connection.finish "#{self.class.name}: code: #{@aws.last_response.code}: '#{@aws.last_response.message}'"
620
- # Is this a 4xx error ?
621
- elsif @aws.last_response.code.to_s[/^4\d\d$/] && @close_on_4xx_probability > rand(100)
622
- @aws.connection.finish "#{self.class.name}: code: #{@aws.last_response.code}: '#{@aws.last_response.message}', " +
623
- "probability: #{@close_on_4xx_probability}%"
624
- end
625
- end
626
- result
644
+ result
645
+ end
646
+
627
647
  end
628
648
 
629
- end
630
649
 
650
+ #-----------------------------------------------------------------
631
651
 
632
- #-----------------------------------------------------------------
652
+ class RightSaxParserCallback #:nodoc:
653
+ def self.include_callback
654
+ include XML::SaxParser::Callbacks
655
+ end
633
656
 
634
- class RightSaxParserCallback #:nodoc:
635
- def self.include_callback
636
- include XML::SaxParser::Callbacks
637
- end
638
- def initialize(right_aws_parser)
639
- @right_aws_parser = right_aws_parser
640
- end
641
- def on_start_element(name, attr_hash)
642
- @right_aws_parser.tag_start(name, attr_hash)
643
- end
644
- def on_characters(chars)
645
- @right_aws_parser.text(chars)
646
- end
647
- def on_end_element(name)
648
- @right_aws_parser.tag_end(name)
649
- end
650
- def on_start_document; end
651
- def on_comment(msg); end
652
- def on_processing_instruction(target, data); end
653
- def on_cdata_block(cdata); end
654
- def on_end_document; end
655
- end
656
-
657
- class RightAWSParser #:nodoc:
658
- # default parsing library
659
- DEFAULT_XML_LIBRARY = 'rexml'
660
- # a list of supported parsers
661
- @@supported_xml_libs = [DEFAULT_XML_LIBRARY, 'libxml']
662
-
663
- @@xml_lib = DEFAULT_XML_LIBRARY # xml library name: 'rexml' | 'libxml'
664
- def self.xml_lib
665
- @@xml_lib
666
- end
667
- def self.xml_lib=(new_lib_name)
668
- @@xml_lib = new_lib_name
669
- end
657
+ def initialize(right_aws_parser)
658
+ @right_aws_parser = right_aws_parser
659
+ end
670
660
 
671
- attr_accessor :result
672
- attr_reader :xmlpath
673
- attr_accessor :xml_lib
674
-
675
- def initialize(params={})
676
- @xmlpath = ''
677
- @result = false
678
- @text = ''
679
- @xml_lib = params[:xml_lib] || @@xml_lib
680
- @logger = params[:logger]
681
- reset
682
- end
683
- def tag_start(name, attributes)
684
- @text = ''
685
- tagstart(name, attributes)
686
- @xmlpath += @xmlpath.empty? ? name : "/#{name}"
687
- end
688
- def tag_end(name)
689
- if @xmlpath =~ /^(.*?)\/?#{name}$/
690
- @xmlpath = $1
691
- end
692
- tagend(name)
693
- end
694
- def text(text)
695
- @text += text
696
- tagtext(text)
697
- end
698
- # Parser method.
699
- # Params:
700
- # xml_text - xml message text(String) or Net:HTTPxxx instance (response)
701
- # params[:xml_lib] - library name: 'rexml' | 'libxml'
702
- def parse(xml_text, params={})
703
- # Get response body
704
- unless xml_text.is_a?(String)
705
- xml_text = xml_text.body.respond_to?(:force_encoding) ? xml_text.body.force_encoding("UTF-8") : xml_text.body
706
- end
707
-
708
- @xml_lib = params[:xml_lib] || @xml_lib
709
- # check that we had no problems with this library otherwise use default
710
- @xml_lib = DEFAULT_XML_LIBRARY unless @@supported_xml_libs.include?(@xml_lib)
711
- # load xml library
712
- if @xml_lib=='libxml' && !defined?(XML::SaxParser)
713
- begin
714
- require 'xml/libxml'
715
- # is it new ? - Setup SaxParserCallback
716
- if XML::Parser::VERSION >= '0.5.1.0'
717
- RightSaxParserCallback.include_callback
718
- end
719
- rescue LoadError => e
720
- @@supported_xml_libs.delete(@xml_lib)
721
- @xml_lib = DEFAULT_XML_LIBRARY
722
- if @logger
723
- @logger.error e.inspect
724
- @logger.error e.backtrace
725
- @logger.info "Can not load 'libxml' library. '#{DEFAULT_XML_LIBRARY}' is used for parsing."
726
- end
727
- end
728
- end
729
- # Parse the xml text
730
- case @xml_lib
731
- when 'libxml'
732
- xml = XML::SaxParser.new
733
- xml.string = xml_text
734
- # check libxml-ruby version
735
- if XML::Parser::VERSION >= '0.5.1.0'
736
- xml.callbacks = RightSaxParserCallback.new(self)
737
- else
738
- xml.on_start_element{|name, attr_hash| self.tag_start(name, attr_hash)}
739
- xml.on_characters{ |text| self.text(text)}
740
- xml.on_end_element{ |name| self.tag_end(name)}
741
- end
742
- xml.parse
743
- else
744
- REXML::Document.parse_stream(xml_text, self)
745
- end
661
+ def on_start_element(name, attr_hash)
662
+ @right_aws_parser.tag_start(name, attr_hash)
663
+ end
664
+
665
+ def on_characters(chars)
666
+ @right_aws_parser.text(chars)
667
+ end
668
+
669
+ def on_end_element(name)
670
+ @right_aws_parser.tag_end(name)
671
+ end
672
+
673
+ def on_start_document;
674
+ end
675
+
676
+ def on_comment(msg)
677
+ ;
678
+ end
679
+
680
+ def on_processing_instruction(target, data)
681
+ ;
682
+ end
683
+
684
+ def on_cdata_block(cdata)
685
+ ;
686
+ end
687
+
688
+ def on_end_document;
689
+ end
746
690
  end
747
- # Parser must have a lots of methods
748
- # (see /usr/lib/ruby/1.8/rexml/parsers/streamparser.rb)
749
- # We dont need most of them in RightAWSParser and method_missing helps us
750
- # to skip their definition
751
- def method_missing(method, *params)
752
- # if the method is one of known - just skip it ...
753
- return if [:comment, :attlistdecl, :notationdecl, :elementdecl,
754
- :entitydecl, :cdata, :xmldecl, :attlistdecl, :instruction,
755
- :doctype].include?(method)
756
- # ... else - call super to raise an exception
757
- super(method, params)
691
+
692
+ class RightAWSParser #:nodoc:
693
+ # default parsing library
694
+ DEFAULT_XML_LIBRARY = 'rexml'
695
+ # a list of supported parsers
696
+ @@supported_xml_libs = [DEFAULT_XML_LIBRARY, 'libxml']
697
+
698
+ @@xml_lib = DEFAULT_XML_LIBRARY # xml library name: 'rexml' | 'libxml'
699
+ def self.xml_lib
700
+ @@xml_lib
701
+ end
702
+ def self.xml_lib=(new_lib_name)
703
+ @@xml_lib = new_lib_name
704
+ end
705
+
706
+ attr_accessor :result
707
+ attr_reader :xmlpath
708
+ attr_accessor :xml_lib
709
+
710
+ def initialize(params={})
711
+ @xmlpath = ''
712
+ @result = false
713
+ @text = ''
714
+ @xml_lib = params[:xml_lib] || @@xml_lib
715
+ @logger = params[:logger]
716
+ reset
717
+ end
718
+
719
+ def tag_start(name, attributes)
720
+ @text = ''
721
+ tagstart(name, attributes)
722
+ @xmlpath += @xmlpath.empty? ? name : "/#{name}"
723
+ end
724
+
725
+ def tag_end(name)
726
+ if @xmlpath =~ /^(.*?)\/?#{name}$/
727
+ @xmlpath = $1
728
+ end
729
+ tagend(name)
730
+ end
731
+
732
+ def text(text)
733
+ @text += text
734
+ tagtext(text)
735
+ end
736
+
737
+ # Parser method.
738
+ # Params:
739
+ # xml_text - xml message text(String) or Net:HTTPxxx instance (response)
740
+ # params[:xml_lib] - library name: 'rexml' | 'libxml'
741
+ def parse(xml_text, params={})
742
+ # Get response body
743
+ unless xml_text.is_a?(String)
744
+ xml_text = xml_text.body.respond_to?(:force_encoding) ? xml_text.body.force_encoding("UTF-8") : xml_text.body
745
+ end
746
+
747
+ @xml_lib = params[:xml_lib] || @xml_lib
748
+ # check that we had no problems with this library otherwise use default
749
+ @xml_lib = DEFAULT_XML_LIBRARY unless @@supported_xml_libs.include?(@xml_lib)
750
+ # load xml library
751
+ if @xml_lib=='libxml' && !defined?(XML::SaxParser)
752
+ begin
753
+ require 'xml/libxml'
754
+ # is it new ? - Setup SaxParserCallback
755
+ if XML::Parser::VERSION >= '0.5.1.0'
756
+ RightSaxParserCallback.include_callback
757
+ end
758
+ rescue LoadError => e
759
+ @@supported_xml_libs.delete(@xml_lib)
760
+ @xml_lib = DEFAULT_XML_LIBRARY
761
+ if @logger
762
+ @logger.error e.inspect
763
+ @logger.error e.backtrace
764
+ @logger.info "Can not load 'libxml' library. '#{DEFAULT_XML_LIBRARY}' is used for parsing."
765
+ end
766
+ end
767
+ end
768
+ # Parse the xml text
769
+ case @xml_lib
770
+ when 'libxml'
771
+ xml = XML::SaxParser.new
772
+ xml.string = xml_text
773
+ # check libxml-ruby version
774
+ if XML::Parser::VERSION >= '0.5.1.0'
775
+ xml.callbacks = RightSaxParserCallback.new(self)
776
+ else
777
+ xml.on_start_element{|name, attr_hash| self.tag_start(name, attr_hash)}
778
+ xml.on_characters{ |text| self.text(text)}
779
+ xml.on_end_element{ |name| self.tag_end(name)}
780
+ end
781
+ xml.parse
782
+ else
783
+ REXML::Document.parse_stream(xml_text, self)
784
+ end
785
+ end
786
+
787
+ # Parser must have a lots of methods
788
+ # (see /usr/lib/ruby/1.8/rexml/parsers/streamparser.rb)
789
+ # We dont need most of them in RightAWSParser and method_missing helps us
790
+ # to skip their definition
791
+ def method_missing(method, *params)
792
+ # if the method is one of known - just skip it ...
793
+ return if [:comment, :attlistdecl, :notationdecl, :elementdecl,
794
+ :entitydecl, :cdata, :xmldecl, :attlistdecl, :instruction,
795
+ :doctype].include?(method)
796
+ # ... else - call super to raise an exception
797
+ super(method, params)
798
+ end
799
+
800
+ # the functions to be overriden by children (if nessesery)
801
+ def reset;
802
+ end
803
+
804
+ def tagstart(name, attributes)
805
+ ;
806
+ end
807
+
808
+ def tagend(name)
809
+ ;
810
+ end
811
+
812
+ def tagtext(text)
813
+ ;
814
+ end
758
815
  end
759
- # the functions to be overriden by children (if nessesery)
760
- def reset ; end
761
- def tagstart(name, attributes); end
762
- def tagend(name) ; end
763
- def tagtext(text) ; end
764
- end
765
816
 
766
- #-----------------------------------------------------------------
767
- # PARSERS: Errors
768
- #-----------------------------------------------------------------
817
+ #-----------------------------------------------------------------
818
+ # PARSERS: Errors
819
+ #-----------------------------------------------------------------
769
820
 
770
821
  #<Error>
771
822
  # <Code>TemporaryRedirect</Code>
@@ -776,40 +827,46 @@ module RightAws
776
827
  # <Bucket>bucket-for-k</Bucket>
777
828
  #</Error>
778
829
 
779
- class RightErrorResponseParser < RightAWSParser #:nodoc:
780
- attr_accessor :errors # array of hashes: error/message
781
- attr_accessor :requestID
830
+ class RightErrorResponseParser < RightAWSParser #:nodoc:
831
+ attr_accessor :errors # array of hashes: error/message
832
+ attr_accessor :requestID
782
833
  # attr_accessor :endpoint, :host_id, :bucket
783
- def tagend(name)
784
- case name
785
- when 'RequestID' ; @requestID = @text
786
- when 'Code' ; @code = @text
787
- when 'Message' ; @message = @text
834
+ def tagend(name)
835
+ case name
836
+ when 'RequestID';
837
+ @requestID = @text
838
+ when 'Code';
839
+ @code = @text
840
+ when 'Message';
841
+ @message = @text
788
842
  # when 'Endpoint' ; @endpoint = @text
789
843
  # when 'HostId' ; @host_id = @text
790
844
  # when 'Bucket' ; @bucket = @text
791
- when 'Error' ; @errors << [ @code, @message ]
792
- end
793
- end
794
- def reset
795
- @errors = []
845
+ when 'Error';
846
+ @errors << [ @code, @message ]
847
+ end
848
+ end
849
+
850
+ def reset
851
+ @errors = []
852
+ end
796
853
  end
797
- end
798
-
799
- # Dummy parser - does nothing
800
- # Returns the original params back
801
- class RightDummyParser # :nodoc:
802
- attr_accessor :result
803
- def parse(response, params={})
804
- @result = [response, params]
854
+
855
+ # Dummy parser - does nothing
856
+ # Returns the original params back
857
+ class RightDummyParser # :nodoc:
858
+ attr_accessor :result
859
+
860
+ def parse(response, params={})
861
+ @result = [response, params]
862
+ end
805
863
  end
806
- end
807
864
 
808
- class RightHttp2xxParser < RightAWSParser # :nodoc:
809
- def parse(response)
810
- @result = response.is_a?(Net::HTTPSuccess)
865
+ class RightHttp2xxParser < RightAWSParser # :nodoc:
866
+ def parse(response)
867
+ @result = response.is_a?(Net::HTTPSuccess)
868
+ end
811
869
  end
812
- end
813
870
 
814
871
  end
815
872