appoxy-aws 1.11.34 → 1.11.35

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (5) hide show
  1. data/lib/awsbase/right_awsbase.rb +759 -702
  2. data/lib/ec2/right_mon_interface.rb +222 -0
  3. data/lib/sdb/active_sdb.rb +1 -0
  4. data/lib/sdb/right_sdb_interface.rb +5 -2
  5. metadata +3 -2
data/lib/awsbase/right_awsbase.rb CHANGED
@@ -23,749 +23,800 @@
23
23
 
24
24
  # Test
25
25
  module RightAws
26
- require 'digest/md5'
27
- require 'pp'
28
- require 'cgi'
29
-
30
- class AwsUtils #:nodoc:
31
- @@digest1 = OpenSSL::Digest::Digest.new("sha1")
32
- @@digest256 = nil
33
- if OpenSSL::OPENSSL_VERSION_NUMBER > 0x00908000
34
- @@digest256 = OpenSSL::Digest::Digest.new("sha256") rescue nil # Some installation may not support sha256
35
- end
26
+ require 'digest/md5'
27
+ require 'pp'
28
+ require 'cgi'
29
+ require 'uri'
30
+
31
+ class AwsUtils #:nodoc:
32
+ @@digest1 = OpenSSL::Digest::Digest.new("sha1")
33
+ @@digest256 = nil
34
+ if OpenSSL::OPENSSL_VERSION_NUMBER > 0x00908000
35
+ @@digest256 = OpenSSL::Digest::Digest.new("sha256") rescue nil # Some installation may not support sha256
36
+ end
36
37
 
37
- def self.sign(aws_secret_access_key, auth_string)
38
- Base64.encode64(OpenSSL::HMAC.digest(@@digest1, aws_secret_access_key, auth_string)).strip
39
- end
38
+ def self.sign(aws_secret_access_key, auth_string)
39
+ Base64.encode64(OpenSSL::HMAC.digest(@@digest1, aws_secret_access_key, auth_string)).strip
40
+ end
40
41
 
41
- # Escape a string accordingly Amazon rulles
42
- # http://docs.amazonwebservices.com/AmazonSimpleDB/2007-11-07/DeveloperGuide/index.html?REST_RESTAuth.html
43
- def self.amz_escape(param)
44
- return CGI.escape(param.to_s).gsub("%7E", "~").gsub("+", "%20") # from: http://umlaut.rubyforge.org/svn/trunk/lib/aws_product_sign.rb
45
- #param.to_s.gsub(/([^a-zA-Z0-9._~-]+)/n) do
46
- # '%' + $1.unpack('H2' * $1.size).join('%').upcase
47
- #end
48
- end
49
42
 
50
- # Set a timestamp and a signature version
51
- def self.fix_service_params(service_hash, signature)
52
- service_hash["Timestamp"] ||= Time.now.utc.strftime("%Y-%m-%dT%H:%M:%S.000Z") unless service_hash["Expires"]
53
- service_hash["SignatureVersion"] = signature
54
- service_hash
55
- end
43
+ # Set a timestamp and a signature version
44
+ def self.fix_service_params(service_hash, signature)
45
+ service_hash["Timestamp"] ||= Time.now.utc.strftime("%Y-%m-%dT%H:%M:%S.000Z") unless service_hash["Expires"]
46
+ service_hash["SignatureVersion"] = signature
47
+ service_hash
48
+ end
56
49
 
57
- # Signature Version 0
58
- # A deprecated guy (should work till septemper 2009)
59
- def self.sign_request_v0(aws_secret_access_key, service_hash)
60
- fix_service_params(service_hash, '0')
61
- string_to_sign = "#{service_hash['Action']}#{service_hash['Timestamp'] || service_hash['Expires']}"
62
- service_hash['Signature'] = AwsUtils::sign(aws_secret_access_key, string_to_sign)
63
- service_hash.to_a.collect{|key,val| "#{amz_escape(key)}=#{amz_escape(val.to_s)}" }.join("&")
64
- end
50
+ # Signature Version 0
51
+ # A deprecated guy (should work till septemper 2009)
52
+ def self.sign_request_v0(aws_secret_access_key, service_hash)
53
+ fix_service_params(service_hash, '0')
54
+ string_to_sign = "#{service_hash['Action']}#{service_hash['Timestamp'] || service_hash['Expires']}"
55
+ service_hash['Signature'] = AwsUtils::sign(aws_secret_access_key, string_to_sign)
56
+ service_hash.to_a.collect{|key, val| "#{amz_escape(key)}=#{amz_escape(val.to_s)}" }.join("&")
57
+ end
65
58
 
66
- # Signature Version 1
67
- # Another deprecated guy (should work till septemper 2009)
68
- def self.sign_request_v1(aws_secret_access_key, service_hash)
69
- fix_service_params(service_hash, '1')
70
- string_to_sign = service_hash.sort{|a,b| (a[0].to_s.downcase)<=>(b[0].to_s.downcase)}.to_s
71
- service_hash['Signature'] = AwsUtils::sign(aws_secret_access_key, string_to_sign)
72
- service_hash.to_a.collect{|key,val| "#{amz_escape(key)}=#{amz_escape(val.to_s)}" }.join("&")
73
- end
59
+ # Signature Version 1
60
+ # Another deprecated guy (should work till septemper 2009)
61
+ def self.sign_request_v1(aws_secret_access_key, service_hash)
62
+ fix_service_params(service_hash, '1')
63
+ string_to_sign = service_hash.sort{|a, b| (a[0].to_s.downcase)<=>(b[0].to_s.downcase)}.to_s
64
+ service_hash['Signature'] = AwsUtils::sign(aws_secret_access_key, string_to_sign)
65
+ service_hash.to_a.collect{|key, val| "#{amz_escape(key)}=#{amz_escape(val.to_s)}" }.join("&")
66
+ end
74
67
 
75
- # Signature Version 2
76
- # EC2, SQS and SDB requests must be signed by this guy.
77
- # See: http://docs.amazonwebservices.com/AmazonSimpleDB/2007-11-07/DeveloperGuide/index.html?REST_RESTAuth.html
78
- # http://developer.amazonwebservices.com/connect/entry.jspa?externalID=1928
79
- def self.sign_request_v2(aws_secret_access_key, service_hash, http_verb, host, uri)
80
- fix_service_params(service_hash, '2')
81
- # select a signing method (make an old openssl working with sha1)
82
- # make 'HmacSHA256' to be a default one
83
- service_hash['SignatureMethod'] = 'HmacSHA256' unless ['HmacSHA256', 'HmacSHA1'].include?(service_hash['SignatureMethod'])
84
- service_hash['SignatureMethod'] = 'HmacSHA1' unless @@digest256
85
- # select a digest
86
- digest = (service_hash['SignatureMethod'] == 'HmacSHA256' ? @@digest256 : @@digest1)
87
- # form string to sign
88
- canonical_string = service_hash.keys.sort.map do |key|
89
- "#{amz_escape(key)}=#{amz_escape(service_hash[key])}"
90
- end.join('&')
91
- string_to_sign = "#{http_verb.to_s.upcase}\n#{host.downcase}\n#{uri}\n#{canonical_string}"
92
- # sign the string
93
- signature = amz_escape(Base64.encode64(OpenSSL::HMAC.digest(digest, aws_secret_access_key, string_to_sign)).strip)
94
- "#{canonical_string}&Signature=#{signature}"
95
- end
68
+ # Signature Version 2
69
+ # EC2, SQS and SDB requests must be signed by this guy.
70
+ # See: http://docs.amazonwebservices.com/AmazonSimpleDB/2007-11-07/DeveloperGuide/index.html?REST_RESTAuth.html
71
+ # http://developer.amazonwebservices.com/connect/entry.jspa?externalID=1928
72
+ def self.sign_request_v2(aws_secret_access_key, service_hash, http_verb, host, uri)
73
+ fix_service_params(service_hash, '2')
74
+ # select a signing method (make an old openssl working with sha1)
75
+ # make 'HmacSHA256' to be a default one
76
+ service_hash['SignatureMethod'] = 'HmacSHA256' unless ['HmacSHA256', 'HmacSHA1'].include?(service_hash['SignatureMethod'])
77
+ service_hash['SignatureMethod'] = 'HmacSHA1' unless @@digest256
78
+ # select a digest
79
+ digest = (service_hash['SignatureMethod'] == 'HmacSHA256' ? @@digest256 : @@digest1)
80
+ # form string to sign
81
+ canonical_string = service_hash.keys.sort.map do |key|
82
+ "#{amz_escape(key)}=#{amz_escape(service_hash[key])}"
83
+ end.join('&')
84
+ string_to_sign = "#{http_verb.to_s.upcase}\n#{host.downcase}\n#{uri}\n#{canonical_string}"
85
+ # sign the string
86
+ signature = escape_sig(Base64.encode64(OpenSSL::HMAC.digest(digest, aws_secret_access_key, string_to_sign)).strip)
87
+ "#{canonical_string}&Signature=#{signature}"
88
+ end
96
89
 
97
- # From Amazon's SQS Dev Guide, a brief description of how to escape:
98
- # "URL encode the computed signature and other query parameters as specified in
99
- # RFC1738, section 2.2. In addition, because the + character is interpreted as a blank space
100
- # by Sun Java classes that perform URL decoding, make sure to encode the + character
101
- # although it is not required by RFC1738."
102
- # Avoid using CGI::escape to escape URIs.
103
- # CGI::escape will escape characters in the protocol, host, and port
104
- # sections of the URI. Only target chars in the query
105
- # string should be escaped.
106
- def self.URLencode(raw)
107
- e = URI.escape(raw)
108
- e.gsub(/\+/, "%2b")
109
- end
90
+ # Escape a string accordingly Amazon rulles
91
+ # http://docs.amazonwebservices.com/AmazonSimpleDB/2007-11-07/DeveloperGuide/index.html?REST_RESTAuth.html
92
+ def self.amz_escape(param)
93
+ #return CGI.escape(param.to_s).gsub("%7E", "~").gsub("+", "%20") # from: http://umlaut.rubyforge.org/svn/trunk/lib/aws_product_sign.rb
94
+
95
+ #param.to_s.gsub(/([^a-zA-Z0-9._~-]+)/n) do
96
+ # '%' + $1.unpack('H2' * $1.size).join('%').upcase
97
+ #end
98
+ e = CGI.escape(param.to_s)
99
+ e = e.gsub("+", "%20")
100
+ e = e.gsub("*", "%2A")
101
+ e = e.gsub("%7E", "~")
102
+ end
110
103
 
111
- def self.allow_only(allowed_keys, params)
112
- bogus_args = []
113
- params.keys.each {|p| bogus_args.push(p) unless allowed_keys.include?(p) }
114
- raise AwsError.new("The following arguments were given but are not legal for the function call #{caller_method}: #{bogus_args.inspect}") if bogus_args.length > 0
115
- end
116
104
 
117
- def self.mandatory_arguments(required_args, params)
118
- rargs = required_args.dup
119
- params.keys.each {|p| rargs.delete(p)}
120
- raise AwsError.new("The following mandatory arguments were not provided to #{caller_method}: #{rargs.inspect}") if rargs.length > 0
121
- end
105
+ def self.escape_sig(raw)
106
+ e = CGI.escape(raw)
107
+ end
122
108
 
123
- def self.caller_method
124
- caller[1]=~/`(.*?)'/
125
- $1
126
- end
109
+ # From Amazon's SQS Dev Guide, a brief description of how to escape:
110
+ # "URL encode the computed signature and other query parameters as specified in
111
+ # RFC1738, section 2.2. In addition, because the + character is interpreted as a blank space
112
+ # by Sun Java classes that perform URL decoding, make sure to encode the + character
113
+ # although it is not required by RFC1738."
114
+ # Avoid using CGI::escape to escape URIs.
115
+ # CGI::escape will escape characters in the protocol, host, and port
116
+ # sections of the URI. Only target chars in the query
117
+ # string should be escaped.
118
+ def self.URLencode(raw)
119
+ e = URI.escape(raw)
120
+ e.gsub(/\+/, "%2b")
121
+ end
127
122
 
128
- end
129
123
 
130
- class AwsBenchmarkingBlock #:nodoc:
131
- attr_accessor :xml, :service
132
- def initialize
133
- # Benchmark::Tms instance for service (Ec2, S3, or SQS) access benchmarking.
134
- @service = Benchmark::Tms.new()
135
- # Benchmark::Tms instance for XML parsing benchmarking.
136
- @xml = Benchmark::Tms.new()
124
+ def self.allow_only(allowed_keys, params)
125
+ bogus_args = []
126
+ params.keys.each {|p| bogus_args.push(p) unless allowed_keys.include?(p) }
127
+ raise AwsError.new("The following arguments were given but are not legal for the function call #{caller_method}: #{bogus_args.inspect}") if bogus_args.length > 0
128
+ end
129
+
130
+ def self.mandatory_arguments(required_args, params)
131
+ rargs = required_args.dup
132
+ params.keys.each {|p| rargs.delete(p)}
133
+ raise AwsError.new("The following mandatory arguments were not provided to #{caller_method}: #{rargs.inspect}") if rargs.length > 0
134
+ end
135
+
136
+ def self.caller_method
137
+ caller[1]=~/`(.*?)'/
138
+ $1
139
+ end
140
+
137
141
  end
138
- end
139
-
140
- class AwsNoChange < RuntimeError
141
- end
142
-
143
- class RightAwsBase
144
-
145
- # Amazon HTTP Error handling
146
-
147
- # Text, if found in an error message returned by AWS, indicates that this may be a transient
148
- # error. Transient errors are automatically retried with exponential back-off.
149
- AMAZON_PROBLEMS = [ 'internal service error',
150
- 'is currently unavailable',
151
- 'no response from',
152
- 'Please try again',
153
- 'InternalError',
154
- 'ServiceUnavailable', #from SQS docs
155
- 'Unavailable',
156
- 'This application is not currently available',
157
- 'InsufficientInstanceCapacity'
158
- ]
159
- @@amazon_problems = AMAZON_PROBLEMS
160
- # Returns a list of Amazon service responses which are known to be transient problems.
161
- # We have to re-request if we get any of them, because the problem will probably disappear.
162
- # By default this method returns the same value as the AMAZON_PROBLEMS const.
163
- def self.amazon_problems
164
- @@amazon_problems
142
+
143
+ class AwsBenchmarkingBlock #:nodoc:
144
+ attr_accessor :xml, :service
145
+
146
+ def initialize
147
+ # Benchmark::Tms instance for service (Ec2, S3, or SQS) access benchmarking.
148
+ @service = Benchmark::Tms.new()
149
+ # Benchmark::Tms instance for XML parsing benchmarking.
150
+ @xml = Benchmark::Tms.new()
151
+ end
165
152
  end
166
153
 
167
- # Sets the list of Amazon side problems. Use in conjunction with the
168
- # getter to append problems.
169
- def self.amazon_problems=(problems_list)
170
- @@amazon_problems = problems_list
154
+ class AwsNoChange < RuntimeError
171
155
  end
172
156
 
173
- end
157
+ class RightAwsBase
174
158
 
175
- module RightAwsBaseInterface
176
- DEFAULT_SIGNATURE_VERSION = '2'
159
+ # Amazon HTTP Error handling
160
+
161
+ # Text, if found in an error message returned by AWS, indicates that this may be a transient
162
+ # error. Transient errors are automatically retried with exponential back-off.
163
+ AMAZON_PROBLEMS = [ 'internal service error',
164
+ 'is currently unavailable',
165
+ 'no response from',
166
+ 'Please try again',
167
+ 'InternalError',
168
+ 'ServiceUnavailable', #from SQS docs
169
+ 'Unavailable',
170
+ 'This application is not currently available',
171
+ 'InsufficientInstanceCapacity'
172
+ ]
173
+ @@amazon_problems = AMAZON_PROBLEMS
174
+ # Returns a list of Amazon service responses which are known to be transient problems.
175
+ # We have to re-request if we get any of them, because the problem will probably disappear.
176
+ # By default this method returns the same value as the AMAZON_PROBLEMS const.
177
+ def self.amazon_problems
178
+ @@amazon_problems
179
+ end
180
+
181
+ # Sets the list of Amazon side problems. Use in conjunction with the
182
+ # getter to append problems.
183
+ def self.amazon_problems=(problems_list)
184
+ @@amazon_problems = problems_list
185
+ end
177
186
 
178
- @@caching = false
179
- def self.caching
180
- @@caching
181
- end
182
- def self.caching=(caching)
183
- @@caching = caching
184
187
  end
185
188
 
186
- # Current aws_access_key_id
187
- attr_reader :aws_access_key_id
188
- # Last HTTP request object
189
- attr_reader :last_request
190
- # Last HTTP response object
191
- attr_reader :last_response
192
- # Last AWS errors list (used by AWSErrorHandler)
193
- attr_accessor :last_errors
194
- # Last AWS request id (used by AWSErrorHandler)
195
- attr_accessor :last_request_id
196
- # Logger object
197
- attr_accessor :logger
198
- # Initial params hash
199
- attr_accessor :params
200
- # RightHttpConnection instance
201
- attr_reader :connection
202
- # Cache
203
- attr_reader :cache
204
- # Signature version (all services except s3)
205
- attr_reader :signature_version
206
-
207
- def init(service_info, aws_access_key_id, aws_secret_access_key, params={}) #:nodoc:
208
- @params = params
209
- raise AwsError.new("AWS access keys are required to operate on #{service_info[:name]}") \
189
+ module RightAwsBaseInterface
190
+ DEFAULT_SIGNATURE_VERSION = '2'
191
+
192
+ @@caching = false
193
+ def self.caching
194
+ @@caching
195
+ end
196
+ def self.caching=(caching)
197
+ @@caching = caching
198
+ end
199
+
200
+ # Current aws_access_key_id
201
+ attr_reader :aws_access_key_id
202
+ # Last HTTP request object
203
+ attr_reader :last_request
204
+ # Last HTTP response object
205
+ attr_reader :last_response
206
+ # Last AWS errors list (used by AWSErrorHandler)
207
+ attr_accessor :last_errors
208
+ # Last AWS request id (used by AWSErrorHandler)
209
+ attr_accessor :last_request_id
210
+ # Logger object
211
+ attr_accessor :logger
212
+ # Initial params hash
213
+ attr_accessor :params
214
+ # RightHttpConnection instance
215
+ attr_reader :connection
216
+ # Cache
217
+ attr_reader :cache
218
+ # Signature version (all services except s3)
219
+ attr_reader :signature_version
220
+
221
+ def init(service_info, aws_access_key_id, aws_secret_access_key, params={}) #:nodoc:
222
+ @params = params
223
+ raise AwsError.new("AWS access keys are required to operate on #{service_info[:name]}") \
210
224
  if aws_access_key_id.blank? || aws_secret_access_key.blank?
211
- @aws_access_key_id = aws_access_key_id
212
- @aws_secret_access_key = aws_secret_access_key
213
- # if the endpoint was explicitly defined - then use it
214
- if @params[:endpoint_url]
215
- @params[:server] = URI.parse(@params[:endpoint_url]).host
216
- @params[:port] = URI.parse(@params[:endpoint_url]).port
217
- @params[:service] = URI.parse(@params[:endpoint_url]).path
218
- @params[:protocol] = URI.parse(@params[:endpoint_url]).scheme
219
- @params[:region] = nil
220
- else
221
- @params[:server] ||= service_info[:default_host]
222
- @params[:server] = "#{@params[:region]}.#{@params[:server]}" if @params[:region]
223
- @params[:port] ||= service_info[:default_port]
224
- @params[:service] ||= service_info[:default_service]
225
- @params[:protocol] ||= service_info[:default_protocol]
226
- end
227
- if !@params[:multi_thread].nil? && @params[:connection_mode].nil? # user defined this
228
- @params[:connection_mode] = @params[:multi_thread] ? :per_thread : :single
229
- end
225
+ @aws_access_key_id = aws_access_key_id
226
+ @aws_secret_access_key = aws_secret_access_key
227
+ # if the endpoint was explicitly defined - then use it
228
+ if @params[:endpoint_url]
229
+ @params[:server] = URI.parse(@params[:endpoint_url]).host
230
+ @params[:port] = URI.parse(@params[:endpoint_url]).port
231
+ @params[:service] = URI.parse(@params[:endpoint_url]).path
232
+ @params[:protocol] = URI.parse(@params[:endpoint_url]).scheme
233
+ @params[:region] = nil
234
+ else
235
+ @params[:server] ||= service_info[:default_host]
236
+ @params[:server] = "#{@params[:region]}.#{@params[:server]}" if @params[:region]
237
+ @params[:port] ||= service_info[:default_port]
238
+ @params[:service] ||= service_info[:default_service]
239
+ @params[:protocol] ||= service_info[:default_protocol]
240
+ end
241
+ if !@params[:multi_thread].nil? && @params[:connection_mode].nil? # user defined this
242
+ @params[:connection_mode] = @params[:multi_thread] ? :per_thread : :single
243
+ end
230
244
  # @params[:multi_thread] ||= defined?(AWS_DAEMON)
231
- @params[:connection_mode] ||= :default
232
- @params[:connection_mode] = :per_request if @params[:connection_mode] == :default
233
- @logger = @params[:logger]
234
- @logger = RAILS_DEFAULT_LOGGER if !@logger && defined?(RAILS_DEFAULT_LOGGER)
235
- @logger = Logger.new(STDOUT) if !@logger
236
- @logger.info "New #{self.class.name} using #{@params[:connection_mode].to_s}-connection mode"
237
- @error_handler = nil
238
- @cache = {}
239
- @signature_version = (params[:signature_version] || DEFAULT_SIGNATURE_VERSION).to_s
240
- end
245
+ @params[:connection_mode] ||= :default
246
+ @params[:connection_mode] = :per_request if @params[:connection_mode] == :default
247
+ @logger = @params[:logger]
248
+ @logger = RAILS_DEFAULT_LOGGER if !@logger && defined?(RAILS_DEFAULT_LOGGER)
249
+ @logger = Logger.new(STDOUT) if !@logger
250
+ @logger.info "New #{self.class.name} using #{@params[:connection_mode].to_s}-connection mode"
251
+ @error_handler = nil
252
+ @cache = {}
253
+ @signature_version = (params[:signature_version] || DEFAULT_SIGNATURE_VERSION).to_s
254
+ end
241
255
 
242
- def signed_service_params(aws_secret_access_key, service_hash, http_verb=nil, host=nil, service=nil )
243
- case signature_version.to_s
244
- when '0' then AwsUtils::sign_request_v0(aws_secret_access_key, service_hash)
245
- when '1' then AwsUtils::sign_request_v1(aws_secret_access_key, service_hash)
246
- when '2' then AwsUtils::sign_request_v2(aws_secret_access_key, service_hash, http_verb, host, service)
247
- else raise AwsError.new("Unknown signature version (#{signature_version.to_s}) requested")
248
- end
249
- end
256
+ def signed_service_params(aws_secret_access_key, service_hash, http_verb=nil, host=nil, service=nil )
257
+ case signature_version.to_s
258
+ when '0' then
259
+ AwsUtils::sign_request_v0(aws_secret_access_key, service_hash)
260
+ when '1' then
261
+ AwsUtils::sign_request_v1(aws_secret_access_key, service_hash)
262
+ when '2' then
263
+ AwsUtils::sign_request_v2(aws_secret_access_key, service_hash, http_verb, host, service)
264
+ else
265
+ raise AwsError.new("Unknown signature version (#{signature_version.to_s}) requested")
266
+ end
267
+ end
250
268
 
251
- # Returns +true+ if the describe_xxx responses are being cached
252
- def caching?
253
- @params.key?(:cache) ? @params[:cache] : @@caching
254
- end
269
+ # Returns +true+ if the describe_xxx responses are being cached
270
+ def caching?
271
+ @params.key?(:cache) ? @params[:cache] : @@caching
272
+ end
255
273
 
256
- # Check if the aws function response hits the cache or not.
257
- # If the cache hits:
258
- # - raises an +AwsNoChange+ exception if +do_raise+ == +:raise+.
259
- # - returnes parsed response from the cache if it exists or +true+ otherwise.
260
- # If the cache miss or the caching is off then returns +false+.
261
- def cache_hits?(function, response, do_raise=:raise)
262
- result = false
263
- if caching?
264
- function = function.to_sym
265
- # get rid of requestId (this bad boy was added for API 2008-08-08+ and it is uniq for every response)
266
- response = response.sub(%r{<requestId>.+?</requestId>}, '')
267
- response_md5 =Digest::MD5.hexdigest(response).to_s
268
- # check for changes
269
- unless @cache[function] && @cache[function][:response_md5] == response_md5
270
- # well, the response is new, reset cache data
271
- update_cache(function, {:response_md5 => response_md5,
272
- :timestamp => Time.now,
273
- :hits => 0,
274
- :parsed => nil})
275
- else
276
- # aha, cache hits, update the data and throw an exception if needed
277
- @cache[function][:hits] += 1
278
- if do_raise == :raise
279
- raise(AwsNoChange, "Cache hit: #{function} response has not changed since "+
280
- "#{@cache[function][:timestamp].strftime('%Y-%m-%d %H:%M:%S')}, "+
281
- "hits: #{@cache[function][:hits]}.")
282
- else
283
- result = @cache[function][:parsed] || true
284
- end
285
- end
286
- end
287
- result
288
- end
274
+ # Check if the aws function response hits the cache or not.
275
+ # If the cache hits:
276
+ # - raises an +AwsNoChange+ exception if +do_raise+ == +:raise+.
277
+ # - returnes parsed response from the cache if it exists or +true+ otherwise.
278
+ # If the cache miss or the caching is off then returns +false+.
279
+ def cache_hits?(function, response, do_raise=:raise)
280
+ result = false
281
+ if caching?
282
+ function = function.to_sym
283
+ # get rid of requestId (this bad boy was added for API 2008-08-08+ and it is uniq for every response)
284
+ response = response.sub(%r{<requestId>.+?</requestId>}, '')
285
+ response_md5 =Digest::MD5.hexdigest(response).to_s
286
+ # check for changes
287
+ unless @cache[function] && @cache[function][:response_md5] == response_md5
288
+ # well, the response is new, reset cache data
289
+ update_cache(function, {:response_md5 => response_md5,
290
+ :timestamp => Time.now,
291
+ :hits => 0,
292
+ :parsed => nil})
293
+ else
294
+ # aha, cache hits, update the data and throw an exception if needed
295
+ @cache[function][:hits] += 1
296
+ if do_raise == :raise
297
+ raise(AwsNoChange, "Cache hit: #{function} response has not changed since "+
298
+ "#{@cache[function][:timestamp].strftime('%Y-%m-%d %H:%M:%S')}, "+
299
+ "hits: #{@cache[function][:hits]}.")
300
+ else
301
+ result = @cache[function][:parsed] || true
302
+ end
303
+ end
304
+ end
305
+ result
306
+ end
289
307
 
290
- def update_cache(function, hash)
291
- (@cache[function.to_sym] ||= {}).merge!(hash) if caching?
292
- end
308
+ def update_cache(function, hash)
309
+ (@cache[function.to_sym] ||= {}).merge!(hash) if caching?
310
+ end
293
311
 
294
- def on_exception(options={:raise=>true, :log=>true}) # :nodoc:
295
- raise if $!.is_a?(AwsNoChange)
296
- AwsError::on_aws_exception(self, options)
297
- end
312
+ def on_exception(options={:raise=>true, :log=>true}) # :nodoc:
313
+ raise if $!.is_a?(AwsNoChange)
314
+ AwsError::on_aws_exception(self, options)
315
+ end
298
316
 
299
- # Return +true+ if this instance works in multi_thread mode and +false+ otherwise.
300
- def multi_thread
301
- @params[:multi_thread]
302
- end
317
+ # Return +true+ if this instance works in multi_thread mode and +false+ otherwise.
318
+ def multi_thread
319
+ @params[:multi_thread]
320
+ end
303
321
 
304
- def request_info_impl(connection, benchblock, request, parser, &block) #:nodoc:
305
- @connection = connection
306
- @last_request = request[:request]
307
- @last_response = nil
308
- response=nil
309
- blockexception = nil
310
-
311
- if(block != nil)
312
- # TRB 9/17/07 Careful - because we are passing in blocks, we get a situation where
313
- # an exception may get thrown in the block body (which is high-level
314
- # code either here or in the application) but gets caught in the
315
- # low-level code of HttpConnection. The solution is not to let any
316
- # exception escape the block that we pass to HttpConnection::request.
317
- # Exceptions can originate from code directly in the block, or from user
318
- # code called in the other block which is passed to response.read_body.
319
- benchblock.service.add! do
320
- responsehdr = @connection.request(request) do |response|
321
- #########
322
- begin
323
- @last_response = response
324
- if response.is_a?(Net::HTTPSuccess)
325
- @error_handler = nil
326
- response.read_body(&block)
327
- else
328
- @error_handler = AWSErrorHandler.new(self, parser, :errors_list => self.class.amazon_problems) unless @error_handler
329
- check_result = @error_handler.check(request)
330
- if check_result
331
- @error_handler = nil
332
- return check_result
322
+ def request_info_impl(connection, benchblock, request, parser, &block) #:nodoc:
323
+ @connection = connection
324
+ @last_request = request[:request]
325
+ @last_response = nil
326
+ response=nil
327
+ blockexception = nil
328
+
329
+ if (block != nil)
330
+ # TRB 9/17/07 Careful - because we are passing in blocks, we get a situation where
331
+ # an exception may get thrown in the block body (which is high-level
332
+ # code either here or in the application) but gets caught in the
333
+ # low-level code of HttpConnection. The solution is not to let any
334
+ # exception escape the block that we pass to HttpConnection::request.
335
+ # Exceptions can originate from code directly in the block, or from user
336
+ # code called in the other block which is passed to response.read_body.
337
+ benchblock.service.add! do
338
+ responsehdr = @connection.request(request) do |response|
339
+ #########
340
+ begin
341
+ @last_response = response
342
+ if response.is_a?(Net::HTTPSuccess)
343
+ @error_handler = nil
344
+ response.read_body(&block)
345
+ else
346
+ @error_handler = AWSErrorHandler.new(self, parser, :errors_list => self.class.amazon_problems) unless @error_handler
347
+ check_result = @error_handler.check(request)
348
+ if check_result
349
+ @error_handler = nil
350
+ return check_result
351
+ end
352
+ request_text_data = "#{request[:server]}:#{request[:port]}#{request[:request].path}"
353
+ raise AwsError.new(@last_errors, @last_response.code, @last_request_id, request_text_data)
354
+ end
355
+ rescue Exception => e
356
+ blockexception = e
357
+ end
358
+ end
359
+ #########
360
+
361
+ #OK, now we are out of the block passed to the lower level
362
+ if (blockexception)
363
+ raise blockexception
364
+ end
365
+ benchblock.xml.add! do
366
+ parser.parse(responsehdr)
367
+ end
368
+ return parser.result
369
+ end
370
+ else
371
+ benchblock.service.add!{ response = @connection.request(request) }
372
+ # check response for errors...
373
+ @last_response = response
374
+ if response.is_a?(Net::HTTPSuccess)
375
+ @error_handler = nil
376
+ benchblock.xml.add! { parser.parse(response) }
377
+ return parser.result
378
+ else
379
+ @error_handler = AWSErrorHandler.new(self, parser, :errors_list => self.class.amazon_problems) unless @error_handler
380
+ check_result = @error_handler.check(request)
381
+ if check_result
382
+ @error_handler = nil
383
+ return check_result
384
+ end
385
+ request_text_data = "#{request[:server]}:#{request[:port]}#{request[:request].path}"
386
+ raise AwsError.new(@last_errors, @last_response.code, @last_request_id, request_text_data)
333
387
  end
334
- request_text_data = "#{request[:server]}:#{request[:port]}#{request[:request].path}"
335
- raise AwsError.new(@last_errors, @last_response.code, @last_request_id, request_text_data)
336
- end
337
- rescue Exception => e
338
- blockexception = e
339
388
  end
340
- end
341
- #########
342
-
343
- #OK, now we are out of the block passed to the lower level
344
- if(blockexception)
345
- raise blockexception
346
- end
347
- benchblock.xml.add! do
348
- parser.parse(responsehdr)
349
- end
350
- return parser.result
351
- end
352
- else
353
- benchblock.service.add!{ response = @connection.request(request) }
354
- # check response for errors...
355
- @last_response = response
356
- if response.is_a?(Net::HTTPSuccess)
357
- @error_handler = nil
358
- benchblock.xml.add! { parser.parse(response) }
359
- return parser.result
360
- else
361
- @error_handler = AWSErrorHandler.new(self, parser, :errors_list => self.class.amazon_problems) unless @error_handler
362
- check_result = @error_handler.check(request)
363
- if check_result
389
+ rescue
364
390
  @error_handler = nil
365
- return check_result
366
- end
367
- request_text_data = "#{request[:server]}:#{request[:port]}#{request[:request].path}"
368
- raise AwsError.new(@last_errors, @last_response.code, @last_request_id, request_text_data)
369
- end
370
- end
371
- rescue
372
- @error_handler = nil
373
- raise
374
- end
391
+ raise
392
+ end
375
393
 
376
- def request_cache_or_info(method, link, parser_class, benchblock, use_cache=true) #:nodoc:
377
- # We do not want to break the logic of parsing hence will use a dummy parser to process all the standard
378
- # steps (errors checking etc). The dummy parser does nothig - just returns back the params it received.
379
- # If the caching is enabled and hit then throw AwsNoChange.
380
- # P.S. caching works for the whole images list only! (when the list param is blank)
381
- # check cache
382
- response, params = request_info(link, RightDummyParser.new)
383
- cache_hits?(method.to_sym, response.body) if use_cache
384
- parser = parser_class.new(:logger => @logger)
385
- benchblock.xml.add!{ parser.parse(response, params) }
386
- result = block_given? ? yield(parser) : parser.result
387
- # update parsed data
388
- update_cache(method.to_sym, :parsed => result) if use_cache
389
- result
390
- end
394
+ def request_cache_or_info(method, link, parser_class, benchblock, use_cache=true) #:nodoc:
395
+ # We do not want to break the logic of parsing hence will use a dummy parser to process all the standard
396
+ # steps (errors checking etc). The dummy parser does nothig - just returns back the params it received.
397
+ # If the caching is enabled and hit then throw AwsNoChange.
398
+ # P.S. caching works for the whole images list only! (when the list param is blank)
399
+ # check cache
400
+ response, params = request_info(link, RightDummyParser.new)
401
+ cache_hits?(method.to_sym, response.body) if use_cache
402
+ parser = parser_class.new(:logger => @logger)
403
+ benchblock.xml.add!{ parser.parse(response, params) }
404
+ result = block_given? ? yield(parser) : parser.result
405
+ # update parsed data
406
+ update_cache(method.to_sym, :parsed => result) if use_cache
407
+ result
408
+ end
409
+
410
+ # Returns Amazons request ID for the latest request
411
+ def last_request_id
412
+ @last_response && @last_response.body.to_s[%r{<requestId>(.+?)</requestId>}] && $1
413
+ end
391
414
 
392
- # Returns Amazons request ID for the latest request
393
- def last_request_id
394
- @last_response && @last_response.body.to_s[%r{<requestId>(.+?)</requestId>}] && $1
395
415
  end
396
416
 
397
- end
398
417
 
418
+ # Exception class to signal any Amazon errors. All errors occuring during calls to Amazon's
419
+ # web services raise this type of error.
420
+ # Attribute inherited by RuntimeError:
421
+ # message - the text of the error, generally as returned by AWS in its XML response.
422
+ class AwsError < RuntimeError
399
423
 
400
- # Exception class to signal any Amazon errors. All errors occuring during calls to Amazon's
401
- # web services raise this type of error.
402
- # Attribute inherited by RuntimeError:
403
- # message - the text of the error, generally as returned by AWS in its XML response.
404
- class AwsError < RuntimeError
424
+ # either an array of errors where each item is itself an array of [code, message]),
425
+ # or an error string if the error was raised manually, as in <tt>AwsError.new('err_text')</tt>
426
+ attr_reader :errors
405
427
 
406
- # either an array of errors where each item is itself an array of [code, message]),
407
- # or an error string if the error was raised manually, as in <tt>AwsError.new('err_text')</tt>
408
- attr_reader :errors
428
+ # Request id (if exists)
429
+ attr_reader :request_id
409
430
 
410
- # Request id (if exists)
411
- attr_reader :request_id
431
+ # Response HTTP error code
432
+ attr_reader :http_code
412
433
 
413
- # Response HTTP error code
414
- attr_reader :http_code
434
+ # Raw request text data to AWS
435
+ attr_reader :request_data
415
436
 
416
- # Raw request text data to AWS
417
- attr_reader :request_data
437
+ def initialize(errors=nil, http_code=nil, request_id=nil, request_data=nil)
438
+ @errors = errors
439
+ @request_id = request_id
440
+ @http_code = http_code
441
+ @request_data = request_data
442
+ msg = @errors.is_a?(Array) ? @errors.map{|code, msg| "#{code}: #{msg}"}.join("; ") : @errors.to_s
443
+ msg += "\nREQUEST(#{@request_data})" unless @request_data.nil?
444
+ super(msg)
445
+ end
418
446
 
419
- def initialize(errors=nil, http_code=nil, request_id=nil, request_data=nil)
420
- @errors = errors
421
- @request_id = request_id
422
- @http_code = http_code
423
- @request_data = request_data
424
- msg = @errors.is_a?(Array) ? @errors.map{|code, msg| "#{code}: #{msg}"}.join("; ") : @errors.to_s
425
- msg += "\nREQUEST(#{@request_data})" unless @request_data.nil?
426
- super(msg)
427
- end
447
+ # Does any of the error messages include the regexp +pattern+?
448
+ # Used to determine whether to retry request.
449
+ def include?(pattern)
450
+ if @errors.is_a?(Array)
451
+ @errors.each{ |code, msg| return true if code =~ pattern }
452
+ else
453
+ return true if @errors_str =~ pattern
454
+ end
455
+ false
456
+ end
428
457
 
429
- # Does any of the error messages include the regexp +pattern+?
430
- # Used to determine whether to retry request.
431
- def include?(pattern)
432
- if @errors.is_a?(Array)
433
- @errors.each{ |code, msg| return true if code =~ pattern }
434
- else
435
- return true if @errors_str =~ pattern
436
- end
437
- false
438
- end
458
+ # Generic handler for AwsErrors. +aws+ is the RightAws::S3, RightAws::EC2, or RightAws::SQS
459
+ # object that caused the exception (it must provide last_request and last_response). Supported
460
+ # boolean options are:
461
+ # * <tt>:log</tt> print a message into the log using aws.logger to access the Logger
462
+ # * <tt>:puts</tt> do a "puts" of the error
463
+ # * <tt>:raise</tt> re-raise the error after logging
464
+ def self.on_aws_exception(aws, options={:raise=>true, :log=>true})
465
+ # Only log & notify if not user error
466
+ if !options[:raise] || system_error?($!)
467
+ error_text = "#{$!.inspect}\n#{$@}.join('\n')}"
468
+ puts error_text if options[:puts]
469
+ # Log the error
470
+ if options[:log]
471
+ request = aws.last_request ? aws.last_request.path : '-none-'
472
+ response = aws.last_response ? "#{aws.last_response.code} -- #{aws.last_response.message} -- #{aws.last_response.body}" : '-none-'
473
+ aws.logger.error error_text
474
+ aws.logger.error "Request was: #{request}"
475
+ aws.logger.error "Response was: #{response}"
476
+ end
477
+ end
478
+ raise if options[:raise] # re-raise an exception
479
+ return nil
480
+ end
439
481
 
440
- # Generic handler for AwsErrors. +aws+ is the RightAws::S3, RightAws::EC2, or RightAws::SQS
441
- # object that caused the exception (it must provide last_request and last_response). Supported
442
- # boolean options are:
443
- # * <tt>:log</tt> print a message into the log using aws.logger to access the Logger
444
- # * <tt>:puts</tt> do a "puts" of the error
445
- # * <tt>:raise</tt> re-raise the error after logging
446
- def self.on_aws_exception(aws, options={:raise=>true, :log=>true})
447
- # Only log & notify if not user error
448
- if !options[:raise] || system_error?($!)
449
- error_text = "#{$!.inspect}\n#{$@}.join('\n')}"
450
- puts error_text if options[:puts]
451
- # Log the error
452
- if options[:log]
453
- request = aws.last_request ? aws.last_request.path : '-none-'
454
- response = aws.last_response ? "#{aws.last_response.code} -- #{aws.last_response.message} -- #{aws.last_response.body}" : '-none-'
455
- aws.logger.error error_text
456
- aws.logger.error "Request was: #{request}"
457
- aws.logger.error "Response was: #{response}"
458
- end
459
- end
460
- raise if options[:raise] # re-raise an exception
461
- return nil
462
- end
482
+ # True if e is an AWS system error, i.e. something that is for sure not the caller's fault.
483
+ # Used to force logging.
484
+ def self.system_error?(e)
485
+ !e.is_a?(self) || e.message =~ /InternalError|InsufficientInstanceCapacity|Unavailable/
486
+ end
463
487
 
464
- # True if e is an AWS system error, i.e. something that is for sure not the caller's fault.
465
- # Used to force logging.
466
- def self.system_error?(e)
467
- !e.is_a?(self) || e.message =~ /InternalError|InsufficientInstanceCapacity|Unavailable/
468
488
  end
469
489
 
470
- end
471
490
 
491
+ class AWSErrorHandler
492
+ # 0-100 (%)
493
+ DEFAULT_CLOSE_ON_4XX_PROBABILITY = 10
472
494
 
473
- class AWSErrorHandler
474
- # 0-100 (%)
475
- DEFAULT_CLOSE_ON_4XX_PROBABILITY = 10
495
+ @@reiteration_start_delay = 0.2
496
+ def self.reiteration_start_delay
497
+ @@reiteration_start_delay
498
+ end
499
+ def self.reiteration_start_delay=(reiteration_start_delay)
500
+ @@reiteration_start_delay = reiteration_start_delay
501
+ end
476
502
 
477
- @@reiteration_start_delay = 0.2
478
- def self.reiteration_start_delay
479
- @@reiteration_start_delay
480
- end
481
- def self.reiteration_start_delay=(reiteration_start_delay)
482
- @@reiteration_start_delay = reiteration_start_delay
483
- end
503
+ @@reiteration_time = 5
504
+ def self.reiteration_time
505
+ @@reiteration_time
506
+ end
507
+ def self.reiteration_time=(reiteration_time)
508
+ @@reiteration_time = reiteration_time
509
+ end
484
510
 
485
- @@reiteration_time = 5
486
- def self.reiteration_time
487
- @@reiteration_time
488
- end
489
- def self.reiteration_time=(reiteration_time)
490
- @@reiteration_time = reiteration_time
491
- end
511
+ @@close_on_error = true
512
+ def self.close_on_error
513
+ @@close_on_error
514
+ end
515
+ def self.close_on_error=(close_on_error)
516
+ @@close_on_error = close_on_error
517
+ end
492
518
 
493
- @@close_on_error = true
494
- def self.close_on_error
495
- @@close_on_error
496
- end
497
- def self.close_on_error=(close_on_error)
498
- @@close_on_error = close_on_error
499
- end
519
+ @@close_on_4xx_probability = DEFAULT_CLOSE_ON_4XX_PROBABILITY
520
+ def self.close_on_4xx_probability
521
+ @@close_on_4xx_probability
522
+ end
523
+ def self.close_on_4xx_probability=(close_on_4xx_probability)
524
+ @@close_on_4xx_probability = close_on_4xx_probability
525
+ end
500
526
 
501
- @@close_on_4xx_probability = DEFAULT_CLOSE_ON_4XX_PROBABILITY
502
- def self.close_on_4xx_probability
503
- @@close_on_4xx_probability
504
- end
505
- def self.close_on_4xx_probability=(close_on_4xx_probability)
506
- @@close_on_4xx_probability = close_on_4xx_probability
507
- end
527
+ # params:
528
+ # :reiteration_time
529
+ # :errors_list
530
+ # :close_on_error = true | false
531
+ # :close_on_4xx_probability = 1-100
532
+ def initialize(aws, parser, params={}) #:nodoc:
533
+ @aws = aws # Link to RightEc2 | RightSqs | RightS3 instance
534
+ @parser = parser # parser to parse Amazon response
535
+ @started_at = Time.now
536
+ @stop_at = @started_at + (params[:reiteration_time] || @@reiteration_time)
537
+ @errors_list = params[:errors_list] || []
538
+ @reiteration_delay = @@reiteration_start_delay
539
+ @retries = 0
540
+ # close current HTTP(S) connection on 5xx, errors from list and 4xx errors
541
+ @close_on_error = params[:close_on_error].nil? ? @@close_on_error : params[:close_on_error]
542
+ @close_on_4xx_probability = params[:close_on_4xx_probability] || @@close_on_4xx_probability
543
+ end
508
544
 
509
- # params:
510
- # :reiteration_time
511
- # :errors_list
512
- # :close_on_error = true | false
513
- # :close_on_4xx_probability = 1-100
514
- def initialize(aws, parser, params={}) #:nodoc:
515
- @aws = aws # Link to RightEc2 | RightSqs | RightS3 instance
516
- @parser = parser # parser to parse Amazon response
517
- @started_at = Time.now
518
- @stop_at = @started_at + (params[:reiteration_time] || @@reiteration_time)
519
- @errors_list = params[:errors_list] || []
520
- @reiteration_delay = @@reiteration_start_delay
521
- @retries = 0
522
- # close current HTTP(S) connection on 5xx, errors from list and 4xx errors
523
- @close_on_error = params[:close_on_error].nil? ? @@close_on_error : params[:close_on_error]
524
- @close_on_4xx_probability = params[:close_on_4xx_probability] || @@close_on_4xx_probability
525
- end
545
+ # Returns false if
546
+ def check(request) #:nodoc:
547
+ result = false
548
+ error_found = false
549
+ redirect_detected= false
550
+ error_match = nil
551
+ last_errors_text = ''
552
+ response = @aws.last_response
553
+ # log error
554
+ request_text_data = "#{request[:server]}:#{request[:port]}#{request[:request].path}"
555
+ # is this a redirect?
556
+ # yes!
557
+ if response.is_a?(Net::HTTPRedirection)
558
+ redirect_detected = true
559
+ else
560
+ # no, it's an error ...
561
+ @aws.logger.warn("##### #{@aws.class.name} returned an error: #{response.code} #{response.message}\n#{response.body} #####")
562
+ @aws.logger.warn("##### #{@aws.class.name} request: #{request_text_data} ####")
563
+ end
564
+ # Check response body: if it is an Amazon XML document or not:
565
+ if redirect_detected || (response.body && response.body[/<\?xml/]) # ... it is a xml document
566
+ @aws.class.bench_xml.add! do
567
+ error_parser = RightErrorResponseParser.new
568
+ error_parser.parse(response)
569
+ @aws.last_errors = error_parser.errors
570
+ @aws.last_request_id = error_parser.requestID
571
+ last_errors_text = @aws.last_errors.flatten.join("\n")
572
+ # on redirect :
573
+ if redirect_detected
574
+ location = response['location']
575
+ # ... log information and ...
576
+ @aws.logger.info("##### #{@aws.class.name} redirect requested: #{response.code} #{response.message} #####")
577
+ @aws.logger.info("##### New location: #{location} #####")
578
+ # ... fix the connection data
579
+ request[:server] = URI.parse(location).host
580
+ request[:protocol] = URI.parse(location).scheme
581
+ request[:port] = URI.parse(location).port
582
+ end
583
+ end
584
+ else # ... it is not a xml document(probably just a html page?)
585
+ @aws.last_errors = [[response.code, "#{response.message} (#{request_text_data})"]]
586
+ @aws.last_request_id = '-undefined-'
587
+ last_errors_text = response.message
588
+ end
589
+ # now - check the error
590
+ unless redirect_detected
591
+ @errors_list.each do |error_to_find|
592
+ if last_errors_text[/#{error_to_find}/i]
593
+ error_found = true
594
+ error_match = error_to_find
595
+ @aws.logger.warn("##### Retry is needed, error pattern match: #{error_to_find} #####")
596
+ break
597
+ end
598
+ end
599
+ end
600
+ # check the time has gone from the first error come
601
+ if redirect_detected || error_found
602
+ # Close the connection to the server and recreate a new one.
603
+ # It may have a chance that one server is a semi-down and reconnection
604
+ # will help us to connect to the other server
605
+ if !redirect_detected && @close_on_error
606
+ @aws.connection.finish "#{self.class.name}: error match to pattern '#{error_match}'"
607
+ end
526
608
 
527
- # Returns false if
528
- def check(request) #:nodoc:
529
- result = false
530
- error_found = false
531
- redirect_detected= false
532
- error_match = nil
533
- last_errors_text = ''
534
- response = @aws.last_response
535
- # log error
536
- request_text_data = "#{request[:server]}:#{request[:port]}#{request[:request].path}"
537
- # is this a redirect?
538
- # yes!
539
- if response.is_a?(Net::HTTPRedirection)
540
- redirect_detected = true
541
- else
542
- # no, it's an error ...
543
- @aws.logger.warn("##### #{@aws.class.name} returned an error: #{response.code} #{response.message}\n#{response.body} #####")
544
- @aws.logger.warn("##### #{@aws.class.name} request: #{request_text_data} ####")
545
- end
546
- # Check response body: if it is an Amazon XML document or not:
547
- if redirect_detected || (response.body && response.body[/<\?xml/]) # ... it is a xml document
548
- @aws.class.bench_xml.add! do
549
- error_parser = RightErrorResponseParser.new
550
- error_parser.parse(response)
551
- @aws.last_errors = error_parser.errors
552
- @aws.last_request_id = error_parser.requestID
553
- last_errors_text = @aws.last_errors.flatten.join("\n")
554
- # on redirect :
555
- if redirect_detected
556
- location = response['location']
557
- # ... log information and ...
558
- @aws.logger.info("##### #{@aws.class.name} redirect requested: #{response.code} #{response.message} #####")
559
- @aws.logger.info("##### New location: #{location} #####")
560
- # ... fix the connection data
561
- request[:server] = URI.parse(location).host
562
- request[:protocol] = URI.parse(location).scheme
563
- request[:port] = URI.parse(location).port
564
- end
565
- end
566
- else # ... it is not a xml document(probably just a html page?)
567
- @aws.last_errors = [[response.code, "#{response.message} (#{request_text_data})"]]
568
- @aws.last_request_id = '-undefined-'
569
- last_errors_text = response.message
570
- end
571
- # now - check the error
572
- unless redirect_detected
573
- @errors_list.each do |error_to_find|
574
- if last_errors_text[/#{error_to_find}/i]
575
- error_found = true
576
- error_match = error_to_find
577
- @aws.logger.warn("##### Retry is needed, error pattern match: #{error_to_find} #####")
578
- break
579
- end
580
- end
581
- end
582
- # check the time has gone from the first error come
583
- if redirect_detected || error_found
584
- # Close the connection to the server and recreate a new one.
585
- # It may have a chance that one server is a semi-down and reconnection
586
- # will help us to connect to the other server
587
- if !redirect_detected && @close_on_error
588
- @aws.connection.finish "#{self.class.name}: error match to pattern '#{error_match}'"
589
- end
590
-
591
- if (Time.now < @stop_at)
592
- @retries += 1
593
- unless redirect_detected
594
- @aws.logger.warn("##### Retry ##{@retries} is being performed. Sleeping for #{@reiteration_delay} sec. Whole time: #{Time.now-@started_at} sec ####")
595
- sleep @reiteration_delay
596
- @reiteration_delay *= 2
597
-
598
- # Always make sure that the fp is set to point to the beginning(?)
599
- # of the File/IO. TODO: it assumes that offset is 0, which is bad.
600
- if(request[:request].body_stream && request[:request].body_stream.respond_to?(:pos))
601
- begin
602
- request[:request].body_stream.pos = 0
603
- rescue Exception => e
604
- @logger.warn("Retry may fail due to unable to reset the file pointer" +
605
- " -- #{self.class.name} : #{e.inspect}")
606
- end
609
+ if (Time.now < @stop_at)
610
+ @retries += 1
611
+ unless redirect_detected
612
+ @aws.logger.warn("##### Retry ##{@retries} is being performed. Sleeping for #{@reiteration_delay} sec. Whole time: #{Time.now-@started_at} sec ####")
613
+ sleep @reiteration_delay
614
+ @reiteration_delay *= 2
615
+
616
+ # Always make sure that the fp is set to point to the beginning(?)
617
+ # of the File/IO. TODO: it assumes that offset is 0, which is bad.
618
+ if (request[:request].body_stream && request[:request].body_stream.respond_to?(:pos))
619
+ begin
620
+ request[:request].body_stream.pos = 0
621
+ rescue Exception => e
622
+ @logger.warn("Retry may fail due to unable to reset the file pointer" +
623
+ " -- #{self.class.name} : #{e.inspect}")
624
+ end
625
+ end
626
+ else
627
+ @aws.logger.info("##### Retry ##{@retries} is being performed due to a redirect. ####")
628
+ end
629
+ result = @aws.request_info(request, @parser)
630
+ else
631
+ @aws.logger.warn("##### Ooops, time is over... ####")
632
+ end
633
+ # aha, this is unhandled error:
634
+ elsif @close_on_error
635
+ # Is this a 5xx error ?
636
+ if @aws.last_response.code.to_s[/^5\d\d$/]
637
+ @aws.connection.finish "#{self.class.name}: code: #{@aws.last_response.code}: '#{@aws.last_response.message}'"
638
+ # Is this a 4xx error ?
639
+ elsif @aws.last_response.code.to_s[/^4\d\d$/] && @close_on_4xx_probability > rand(100)
640
+ @aws.connection.finish "#{self.class.name}: code: #{@aws.last_response.code}: '#{@aws.last_response.message}', " +
641
+ "probability: #{@close_on_4xx_probability}%"
642
+ end
607
643
  end
608
- else
609
- @aws.logger.info("##### Retry ##{@retries} is being performed due to a redirect. ####")
610
- end
611
- result = @aws.request_info(request, @parser)
612
- else
613
- @aws.logger.warn("##### Ooops, time is over... ####")
614
- end
615
- # aha, this is unhandled error:
616
- elsif @close_on_error
617
- # Is this a 5xx error ?
618
- if @aws.last_response.code.to_s[/^5\d\d$/]
619
- @aws.connection.finish "#{self.class.name}: code: #{@aws.last_response.code}: '#{@aws.last_response.message}'"
620
- # Is this a 4xx error ?
621
- elsif @aws.last_response.code.to_s[/^4\d\d$/] && @close_on_4xx_probability > rand(100)
622
- @aws.connection.finish "#{self.class.name}: code: #{@aws.last_response.code}: '#{@aws.last_response.message}', " +
623
- "probability: #{@close_on_4xx_probability}%"
624
- end
625
- end
626
- result
644
+ result
645
+ end
646
+
627
647
  end
628
648
 
629
- end
630
649
 
650
+ #-----------------------------------------------------------------
631
651
 
632
- #-----------------------------------------------------------------
652
+ class RightSaxParserCallback #:nodoc:
653
+ def self.include_callback
654
+ include XML::SaxParser::Callbacks
655
+ end
633
656
 
634
- class RightSaxParserCallback #:nodoc:
635
- def self.include_callback
636
- include XML::SaxParser::Callbacks
637
- end
638
- def initialize(right_aws_parser)
639
- @right_aws_parser = right_aws_parser
640
- end
641
- def on_start_element(name, attr_hash)
642
- @right_aws_parser.tag_start(name, attr_hash)
643
- end
644
- def on_characters(chars)
645
- @right_aws_parser.text(chars)
646
- end
647
- def on_end_element(name)
648
- @right_aws_parser.tag_end(name)
649
- end
650
- def on_start_document; end
651
- def on_comment(msg); end
652
- def on_processing_instruction(target, data); end
653
- def on_cdata_block(cdata); end
654
- def on_end_document; end
655
- end
656
-
657
- class RightAWSParser #:nodoc:
658
- # default parsing library
659
- DEFAULT_XML_LIBRARY = 'rexml'
660
- # a list of supported parsers
661
- @@supported_xml_libs = [DEFAULT_XML_LIBRARY, 'libxml']
662
-
663
- @@xml_lib = DEFAULT_XML_LIBRARY # xml library name: 'rexml' | 'libxml'
664
- def self.xml_lib
665
- @@xml_lib
666
- end
667
- def self.xml_lib=(new_lib_name)
668
- @@xml_lib = new_lib_name
669
- end
657
+ def initialize(right_aws_parser)
658
+ @right_aws_parser = right_aws_parser
659
+ end
670
660
 
671
- attr_accessor :result
672
- attr_reader :xmlpath
673
- attr_accessor :xml_lib
674
-
675
- def initialize(params={})
676
- @xmlpath = ''
677
- @result = false
678
- @text = ''
679
- @xml_lib = params[:xml_lib] || @@xml_lib
680
- @logger = params[:logger]
681
- reset
682
- end
683
- def tag_start(name, attributes)
684
- @text = ''
685
- tagstart(name, attributes)
686
- @xmlpath += @xmlpath.empty? ? name : "/#{name}"
687
- end
688
- def tag_end(name)
689
- if @xmlpath =~ /^(.*?)\/?#{name}$/
690
- @xmlpath = $1
691
- end
692
- tagend(name)
693
- end
694
- def text(text)
695
- @text += text
696
- tagtext(text)
697
- end
698
- # Parser method.
699
- # Params:
700
- # xml_text - xml message text(String) or Net:HTTPxxx instance (response)
701
- # params[:xml_lib] - library name: 'rexml' | 'libxml'
702
- def parse(xml_text, params={})
703
- # Get response body
704
- unless xml_text.is_a?(String)
705
- xml_text = xml_text.body.respond_to?(:force_encoding) ? xml_text.body.force_encoding("UTF-8") : xml_text.body
706
- end
707
-
708
- @xml_lib = params[:xml_lib] || @xml_lib
709
- # check that we had no problems with this library otherwise use default
710
- @xml_lib = DEFAULT_XML_LIBRARY unless @@supported_xml_libs.include?(@xml_lib)
711
- # load xml library
712
- if @xml_lib=='libxml' && !defined?(XML::SaxParser)
713
- begin
714
- require 'xml/libxml'
715
- # is it new ? - Setup SaxParserCallback
716
- if XML::Parser::VERSION >= '0.5.1.0'
717
- RightSaxParserCallback.include_callback
718
- end
719
- rescue LoadError => e
720
- @@supported_xml_libs.delete(@xml_lib)
721
- @xml_lib = DEFAULT_XML_LIBRARY
722
- if @logger
723
- @logger.error e.inspect
724
- @logger.error e.backtrace
725
- @logger.info "Can not load 'libxml' library. '#{DEFAULT_XML_LIBRARY}' is used for parsing."
726
- end
727
- end
728
- end
729
- # Parse the xml text
730
- case @xml_lib
731
- when 'libxml'
732
- xml = XML::SaxParser.new
733
- xml.string = xml_text
734
- # check libxml-ruby version
735
- if XML::Parser::VERSION >= '0.5.1.0'
736
- xml.callbacks = RightSaxParserCallback.new(self)
737
- else
738
- xml.on_start_element{|name, attr_hash| self.tag_start(name, attr_hash)}
739
- xml.on_characters{ |text| self.text(text)}
740
- xml.on_end_element{ |name| self.tag_end(name)}
741
- end
742
- xml.parse
743
- else
744
- REXML::Document.parse_stream(xml_text, self)
745
- end
661
+ def on_start_element(name, attr_hash)
662
+ @right_aws_parser.tag_start(name, attr_hash)
663
+ end
664
+
665
+ def on_characters(chars)
666
+ @right_aws_parser.text(chars)
667
+ end
668
+
669
+ def on_end_element(name)
670
+ @right_aws_parser.tag_end(name)
671
+ end
672
+
673
+ def on_start_document;
674
+ end
675
+
676
+ def on_comment(msg)
677
+ ;
678
+ end
679
+
680
+ def on_processing_instruction(target, data)
681
+ ;
682
+ end
683
+
684
+ def on_cdata_block(cdata)
685
+ ;
686
+ end
687
+
688
+ def on_end_document;
689
+ end
746
690
  end
747
- # Parser must have a lots of methods
748
- # (see /usr/lib/ruby/1.8/rexml/parsers/streamparser.rb)
749
- # We dont need most of them in RightAWSParser and method_missing helps us
750
- # to skip their definition
751
- def method_missing(method, *params)
752
- # if the method is one of known - just skip it ...
753
- return if [:comment, :attlistdecl, :notationdecl, :elementdecl,
754
- :entitydecl, :cdata, :xmldecl, :attlistdecl, :instruction,
755
- :doctype].include?(method)
756
- # ... else - call super to raise an exception
757
- super(method, params)
691
+
692
+ class RightAWSParser #:nodoc:
693
+ # default parsing library
694
+ DEFAULT_XML_LIBRARY = 'rexml'
695
+ # a list of supported parsers
696
+ @@supported_xml_libs = [DEFAULT_XML_LIBRARY, 'libxml']
697
+
698
+ @@xml_lib = DEFAULT_XML_LIBRARY # xml library name: 'rexml' | 'libxml'
699
+ def self.xml_lib
700
+ @@xml_lib
701
+ end
702
+ def self.xml_lib=(new_lib_name)
703
+ @@xml_lib = new_lib_name
704
+ end
705
+
706
+ attr_accessor :result
707
+ attr_reader :xmlpath
708
+ attr_accessor :xml_lib
709
+
710
+ def initialize(params={})
711
+ @xmlpath = ''
712
+ @result = false
713
+ @text = ''
714
+ @xml_lib = params[:xml_lib] || @@xml_lib
715
+ @logger = params[:logger]
716
+ reset
717
+ end
718
+
719
+ def tag_start(name, attributes)
720
+ @text = ''
721
+ tagstart(name, attributes)
722
+ @xmlpath += @xmlpath.empty? ? name : "/#{name}"
723
+ end
724
+
725
+ def tag_end(name)
726
+ if @xmlpath =~ /^(.*?)\/?#{name}$/
727
+ @xmlpath = $1
728
+ end
729
+ tagend(name)
730
+ end
731
+
732
+ def text(text)
733
+ @text += text
734
+ tagtext(text)
735
+ end
736
+
737
+ # Parser method.
738
+ # Params:
739
+ # xml_text - xml message text(String) or Net:HTTPxxx instance (response)
740
+ # params[:xml_lib] - library name: 'rexml' | 'libxml'
741
+ def parse(xml_text, params={})
742
+ # Get response body
743
+ unless xml_text.is_a?(String)
744
+ xml_text = xml_text.body.respond_to?(:force_encoding) ? xml_text.body.force_encoding("UTF-8") : xml_text.body
745
+ end
746
+
747
+ @xml_lib = params[:xml_lib] || @xml_lib
748
+ # check that we had no problems with this library otherwise use default
749
+ @xml_lib = DEFAULT_XML_LIBRARY unless @@supported_xml_libs.include?(@xml_lib)
750
+ # load xml library
751
+ if @xml_lib=='libxml' && !defined?(XML::SaxParser)
752
+ begin
753
+ require 'xml/libxml'
754
+ # is it new ? - Setup SaxParserCallback
755
+ if XML::Parser::VERSION >= '0.5.1.0'
756
+ RightSaxParserCallback.include_callback
757
+ end
758
+ rescue LoadError => e
759
+ @@supported_xml_libs.delete(@xml_lib)
760
+ @xml_lib = DEFAULT_XML_LIBRARY
761
+ if @logger
762
+ @logger.error e.inspect
763
+ @logger.error e.backtrace
764
+ @logger.info "Can not load 'libxml' library. '#{DEFAULT_XML_LIBRARY}' is used for parsing."
765
+ end
766
+ end
767
+ end
768
+ # Parse the xml text
769
+ case @xml_lib
770
+ when 'libxml'
771
+ xml = XML::SaxParser.new
772
+ xml.string = xml_text
773
+ # check libxml-ruby version
774
+ if XML::Parser::VERSION >= '0.5.1.0'
775
+ xml.callbacks = RightSaxParserCallback.new(self)
776
+ else
777
+ xml.on_start_element{|name, attr_hash| self.tag_start(name, attr_hash)}
778
+ xml.on_characters{ |text| self.text(text)}
779
+ xml.on_end_element{ |name| self.tag_end(name)}
780
+ end
781
+ xml.parse
782
+ else
783
+ REXML::Document.parse_stream(xml_text, self)
784
+ end
785
+ end
786
+
787
+ # Parser must have a lots of methods
788
+ # (see /usr/lib/ruby/1.8/rexml/parsers/streamparser.rb)
789
+ # We dont need most of them in RightAWSParser and method_missing helps us
790
+ # to skip their definition
791
+ def method_missing(method, *params)
792
+ # if the method is one of known - just skip it ...
793
+ return if [:comment, :attlistdecl, :notationdecl, :elementdecl,
794
+ :entitydecl, :cdata, :xmldecl, :attlistdecl, :instruction,
795
+ :doctype].include?(method)
796
+ # ... else - call super to raise an exception
797
+ super(method, params)
798
+ end
799
+
800
+ # the functions to be overriden by children (if nessesery)
801
+ def reset;
802
+ end
803
+
804
+ def tagstart(name, attributes)
805
+ ;
806
+ end
807
+
808
+ def tagend(name)
809
+ ;
810
+ end
811
+
812
+ def tagtext(text)
813
+ ;
814
+ end
758
815
  end
759
- # the functions to be overriden by children (if nessesery)
760
- def reset ; end
761
- def tagstart(name, attributes); end
762
- def tagend(name) ; end
763
- def tagtext(text) ; end
764
- end
765
816
 
766
- #-----------------------------------------------------------------
767
- # PARSERS: Errors
768
- #-----------------------------------------------------------------
817
+ #-----------------------------------------------------------------
818
+ # PARSERS: Errors
819
+ #-----------------------------------------------------------------
769
820
 
770
821
  #<Error>
771
822
  # <Code>TemporaryRedirect</Code>
@@ -776,40 +827,46 @@ module RightAws
776
827
  # <Bucket>bucket-for-k</Bucket>
777
828
  #</Error>
778
829
 
779
- class RightErrorResponseParser < RightAWSParser #:nodoc:
780
- attr_accessor :errors # array of hashes: error/message
781
- attr_accessor :requestID
830
+ class RightErrorResponseParser < RightAWSParser #:nodoc:
831
+ attr_accessor :errors # array of hashes: error/message
832
+ attr_accessor :requestID
782
833
  # attr_accessor :endpoint, :host_id, :bucket
783
- def tagend(name)
784
- case name
785
- when 'RequestID' ; @requestID = @text
786
- when 'Code' ; @code = @text
787
- when 'Message' ; @message = @text
834
+ def tagend(name)
835
+ case name
836
+ when 'RequestID';
837
+ @requestID = @text
838
+ when 'Code';
839
+ @code = @text
840
+ when 'Message';
841
+ @message = @text
788
842
  # when 'Endpoint' ; @endpoint = @text
789
843
  # when 'HostId' ; @host_id = @text
790
844
  # when 'Bucket' ; @bucket = @text
791
- when 'Error' ; @errors << [ @code, @message ]
792
- end
793
- end
794
- def reset
795
- @errors = []
845
+ when 'Error';
846
+ @errors << [ @code, @message ]
847
+ end
848
+ end
849
+
850
+ def reset
851
+ @errors = []
852
+ end
796
853
  end
797
- end
798
-
799
- # Dummy parser - does nothing
800
- # Returns the original params back
801
- class RightDummyParser # :nodoc:
802
- attr_accessor :result
803
- def parse(response, params={})
804
- @result = [response, params]
854
+
855
+ # Dummy parser - does nothing
856
+ # Returns the original params back
857
+ class RightDummyParser # :nodoc:
858
+ attr_accessor :result
859
+
860
+ def parse(response, params={})
861
+ @result = [response, params]
862
+ end
805
863
  end
806
- end
807
864
 
808
- class RightHttp2xxParser < RightAWSParser # :nodoc:
809
- def parse(response)
810
- @result = response.is_a?(Net::HTTPSuccess)
865
+ class RightHttp2xxParser < RightAWSParser # :nodoc:
866
+ def parse(response)
867
+ @result = response.is_a?(Net::HTTPSuccess)
868
+ end
811
869
  end
812
- end
813
870
 
814
871
  end
815
872