aws 1.10.1
Sign up to get free protection for your applications and to get access to all the features.
- data/History.txt +246 -0
- data/Manifest.txt +19 -0
- data/README.txt +168 -0
- data/Rakefile +17 -0
- data/lib/acf/right_acf_interface.rb +379 -0
- data/lib/awsbase/benchmark_fix.rb +39 -0
- data/lib/awsbase/right_awsbase.rb +797 -0
- data/lib/awsbase/support.rb +111 -0
- data/lib/ec2/right_ec2.rb +1737 -0
- data/lib/right_aws.rb +69 -0
- data/lib/s3/right_s3.rb +1094 -0
- data/lib/s3/right_s3_interface.rb +1175 -0
- data/lib/sdb/active_sdb.rb +936 -0
- data/lib/sdb/right_sdb_interface.rb +701 -0
- data/lib/sqs/right_sqs.rb +388 -0
- data/lib/sqs/right_sqs_gen2.rb +286 -0
- data/lib/sqs/right_sqs_gen2_interface.rb +444 -0
- data/lib/sqs/right_sqs_interface.rb +596 -0
- metadata +96 -0
@@ -0,0 +1,379 @@
|
|
1
|
+
#
|
2
|
+
# Copyright (c) 2008 RightScale Inc
|
3
|
+
#
|
4
|
+
# Permission is hereby granted, free of charge, to any person obtaining
|
5
|
+
# a copy of this software and associated documentation files (the
|
6
|
+
# "Software"), to deal in the Software without restriction, including
|
7
|
+
# without limitation the rights to use, copy, modify, merge, publish,
|
8
|
+
# distribute, sublicense, and/or sell copies of the Software, and to
|
9
|
+
# permit persons to whom the Software is furnished to do so, subject to
|
10
|
+
# the following conditions:
|
11
|
+
#
|
12
|
+
# The above copyright notice and this permission notice shall be
|
13
|
+
# included in all copies or substantial portions of the Software.
|
14
|
+
#
|
15
|
+
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
16
|
+
# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
17
|
+
# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
18
|
+
# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
|
19
|
+
# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
|
20
|
+
# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
|
21
|
+
# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
22
|
+
#
|
23
|
+
|
24
|
+
module RightAws
|
25
|
+
|
26
|
+
# = RightAws::AcfInterface -- RightScale Amazon's CloudFront interface
|
27
|
+
# The AcfInterface class provides a complete interface to Amazon's
|
28
|
+
# CloudFront service.
|
29
|
+
#
|
30
|
+
# For explanations of the semantics of each call, please refer to
|
31
|
+
# Amazon's documentation at
|
32
|
+
# http://developer.amazonwebservices.com/connect/kbcategory.jspa?categoryID=211
|
33
|
+
#
|
34
|
+
# Example:
|
35
|
+
#
|
36
|
+
# acf = RightAws::AcfInterface.new('1E3GDYEOGFJPIT7XXXXXX','hgTHt68JY07JKUY08ftHYtERkjgtfERn57XXXXXX')
|
37
|
+
#
|
38
|
+
# list = acf.list_distributions #=>
|
39
|
+
# [{:status => "Deployed",
|
40
|
+
# :domain_name => "d74zzrxmpmygb.6hops.net",
|
41
|
+
# :aws_id => "E4U91HCJHGXVC",
|
42
|
+
# :origin => "my-bucket.s3.amazonaws.com",
|
43
|
+
# :cnames => ["x1.my-awesome-site.net", "x1.my-awesome-site.net"]
|
44
|
+
# :comment => "My comments",
|
45
|
+
# :last_modified_time => Wed Sep 10 17:00:04 UTC 2008 }, ..., {...} ]
|
46
|
+
#
|
47
|
+
# distibution = list.first
|
48
|
+
#
|
49
|
+
# info = acf.get_distribution(distibution[:aws_id]) #=>
|
50
|
+
# {:enabled => true,
|
51
|
+
# :caller_reference => "200809102100536497863003",
|
52
|
+
# :e_tag => "E39OHHU1ON65SI",
|
53
|
+
# :status => "Deployed",
|
54
|
+
# :domain_name => "d3dxv71tbbt6cd.6hops.net",
|
55
|
+
# :cnames => ["web1.my-awesome-site.net", "web2.my-awesome-site.net"]
|
56
|
+
# :aws_id => "E2REJM3VUN5RSI",
|
57
|
+
# :comment => "Woo-Hoo!",
|
58
|
+
# :origin => "my-bucket.s3.amazonaws.com",
|
59
|
+
# :last_modified_time => Wed Sep 10 17:00:54 UTC 2008 }
|
60
|
+
#
|
61
|
+
# config = acf.get_distribution_config(distibution[:aws_id]) #=>
|
62
|
+
# {:enabled => true,
|
63
|
+
# :caller_reference => "200809102100536497863003",
|
64
|
+
# :e_tag => "E39OHHU1ON65SI",
|
65
|
+
# :cnames => ["web1.my-awesome-site.net", "web2.my-awesome-site.net"]
|
66
|
+
# :comment => "Woo-Hoo!",
|
67
|
+
# :origin => "my-bucket.s3.amazonaws.com"}
|
68
|
+
#
|
69
|
+
# config[:comment] = 'Olah-lah!'
|
70
|
+
# config[:enabled] = false
|
71
|
+
# config[:cnames] << "web3.my-awesome-site.net"
|
72
|
+
#
|
73
|
+
# acf.set_distribution_config(distibution[:aws_id], config) #=> true
|
74
|
+
#
|
75
|
+
class AcfInterface < RightAwsBase
|
76
|
+
|
77
|
+
include RightAwsBaseInterface
|
78
|
+
|
79
|
+
API_VERSION = "2008-06-30"
|
80
|
+
DEFAULT_HOST = 'cloudfront.amazonaws.com'
|
81
|
+
DEFAULT_PORT = 443
|
82
|
+
DEFAULT_PROTOCOL = 'https'
|
83
|
+
DEFAULT_PATH = '/'
|
84
|
+
|
85
|
+
@@bench = AwsBenchmarkingBlock.new
|
86
|
+
def self.bench_xml
|
87
|
+
@@bench.xml
|
88
|
+
end
|
89
|
+
def self.bench_service
|
90
|
+
@@bench.service
|
91
|
+
end
|
92
|
+
|
93
|
+
# Create a new handle to a CloudFront account. All handles share the same per process or per thread
|
94
|
+
# HTTP connection to CloudFront. Each handle is for a specific account. The params have the
|
95
|
+
# following options:
|
96
|
+
# * <tt>:server</tt>: CloudFront service host, default: DEFAULT_HOST
|
97
|
+
# * <tt>:port</tt>: CloudFront service port, default: DEFAULT_PORT
|
98
|
+
# * <tt>:protocol</tt>: 'http' or 'https', default: DEFAULT_PROTOCOL
|
99
|
+
# * <tt>:multi_thread</tt>: true=HTTP connection per thread, false=per process
|
100
|
+
# * <tt>:logger</tt>: for log messages, default: RAILS_DEFAULT_LOGGER else STDOUT
|
101
|
+
# * <tt>:cache</tt>: true/false: caching for list_distributions method, default: false.
|
102
|
+
#
|
103
|
+
# acf = RightAws::AcfInterface.new('1E3GDYEOGFJPIT7XXXXXX','hgTHt68JY07JKUY08ftHYtERkjgtfERn57XXXXXX',
|
104
|
+
# {:multi_thread => true, :logger => Logger.new('/tmp/x.log')}) #=> #<RightAws::AcfInterface::0xb7b3c30c>
|
105
|
+
#
|
106
|
+
def initialize(aws_access_key_id=nil, aws_secret_access_key=nil, params={})
|
107
|
+
init({ :name => 'ACF',
|
108
|
+
:default_host => ENV['ACF_URL'] ? URI.parse(ENV['ACF_URL']).host : DEFAULT_HOST,
|
109
|
+
:default_port => ENV['ACF_URL'] ? URI.parse(ENV['ACF_URL']).port : DEFAULT_PORT,
|
110
|
+
:default_service => ENV['ACF_URL'] ? URI.parse(ENV['ACF_URL']).path : DEFAULT_PATH,
|
111
|
+
:default_protocol => ENV['ACF_URL'] ? URI.parse(ENV['ACF_URL']).scheme : DEFAULT_PROTOCOL },
|
112
|
+
aws_access_key_id || ENV['AWS_ACCESS_KEY_ID'],
|
113
|
+
aws_secret_access_key || ENV['AWS_SECRET_ACCESS_KEY'],
|
114
|
+
params)
|
115
|
+
end
|
116
|
+
|
117
|
+
#-----------------------------------------------------------------
|
118
|
+
# Requests
|
119
|
+
#-----------------------------------------------------------------
|
120
|
+
|
121
|
+
# Generates request hash for REST API.
|
122
|
+
def generate_request(method, path, body=nil, headers={}) # :nodoc:
|
123
|
+
headers['content-type'] ||= 'text/xml' if body
|
124
|
+
headers['date'] = Time.now.httpdate
|
125
|
+
# Auth
|
126
|
+
signature = AwsUtils::sign(@aws_secret_access_key, headers['date'])
|
127
|
+
headers['Authorization'] = "AWS #{@aws_access_key_id}:#{signature}"
|
128
|
+
# Request
|
129
|
+
path = "#{@params[:default_service]}/#{API_VERSION}/#{path}"
|
130
|
+
request = "Net::HTTP::#{method.capitalize}".constantize.new(path)
|
131
|
+
request.body = body if body
|
132
|
+
# Set request headers
|
133
|
+
headers.each { |key, value| request[key.to_s] = value }
|
134
|
+
# prepare output hash
|
135
|
+
{ :request => request,
|
136
|
+
:server => @params[:server],
|
137
|
+
:port => @params[:port],
|
138
|
+
:protocol => @params[:protocol] }
|
139
|
+
end
|
140
|
+
|
141
|
+
# Sends request to Amazon and parses the response.
|
142
|
+
# Raises AwsError if any banana happened.
|
143
|
+
def request_info(request, parser, &block) # :nodoc:
|
144
|
+
thread = @params[:multi_thread] ? Thread.current : Thread.main
|
145
|
+
thread[:acf_connection] ||= Rightscale::HttpConnection.new(:exception => RightAws::AwsError, :logger => @logger)
|
146
|
+
request_info_impl(thread[:acf_connection], @@bench, request, parser, &block)
|
147
|
+
end
|
148
|
+
|
149
|
+
#-----------------------------------------------------------------
|
150
|
+
# Helpers:
|
151
|
+
#-----------------------------------------------------------------
|
152
|
+
|
153
|
+
def self.escape(text) # :nodoc:
|
154
|
+
REXML::Text::normalize(text)
|
155
|
+
end
|
156
|
+
|
157
|
+
def self.unescape(text) # :nodoc:
|
158
|
+
REXML::Text::unnormalize(text)
|
159
|
+
end
|
160
|
+
|
161
|
+
def xmlns # :nodoc:
|
162
|
+
%Q{"http://#{@params[:server]}/doc/#{API_VERSION}/"}
|
163
|
+
end
|
164
|
+
|
165
|
+
def generate_call_reference # :nodoc:
|
166
|
+
result = Time.now.strftime('%Y%m%d%H%M%S')
|
167
|
+
10.times{ result << rand(10).to_s }
|
168
|
+
result
|
169
|
+
end
|
170
|
+
|
171
|
+
def merge_headers(hash) # :nodoc:
|
172
|
+
hash[:location] = @last_response['Location'] if @last_response['Location']
|
173
|
+
hash[:e_tag] = @last_response['ETag'] if @last_response['ETag']
|
174
|
+
hash
|
175
|
+
end
|
176
|
+
|
177
|
+
#-----------------------------------------------------------------
|
178
|
+
# API Calls:
|
179
|
+
#-----------------------------------------------------------------
|
180
|
+
|
181
|
+
# List distributions.
|
182
|
+
# Returns an array of distributions or RightAws::AwsError exception.
|
183
|
+
#
|
184
|
+
# acf.list_distributions #=>
|
185
|
+
# [{:status => "Deployed",
|
186
|
+
# :domain_name => "d74zzrxmpmygb.6hops.net",
|
187
|
+
# :aws_id => "E4U91HCJHGXVC",
|
188
|
+
# :cnames => ["web1.my-awesome-site.net", "web2.my-awesome-site.net"]
|
189
|
+
# :origin => "my-bucket.s3.amazonaws.com",
|
190
|
+
# :comment => "My comments",
|
191
|
+
# :last_modified_time => Wed Sep 10 17:00:04 UTC 2008 }, ..., {...} ]
|
192
|
+
#
|
193
|
+
def list_distributions
|
194
|
+
request_hash = generate_request('GET', 'distribution')
|
195
|
+
request_cache_or_info :list_distributions, request_hash, AcfDistributionListParser, @@bench
|
196
|
+
end
|
197
|
+
|
198
|
+
# Create a new distribution.
|
199
|
+
# Returns the just created distribution or RightAws::AwsError exception.
|
200
|
+
#
|
201
|
+
# acf.create_distribution('bucket-for-k-dzreyev.s3.amazonaws.com', 'Woo-Hoo!', true, ['web1.my-awesome-site.net'] ) #=>
|
202
|
+
# {:comment => "Woo-Hoo!",
|
203
|
+
# :enabled => true,
|
204
|
+
# :location => "https://cloudfront.amazonaws.com/2008-06-30/distribution/E2REJM3VUN5RSI",
|
205
|
+
# :status => "InProgress",
|
206
|
+
# :aws_id => "E2REJM3VUN5RSI",
|
207
|
+
# :domain_name => "d3dxv71tbbt6cd.6hops.net",
|
208
|
+
# :origin => "my-bucket.s3.amazonaws.com",
|
209
|
+
# :cnames => ["web1.my-awesome-site.net"]
|
210
|
+
# :last_modified_time => Wed Sep 10 17:00:54 UTC 2008,
|
211
|
+
# :caller_reference => "200809102100536497863003"}
|
212
|
+
#
|
213
|
+
def create_distribution(origin, comment='', enabled=true, cnames=[], caller_reference=nil)
|
214
|
+
# join CNAMES
|
215
|
+
cnames_str = ''
|
216
|
+
unless cnames.blank?
|
217
|
+
cnames.to_a.each { |cname| cnames_str += "\n <CNAME>#{cname}</CNAME>" }
|
218
|
+
end
|
219
|
+
# reference
|
220
|
+
caller_reference ||= generate_call_reference
|
221
|
+
body = <<-EOXML
|
222
|
+
<?xml version="1.0" encoding="UTF-8"?>
|
223
|
+
<DistributionConfig xmlns=#{xmlns}>
|
224
|
+
<Origin>#{origin}</Origin>
|
225
|
+
<CallerReference>#{caller_reference}</CallerReference>
|
226
|
+
#{cnames_str.lstrip}
|
227
|
+
<Comment>#{AcfInterface::escape(comment.to_s)}</Comment>
|
228
|
+
<Enabled>#{enabled}</Enabled>
|
229
|
+
</DistributionConfig>
|
230
|
+
EOXML
|
231
|
+
request_hash = generate_request('POST', 'distribution', body.strip)
|
232
|
+
merge_headers(request_info(request_hash, AcfDistributionParser.new))
|
233
|
+
end
|
234
|
+
|
235
|
+
# Get a distribution's information.
|
236
|
+
# Returns a distribution's information or RightAws::AwsError exception.
|
237
|
+
#
|
238
|
+
# acf.get_distribution('E2REJM3VUN5RSI') #=>
|
239
|
+
# {:enabled => true,
|
240
|
+
# :caller_reference => "200809102100536497863003",
|
241
|
+
# :e_tag => "E39OHHU1ON65SI",
|
242
|
+
# :status => "Deployed",
|
243
|
+
# :domain_name => "d3dxv71tbbt6cd.6hops.net",
|
244
|
+
# :cnames => ["web1.my-awesome-site.net", "web2.my-awesome-site.net"]
|
245
|
+
# :aws_id => "E2REJM3VUN5RSI",
|
246
|
+
# :comment => "Woo-Hoo!",
|
247
|
+
# :origin => "my-bucket.s3.amazonaws.com",
|
248
|
+
# :last_modified_time => Wed Sep 10 17:00:54 UTC 2008 }
|
249
|
+
#
|
250
|
+
def get_distribution(aws_id)
|
251
|
+
request_hash = generate_request('GET', "distribution/#{aws_id}")
|
252
|
+
merge_headers(request_info(request_hash, AcfDistributionParser.new))
|
253
|
+
end
|
254
|
+
|
255
|
+
# Get a distribution's configuration.
|
256
|
+
# Returns a distribution's configuration or RightAws::AwsError exception.
|
257
|
+
#
|
258
|
+
# acf.get_distribution_config('E2REJM3VUN5RSI') #=>
|
259
|
+
# {:enabled => true,
|
260
|
+
# :caller_reference => "200809102100536497863003",
|
261
|
+
# :e_tag => "E39OHHU1ON65SI",
|
262
|
+
# :cnames => ["web1.my-awesome-site.net", "web2.my-awesome-site.net"]
|
263
|
+
# :comment => "Woo-Hoo!",
|
264
|
+
# :origin => "my-bucket.s3.amazonaws.com"}
|
265
|
+
#
|
266
|
+
def get_distribution_config(aws_id)
|
267
|
+
request_hash = generate_request('GET', "distribution/#{aws_id}/config")
|
268
|
+
merge_headers(request_info(request_hash, AcfDistributionConfigParser.new))
|
269
|
+
end
|
270
|
+
|
271
|
+
# Set a distribution's configuration
|
272
|
+
# (the :origin and the :caller_reference cannot be changed).
|
273
|
+
# Returns +true+ on success or RightAws::AwsError exception.
|
274
|
+
#
|
275
|
+
# config = acf.get_distribution_config('E2REJM3VUN5RSI') #=>
|
276
|
+
# {:enabled => true,
|
277
|
+
# :caller_reference => "200809102100536497863003",
|
278
|
+
# :e_tag => "E39OHHU1ON65SI",
|
279
|
+
# :cnames => ["web1.my-awesome-site.net", "web2.my-awesome-site.net"]
|
280
|
+
# :comment => "Woo-Hoo!",
|
281
|
+
# :origin => "my-bucket.s3.amazonaws.com"}
|
282
|
+
# config[:comment] = 'Olah-lah!'
|
283
|
+
# config[:enabled] = false
|
284
|
+
# acf.set_distribution_config('E2REJM3VUN5RSI', config) #=> true
|
285
|
+
#
|
286
|
+
def set_distribution_config(aws_id, config)
|
287
|
+
# join CNAMES
|
288
|
+
cnames_str = ''
|
289
|
+
unless config[:cnames].blank?
|
290
|
+
config[:cnames].to_a.each { |cname| cnames_str += "\n <CNAME>#{cname}</CNAME>" }
|
291
|
+
end
|
292
|
+
# format request's XML body
|
293
|
+
body = <<-EOXML
|
294
|
+
<?xml version="1.0" encoding="UTF-8"?>
|
295
|
+
<DistributionConfig xmlns=#{xmlns}>
|
296
|
+
<Origin>#{config[:origin]}</Origin>
|
297
|
+
<CallerReference>#{config[:caller_reference]}</CallerReference>
|
298
|
+
#{cnames_str.lstrip}
|
299
|
+
<Comment>#{AcfInterface::escape(config[:comment].to_s)}</Comment>
|
300
|
+
<Enabled>#{config[:enabled]}</Enabled>
|
301
|
+
</DistributionConfig>
|
302
|
+
EOXML
|
303
|
+
request_hash = generate_request('PUT', "distribution/#{aws_id}/config", body.strip,
|
304
|
+
'If-Match' => config[:e_tag])
|
305
|
+
request_info(request_hash, RightHttp2xxParser.new)
|
306
|
+
end
|
307
|
+
|
308
|
+
# Delete a distribution. The enabled distribution cannot be deleted.
|
309
|
+
# Returns +true+ on success or RightAws::AwsError exception.
|
310
|
+
#
|
311
|
+
# acf.delete_distribution('E2REJM3VUN5RSI', 'E39OHHU1ON65SI') #=> true
|
312
|
+
#
|
313
|
+
def delete_distribution(aws_id, e_tag)
|
314
|
+
request_hash = generate_request('DELETE', "distribution/#{aws_id}", nil,
|
315
|
+
'If-Match' => e_tag)
|
316
|
+
request_info(request_hash, RightHttp2xxParser.new)
|
317
|
+
end
|
318
|
+
|
319
|
+
#-----------------------------------------------------------------
|
320
|
+
# PARSERS:
|
321
|
+
#-----------------------------------------------------------------
|
322
|
+
|
323
|
+
class AcfDistributionListParser < RightAWSParser # :nodoc:
|
324
|
+
def reset
|
325
|
+
@result = []
|
326
|
+
end
|
327
|
+
def tagstart(name, attributes)
|
328
|
+
@distribution = { :cnames => [] } if name == 'DistributionSummary'
|
329
|
+
end
|
330
|
+
def tagend(name)
|
331
|
+
case name
|
332
|
+
when 'Id' then @distribution[:aws_id] = @text
|
333
|
+
when 'Status' then @distribution[:status] = @text
|
334
|
+
when 'LastModifiedTime' then @distribution[:last_modified_time] = Time.parse(@text)
|
335
|
+
when 'DomainName' then @distribution[:domain_name] = @text
|
336
|
+
when 'Origin' then @distribution[:origin] = @text
|
337
|
+
when 'Comment' then @distribution[:comment] = AcfInterface::unescape(@text)
|
338
|
+
when 'CNAME' then @distribution[:cnames] << @text
|
339
|
+
when 'DistributionSummary' then @result << @distribution
|
340
|
+
end
|
341
|
+
end
|
342
|
+
end
|
343
|
+
|
344
|
+
class AcfDistributionParser < RightAWSParser # :nodoc:
|
345
|
+
def reset
|
346
|
+
@result = { :cnames => [] }
|
347
|
+
end
|
348
|
+
def tagend(name)
|
349
|
+
case name
|
350
|
+
when 'Id' then @result[:aws_id] = @text
|
351
|
+
when 'Status' then @result[:status] = @text
|
352
|
+
when 'LastModifiedTime' then @result[:last_modified_time] = Time.parse(@text)
|
353
|
+
when 'DomainName' then @result[:domain_name] = @text
|
354
|
+
when 'Origin' then @result[:origin] = @text
|
355
|
+
when 'CallerReference' then @result[:caller_reference] = @text
|
356
|
+
when 'Comment' then @result[:comment] = AcfInterface::unescape(@text)
|
357
|
+
when 'Enabled' then @result[:enabled] = @text == 'true' ? true : false
|
358
|
+
when 'CNAME' then @result[:cnames] << @text
|
359
|
+
end
|
360
|
+
end
|
361
|
+
end
|
362
|
+
|
363
|
+
class AcfDistributionConfigParser < RightAWSParser # :nodoc:
|
364
|
+
def reset
|
365
|
+
@result = { :cnames => [] }
|
366
|
+
end
|
367
|
+
def tagend(name)
|
368
|
+
case name
|
369
|
+
when 'Origin' then @result[:origin] = @text
|
370
|
+
when 'CallerReference' then @result[:caller_reference] = @text
|
371
|
+
when 'Comment' then @result[:comment] = AcfInterface::unescape(@text)
|
372
|
+
when 'Enabled' then @result[:enabled] = @text == 'true' ? true : false
|
373
|
+
when 'CNAME' then @result[:cnames] << @text
|
374
|
+
end
|
375
|
+
end
|
376
|
+
end
|
377
|
+
|
378
|
+
end
|
379
|
+
end
|
@@ -0,0 +1,39 @@
|
|
1
|
+
#
|
2
|
+
# Copyright (c) 2007-2008 RightScale Inc
|
3
|
+
#
|
4
|
+
# Permission is hereby granted, free of charge, to any person obtaining
|
5
|
+
# a copy of this software and associated documentation files (the
|
6
|
+
# "Software"), to deal in the Software without restriction, including
|
7
|
+
# without limitation the rights to use, copy, modify, merge, publish,
|
8
|
+
# distribute, sublicense, and/or sell copies of the Software, and to
|
9
|
+
# permit persons to whom the Software is furnished to do so, subject to
|
10
|
+
# the following conditions:
|
11
|
+
#
|
12
|
+
# The above copyright notice and this permission notice shall be
|
13
|
+
# included in all copies or substantial portions of the Software.
|
14
|
+
#
|
15
|
+
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
16
|
+
# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
17
|
+
# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
18
|
+
# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
|
19
|
+
# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
|
20
|
+
# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
|
21
|
+
# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
22
|
+
#
|
23
|
+
#
|
24
|
+
|
25
|
+
|
26
|
+
# A hack because there's a bug in add! in Benchmark::Tms
|
27
|
+
module Benchmark #:nodoc:
|
28
|
+
class Tms #:nodoc:
|
29
|
+
def add!(&blk)
|
30
|
+
t = Benchmark::measure(&blk)
|
31
|
+
@utime = utime + t.utime
|
32
|
+
@stime = stime + t.stime
|
33
|
+
@cutime = cutime + t.cutime
|
34
|
+
@cstime = cstime + t.cstime
|
35
|
+
@real = real + t.real
|
36
|
+
self
|
37
|
+
end
|
38
|
+
end
|
39
|
+
end
|
@@ -0,0 +1,797 @@
|
|
1
|
+
#
|
2
|
+
# Copyright (c) 2007-2008 RightScale Inc
|
3
|
+
#
|
4
|
+
# Permission is hereby granted, free of charge, to any person obtaining
|
5
|
+
# a copy of this software and associated documentation files (the
|
6
|
+
# "Software"), to deal in the Software without restriction, including
|
7
|
+
# without limitation the rights to use, copy, modify, merge, publish,
|
8
|
+
# distribute, sublicense, and/or sell copies of the Software, and to
|
9
|
+
# permit persons to whom the Software is furnished to do so, subject to
|
10
|
+
# the following conditions:
|
11
|
+
#
|
12
|
+
# The above copyright notice and this permission notice shall be
|
13
|
+
# included in all copies or substantial portions of the Software.
|
14
|
+
#
|
15
|
+
# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
|
16
|
+
# EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
|
17
|
+
# MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
|
18
|
+
# NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
|
19
|
+
# LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
|
20
|
+
# OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
|
21
|
+
# WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
|
22
|
+
#
|
23
|
+
|
24
|
+
# Test
|
25
|
+
module RightAws
|
26
|
+
require 'md5'
|
27
|
+
require 'pp'
|
28
|
+
|
29
|
+
class AwsUtils #:nodoc:
|
30
|
+
@@digest1 = OpenSSL::Digest::Digest.new("sha1")
|
31
|
+
@@digest256 = nil
|
32
|
+
if OpenSSL::OPENSSL_VERSION_NUMBER > 0x00908000
|
33
|
+
@@digest256 = OpenSSL::Digest::Digest.new("sha256") rescue nil # Some installation may not support sha256
|
34
|
+
end
|
35
|
+
|
36
|
+
def self.sign(aws_secret_access_key, auth_string)
|
37
|
+
Base64.encode64(OpenSSL::HMAC.digest(@@digest1, aws_secret_access_key, auth_string)).strip
|
38
|
+
end
|
39
|
+
|
40
|
+
# Escape a string accordingly Amazon rulles
|
41
|
+
# http://docs.amazonwebservices.com/AmazonSimpleDB/2007-11-07/DeveloperGuide/index.html?REST_RESTAuth.html
|
42
|
+
def self.amz_escape(param)
|
43
|
+
param.to_s.gsub(/([^a-zA-Z0-9._~-]+)/n) do
|
44
|
+
'%' + $1.unpack('H2' * $1.size).join('%').upcase
|
45
|
+
end
|
46
|
+
end
|
47
|
+
|
48
|
+
# Set a timestamp and a signature version
|
49
|
+
def self.fix_service_params(service_hash, signature)
|
50
|
+
service_hash["Timestamp"] ||= Time.now.utc.strftime("%Y-%m-%dT%H:%M:%S.000Z") unless service_hash["Expires"]
|
51
|
+
service_hash["SignatureVersion"] = signature
|
52
|
+
service_hash
|
53
|
+
end
|
54
|
+
|
55
|
+
# Signature Version 0
|
56
|
+
# A deprecated guy (should work till septemper 2009)
|
57
|
+
def self.sign_request_v0(aws_secret_access_key, service_hash)
|
58
|
+
fix_service_params(service_hash, '0')
|
59
|
+
string_to_sign = "#{service_hash['Action']}#{service_hash['Timestamp'] || service_hash['Expires']}"
|
60
|
+
service_hash['Signature'] = AwsUtils::sign(aws_secret_access_key, string_to_sign)
|
61
|
+
service_hash.to_a.collect{|key,val| "#{amz_escape(key)}=#{amz_escape(val.to_s)}" }.join("&")
|
62
|
+
end
|
63
|
+
|
64
|
+
# Signature Version 1
|
65
|
+
# Another deprecated guy (should work till septemper 2009)
|
66
|
+
def self.sign_request_v1(aws_secret_access_key, service_hash)
|
67
|
+
fix_service_params(service_hash, '1')
|
68
|
+
string_to_sign = service_hash.sort{|a,b| (a[0].to_s.downcase)<=>(b[0].to_s.downcase)}.to_s
|
69
|
+
service_hash['Signature'] = AwsUtils::sign(aws_secret_access_key, string_to_sign)
|
70
|
+
service_hash.to_a.collect{|key,val| "#{amz_escape(key)}=#{amz_escape(val.to_s)}" }.join("&")
|
71
|
+
end
|
72
|
+
|
73
|
+
# Signature Version 2
|
74
|
+
# EC2, SQS and SDB requests must be signed by this guy.
|
75
|
+
# See: http://docs.amazonwebservices.com/AmazonSimpleDB/2007-11-07/DeveloperGuide/index.html?REST_RESTAuth.html
|
76
|
+
# http://developer.amazonwebservices.com/connect/entry.jspa?externalID=1928
|
77
|
+
def self.sign_request_v2(aws_secret_access_key, service_hash, http_verb, host, uri)
|
78
|
+
fix_service_params(service_hash, '2')
|
79
|
+
# select a signing method (make an old openssl working with sha1)
|
80
|
+
# make 'HmacSHA256' to be a default one
|
81
|
+
service_hash['SignatureMethod'] = 'HmacSHA256' unless ['HmacSHA256', 'HmacSHA1'].include?(service_hash['SignatureMethod'])
|
82
|
+
service_hash['SignatureMethod'] = 'HmacSHA1' unless @@digest256
|
83
|
+
# select a digest
|
84
|
+
digest = (service_hash['SignatureMethod'] == 'HmacSHA256' ? @@digest256 : @@digest1)
|
85
|
+
# form string to sign
|
86
|
+
canonical_string = service_hash.keys.sort.map do |key|
|
87
|
+
"#{amz_escape(key)}=#{amz_escape(service_hash[key])}"
|
88
|
+
end.join('&')
|
89
|
+
string_to_sign = "#{http_verb.to_s.upcase}\n#{host.downcase}\n#{uri}\n#{canonical_string}"
|
90
|
+
# sign the string
|
91
|
+
signature = amz_escape(Base64.encode64(OpenSSL::HMAC.digest(digest, aws_secret_access_key, string_to_sign)).strip)
|
92
|
+
"#{canonical_string}&Signature=#{signature}"
|
93
|
+
end
|
94
|
+
|
95
|
+
# From Amazon's SQS Dev Guide, a brief description of how to escape:
|
96
|
+
# "URL encode the computed signature and other query parameters as specified in
|
97
|
+
# RFC1738, section 2.2. In addition, because the + character is interpreted as a blank space
|
98
|
+
# by Sun Java classes that perform URL decoding, make sure to encode the + character
|
99
|
+
# although it is not required by RFC1738."
|
100
|
+
# Avoid using CGI::escape to escape URIs.
|
101
|
+
# CGI::escape will escape characters in the protocol, host, and port
|
102
|
+
# sections of the URI. Only target chars in the query
|
103
|
+
# string should be escaped.
|
104
|
+
def self.URLencode(raw)
|
105
|
+
e = URI.escape(raw)
|
106
|
+
e.gsub(/\+/, "%2b")
|
107
|
+
end
|
108
|
+
|
109
|
+
def self.allow_only(allowed_keys, params)
|
110
|
+
bogus_args = []
|
111
|
+
params.keys.each {|p| bogus_args.push(p) unless allowed_keys.include?(p) }
|
112
|
+
raise AwsError.new("The following arguments were given but are not legal for the function call #{caller_method}: #{bogus_args.inspect}") if bogus_args.length > 0
|
113
|
+
end
|
114
|
+
|
115
|
+
def self.mandatory_arguments(required_args, params)
|
116
|
+
rargs = required_args.dup
|
117
|
+
params.keys.each {|p| rargs.delete(p)}
|
118
|
+
raise AwsError.new("The following mandatory arguments were not provided to #{caller_method}: #{rargs.inspect}") if rargs.length > 0
|
119
|
+
end
|
120
|
+
|
121
|
+
def self.caller_method
|
122
|
+
caller[1]=~/`(.*?)'/
|
123
|
+
$1
|
124
|
+
end
|
125
|
+
|
126
|
+
end
|
127
|
+
|
128
|
+
class AwsBenchmarkingBlock #:nodoc:
|
129
|
+
attr_accessor :xml, :service
|
130
|
+
def initialize
|
131
|
+
# Benchmark::Tms instance for service (Ec2, S3, or SQS) access benchmarking.
|
132
|
+
@service = Benchmark::Tms.new()
|
133
|
+
# Benchmark::Tms instance for XML parsing benchmarking.
|
134
|
+
@xml = Benchmark::Tms.new()
|
135
|
+
end
|
136
|
+
end
|
137
|
+
|
138
|
+
class AwsNoChange < RuntimeError
|
139
|
+
end
|
140
|
+
|
141
|
+
class RightAwsBase
|
142
|
+
|
143
|
+
# Amazon HTTP Error handling
|
144
|
+
|
145
|
+
# Text, if found in an error message returned by AWS, indicates that this may be a transient
|
146
|
+
# error. Transient errors are automatically retried with exponential back-off.
|
147
|
+
AMAZON_PROBLEMS = [ 'internal service error',
|
148
|
+
'is currently unavailable',
|
149
|
+
'no response from',
|
150
|
+
'Please try again',
|
151
|
+
'InternalError',
|
152
|
+
'ServiceUnavailable', #from SQS docs
|
153
|
+
'Unavailable',
|
154
|
+
'This application is not currently available',
|
155
|
+
'InsufficientInstanceCapacity'
|
156
|
+
]
|
157
|
+
@@amazon_problems = AMAZON_PROBLEMS
|
158
|
+
# Returns a list of Amazon service responses which are known to be transient problems.
|
159
|
+
# We have to re-request if we get any of them, because the problem will probably disappear.
|
160
|
+
# By default this method returns the same value as the AMAZON_PROBLEMS const.
|
161
|
+
def self.amazon_problems
|
162
|
+
@@amazon_problems
|
163
|
+
end
|
164
|
+
|
165
|
+
# Sets the list of Amazon side problems. Use in conjunction with the
|
166
|
+
# getter to append problems.
|
167
|
+
def self.amazon_problems=(problems_list)
|
168
|
+
@@amazon_problems = problems_list
|
169
|
+
end
|
170
|
+
|
171
|
+
end
|
172
|
+
|
173
|
+
module RightAwsBaseInterface
|
174
|
+
DEFAULT_SIGNATURE_VERSION = '2'
|
175
|
+
|
176
|
+
@@caching = false
|
177
|
+
def self.caching
|
178
|
+
@@caching
|
179
|
+
end
|
180
|
+
def self.caching=(caching)
|
181
|
+
@@caching = caching
|
182
|
+
end
|
183
|
+
|
184
|
+
# Current aws_access_key_id
|
185
|
+
attr_reader :aws_access_key_id
|
186
|
+
# Last HTTP request object
|
187
|
+
attr_reader :last_request
|
188
|
+
# Last HTTP response object
|
189
|
+
attr_reader :last_response
|
190
|
+
# Last AWS errors list (used by AWSErrorHandler)
|
191
|
+
attr_accessor :last_errors
|
192
|
+
# Last AWS request id (used by AWSErrorHandler)
|
193
|
+
attr_accessor :last_request_id
|
194
|
+
# Logger object
|
195
|
+
attr_accessor :logger
|
196
|
+
# Initial params hash
|
197
|
+
attr_accessor :params
|
198
|
+
# RightHttpConnection instance
|
199
|
+
attr_reader :connection
|
200
|
+
# Cache
|
201
|
+
attr_reader :cache
|
202
|
+
# Signature version (all services except s3)
|
203
|
+
attr_reader :signature_version
|
204
|
+
|
205
|
+
def init(service_info, aws_access_key_id, aws_secret_access_key, params={}) #:nodoc:
|
206
|
+
@params = params
|
207
|
+
raise AwsError.new("AWS access keys are required to operate on #{service_info[:name]}") \
|
208
|
+
if aws_access_key_id.blank? || aws_secret_access_key.blank?
|
209
|
+
@aws_access_key_id = aws_access_key_id
|
210
|
+
@aws_secret_access_key = aws_secret_access_key
|
211
|
+
# if the endpoint was explicitly defined - then use it
|
212
|
+
if @params[:endpoint_url]
|
213
|
+
@params[:server] = URI.parse(@params[:endpoint_url]).host
|
214
|
+
@params[:port] = URI.parse(@params[:endpoint_url]).port
|
215
|
+
@params[:service] = URI.parse(@params[:endpoint_url]).path
|
216
|
+
@params[:protocol] = URI.parse(@params[:endpoint_url]).scheme
|
217
|
+
@params[:region] = nil
|
218
|
+
else
|
219
|
+
@params[:server] ||= service_info[:default_host]
|
220
|
+
@params[:server] = "#{@params[:region]}.#{@params[:server]}" if @params[:region]
|
221
|
+
@params[:port] ||= service_info[:default_port]
|
222
|
+
@params[:service] ||= service_info[:default_service]
|
223
|
+
@params[:protocol] ||= service_info[:default_protocol]
|
224
|
+
end
|
225
|
+
@params[:multi_thread] ||= defined?(AWS_DAEMON)
|
226
|
+
@logger = @params[:logger]
|
227
|
+
@logger = RAILS_DEFAULT_LOGGER if !@logger && defined?(RAILS_DEFAULT_LOGGER)
|
228
|
+
@logger = Logger.new(STDOUT) if !@logger
|
229
|
+
@logger.info "New #{self.class.name} using #{@params[:multi_thread] ? 'multi' : 'single'}-threaded mode"
|
230
|
+
@error_handler = nil
|
231
|
+
@cache = {}
|
232
|
+
@signature_version = (params[:signature_version] || DEFAULT_SIGNATURE_VERSION).to_s
|
233
|
+
end
|
234
|
+
|
235
|
+
def signed_service_params(aws_secret_access_key, service_hash, http_verb=nil, host=nil, service=nil )
|
236
|
+
case signature_version.to_s
|
237
|
+
when '0' then AwsUtils::sign_request_v0(aws_secret_access_key, service_hash)
|
238
|
+
when '1' then AwsUtils::sign_request_v1(aws_secret_access_key, service_hash)
|
239
|
+
when '2' then AwsUtils::sign_request_v2(aws_secret_access_key, service_hash, http_verb, host, service)
|
240
|
+
else raise AwsError.new("Unknown signature version (#{signature_version.to_s}) requested")
|
241
|
+
end
|
242
|
+
end
|
243
|
+
|
244
|
+
# Returns +true+ if the describe_xxx responses are being cached
|
245
|
+
def caching?
|
246
|
+
@params.key?(:cache) ? @params[:cache] : @@caching
|
247
|
+
end
|
248
|
+
|
249
|
+
# Check if the aws function response hits the cache or not.
|
250
|
+
# If the cache hits:
|
251
|
+
# - raises an +AwsNoChange+ exception if +do_raise+ == +:raise+.
|
252
|
+
# - returnes parsed response from the cache if it exists or +true+ otherwise.
|
253
|
+
# If the cache miss or the caching is off then returns +false+.
|
254
|
+
def cache_hits?(function, response, do_raise=:raise)
|
255
|
+
result = false
|
256
|
+
if caching?
|
257
|
+
function = function.to_sym
|
258
|
+
# get rid of requestId (this bad boy was added for API 2008-08-08+ and it is uniq for every response)
|
259
|
+
response = response.sub(%r{<requestId>.+?</requestId>}, '')
|
260
|
+
response_md5 = MD5.md5(response).to_s
|
261
|
+
# check for changes
|
262
|
+
unless @cache[function] && @cache[function][:response_md5] == response_md5
|
263
|
+
# well, the response is new, reset cache data
|
264
|
+
update_cache(function, {:response_md5 => response_md5,
|
265
|
+
:timestamp => Time.now,
|
266
|
+
:hits => 0,
|
267
|
+
:parsed => nil})
|
268
|
+
else
|
269
|
+
# aha, cache hits, update the data and throw an exception if needed
|
270
|
+
@cache[function][:hits] += 1
|
271
|
+
if do_raise == :raise
|
272
|
+
raise(AwsNoChange, "Cache hit: #{function} response has not changed since "+
|
273
|
+
"#{@cache[function][:timestamp].strftime('%Y-%m-%d %H:%M:%S')}, "+
|
274
|
+
"hits: #{@cache[function][:hits]}.")
|
275
|
+
else
|
276
|
+
result = @cache[function][:parsed] || true
|
277
|
+
end
|
278
|
+
end
|
279
|
+
end
|
280
|
+
result
|
281
|
+
end
|
282
|
+
|
283
|
+
def update_cache(function, hash)
|
284
|
+
(@cache[function.to_sym] ||= {}).merge!(hash) if caching?
|
285
|
+
end
|
286
|
+
|
287
|
+
def on_exception(options={:raise=>true, :log=>true}) # :nodoc:
|
288
|
+
raise if $!.is_a?(AwsNoChange)
|
289
|
+
AwsError::on_aws_exception(self, options)
|
290
|
+
end
|
291
|
+
|
292
|
+
# Return +true+ if this instance works in multi_thread mode and +false+ otherwise.
|
293
|
+
def multi_thread
|
294
|
+
@params[:multi_thread]
|
295
|
+
end
|
296
|
+
|
297
|
+
def request_info_impl(connection, benchblock, request, parser, &block) #:nodoc:
|
298
|
+
@connection = connection
|
299
|
+
@last_request = request[:request]
|
300
|
+
@last_response = nil
|
301
|
+
response=nil
|
302
|
+
blockexception = nil
|
303
|
+
|
304
|
+
if(block != nil)
|
305
|
+
# TRB 9/17/07 Careful - because we are passing in blocks, we get a situation where
|
306
|
+
# an exception may get thrown in the block body (which is high-level
|
307
|
+
# code either here or in the application) but gets caught in the
|
308
|
+
# low-level code of HttpConnection. The solution is not to let any
|
309
|
+
# exception escape the block that we pass to HttpConnection::request.
|
310
|
+
# Exceptions can originate from code directly in the block, or from user
|
311
|
+
# code called in the other block which is passed to response.read_body.
|
312
|
+
benchblock.service.add! do
|
313
|
+
responsehdr = @connection.request(request) do |response|
|
314
|
+
#########
|
315
|
+
begin
|
316
|
+
@last_response = response
|
317
|
+
if response.is_a?(Net::HTTPSuccess)
|
318
|
+
@error_handler = nil
|
319
|
+
response.read_body(&block)
|
320
|
+
else
|
321
|
+
@error_handler = AWSErrorHandler.new(self, parser, :errors_list => self.class.amazon_problems) unless @error_handler
|
322
|
+
check_result = @error_handler.check(request)
|
323
|
+
if check_result
|
324
|
+
@error_handler = nil
|
325
|
+
return check_result
|
326
|
+
end
|
327
|
+
raise AwsError.new(@last_errors, @last_response.code, @last_request_id)
|
328
|
+
end
|
329
|
+
rescue Exception => e
|
330
|
+
blockexception = e
|
331
|
+
end
|
332
|
+
end
|
333
|
+
#########
|
334
|
+
|
335
|
+
#OK, now we are out of the block passed to the lower level
|
336
|
+
if(blockexception)
|
337
|
+
raise blockexception
|
338
|
+
end
|
339
|
+
benchblock.xml.add! do
|
340
|
+
parser.parse(responsehdr)
|
341
|
+
end
|
342
|
+
return parser.result
|
343
|
+
end
|
344
|
+
else
|
345
|
+
benchblock.service.add!{ response = @connection.request(request) }
|
346
|
+
# check response for errors...
|
347
|
+
@last_response = response
|
348
|
+
if response.is_a?(Net::HTTPSuccess)
|
349
|
+
@error_handler = nil
|
350
|
+
benchblock.xml.add! { parser.parse(response) }
|
351
|
+
return parser.result
|
352
|
+
else
|
353
|
+
@error_handler = AWSErrorHandler.new(self, parser, :errors_list => self.class.amazon_problems) unless @error_handler
|
354
|
+
check_result = @error_handler.check(request)
|
355
|
+
if check_result
|
356
|
+
@error_handler = nil
|
357
|
+
return check_result
|
358
|
+
end
|
359
|
+
raise AwsError.new(@last_errors, @last_response.code, @last_request_id)
|
360
|
+
end
|
361
|
+
end
|
362
|
+
rescue
|
363
|
+
@error_handler = nil
|
364
|
+
raise
|
365
|
+
end
|
366
|
+
|
367
|
+
def request_cache_or_info(method, link, parser_class, benchblock, use_cache=true) #:nodoc:
|
368
|
+
# We do not want to break the logic of parsing hence will use a dummy parser to process all the standard
|
369
|
+
# steps (errors checking etc). The dummy parser does nothig - just returns back the params it received.
|
370
|
+
# If the caching is enabled and hit then throw AwsNoChange.
|
371
|
+
# P.S. caching works for the whole images list only! (when the list param is blank)
|
372
|
+
# check cache
|
373
|
+
response, params = request_info(link, RightDummyParser.new)
|
374
|
+
cache_hits?(method.to_sym, response.body) if use_cache
|
375
|
+
parser = parser_class.new(:logger => @logger)
|
376
|
+
benchblock.xml.add!{ parser.parse(response, params) }
|
377
|
+
result = block_given? ? yield(parser) : parser.result
|
378
|
+
# update parsed data
|
379
|
+
update_cache(method.to_sym, :parsed => result) if use_cache
|
380
|
+
result
|
381
|
+
end
|
382
|
+
|
383
|
+
# Returns Amazons request ID for the latest request
|
384
|
+
def last_request_id
|
385
|
+
@last_response && @last_response.body.to_s[%r{<requestId>(.+?)</requestId>}] && $1
|
386
|
+
end
|
387
|
+
|
388
|
+
end
|
389
|
+
|
390
|
+
|
391
|
+
# Exception class to signal any Amazon errors. All errors occuring during calls to Amazon's
|
392
|
+
# web services raise this type of error.
|
393
|
+
# Attribute inherited by RuntimeError:
|
394
|
+
# message - the text of the error, generally as returned by AWS in its XML response.
|
395
|
+
class AwsError < RuntimeError
|
396
|
+
|
397
|
+
# either an array of errors where each item is itself an array of [code, message]),
|
398
|
+
# or an error string if the error was raised manually, as in <tt>AwsError.new('err_text')</tt>
|
399
|
+
attr_reader :errors
|
400
|
+
|
401
|
+
# Request id (if exists)
|
402
|
+
attr_reader :request_id
|
403
|
+
|
404
|
+
# Response HTTP error code
|
405
|
+
attr_reader :http_code
|
406
|
+
|
407
|
+
def initialize(errors=nil, http_code=nil, request_id=nil)
|
408
|
+
@errors = errors
|
409
|
+
@request_id = request_id
|
410
|
+
@http_code = http_code
|
411
|
+
super(@errors.is_a?(Array) ? @errors.map{|code, msg| "#{code}: #{msg}"}.join("; ") : @errors.to_s)
|
412
|
+
end
|
413
|
+
|
414
|
+
# Does any of the error messages include the regexp +pattern+?
|
415
|
+
# Used to determine whether to retry request.
|
416
|
+
def include?(pattern)
|
417
|
+
if @errors.is_a?(Array)
|
418
|
+
@errors.each{ |code, msg| return true if code =~ pattern }
|
419
|
+
else
|
420
|
+
return true if @errors_str =~ pattern
|
421
|
+
end
|
422
|
+
false
|
423
|
+
end
|
424
|
+
|
425
|
+
# Generic handler for AwsErrors. +aws+ is the RightAws::S3, RightAws::EC2, or RightAws::SQS
|
426
|
+
# object that caused the exception (it must provide last_request and last_response). Supported
|
427
|
+
# boolean options are:
|
428
|
+
# * <tt>:log</tt> print a message into the log using aws.logger to access the Logger
|
429
|
+
# * <tt>:puts</tt> do a "puts" of the error
|
430
|
+
# * <tt>:raise</tt> re-raise the error after logging
|
431
|
+
def self.on_aws_exception(aws, options={:raise=>true, :log=>true})
|
432
|
+
# Only log & notify if not user error
|
433
|
+
if !options[:raise] || system_error?($!)
|
434
|
+
error_text = "#{$!.inspect}\n#{$@}.join('\n')}"
|
435
|
+
puts error_text if options[:puts]
|
436
|
+
# Log the error
|
437
|
+
if options[:log]
|
438
|
+
request = aws.last_request ? aws.last_request.path : '-none-'
|
439
|
+
response = aws.last_response ? "#{aws.last_response.code} -- #{aws.last_response.message} -- #{aws.last_response.body}" : '-none-'
|
440
|
+
aws.logger.error error_text
|
441
|
+
aws.logger.error "Request was: #{request}"
|
442
|
+
aws.logger.error "Response was: #{response}"
|
443
|
+
end
|
444
|
+
end
|
445
|
+
raise if options[:raise] # re-raise an exception
|
446
|
+
return nil
|
447
|
+
end
|
448
|
+
|
449
|
+
# True if e is an AWS system error, i.e. something that is for sure not the caller's fault.
|
450
|
+
# Used to force logging.
|
451
|
+
def self.system_error?(e)
|
452
|
+
!e.is_a?(self) || e.message =~ /InternalError|InsufficientInstanceCapacity|Unavailable/
|
453
|
+
end
|
454
|
+
|
455
|
+
end
|
456
|
+
|
457
|
+
|
458
|
+
class AWSErrorHandler
|
459
|
+
# 0-100 (%)
|
460
|
+
DEFAULT_CLOSE_ON_4XX_PROBABILITY = 10
|
461
|
+
|
462
|
+
@@reiteration_start_delay = 0.2
|
463
|
+
def self.reiteration_start_delay
|
464
|
+
@@reiteration_start_delay
|
465
|
+
end
|
466
|
+
def self.reiteration_start_delay=(reiteration_start_delay)
|
467
|
+
@@reiteration_start_delay = reiteration_start_delay
|
468
|
+
end
|
469
|
+
|
470
|
+
@@reiteration_time = 5
|
471
|
+
def self.reiteration_time
|
472
|
+
@@reiteration_time
|
473
|
+
end
|
474
|
+
def self.reiteration_time=(reiteration_time)
|
475
|
+
@@reiteration_time = reiteration_time
|
476
|
+
end
|
477
|
+
|
478
|
+
@@close_on_error = true
|
479
|
+
def self.close_on_error
|
480
|
+
@@close_on_error
|
481
|
+
end
|
482
|
+
def self.close_on_error=(close_on_error)
|
483
|
+
@@close_on_error = close_on_error
|
484
|
+
end
|
485
|
+
|
486
|
+
@@close_on_4xx_probability = DEFAULT_CLOSE_ON_4XX_PROBABILITY
|
487
|
+
def self.close_on_4xx_probability
|
488
|
+
@@close_on_4xx_probability
|
489
|
+
end
|
490
|
+
def self.close_on_4xx_probability=(close_on_4xx_probability)
|
491
|
+
@@close_on_4xx_probability = close_on_4xx_probability
|
492
|
+
end
|
493
|
+
|
494
|
+
# params:
|
495
|
+
# :reiteration_time
|
496
|
+
# :errors_list
|
497
|
+
# :close_on_error = true | false
|
498
|
+
# :close_on_4xx_probability = 1-100
|
499
|
+
def initialize(aws, parser, params={}) #:nodoc:
|
500
|
+
@aws = aws # Link to RightEc2 | RightSqs | RightS3 instance
|
501
|
+
@parser = parser # parser to parse Amazon response
|
502
|
+
@started_at = Time.now
|
503
|
+
@stop_at = @started_at + (params[:reiteration_time] || @@reiteration_time)
|
504
|
+
@errors_list = params[:errors_list] || []
|
505
|
+
@reiteration_delay = @@reiteration_start_delay
|
506
|
+
@retries = 0
|
507
|
+
# close current HTTP(S) connection on 5xx, errors from list and 4xx errors
|
508
|
+
@close_on_error = params[:close_on_error].nil? ? @@close_on_error : params[:close_on_error]
|
509
|
+
@close_on_4xx_probability = params[:close_on_4xx_probability] || @@close_on_4xx_probability
|
510
|
+
end
|
511
|
+
|
512
|
+
# Returns false if
|
513
|
+
def check(request) #:nodoc:
|
514
|
+
result = false
|
515
|
+
error_found = false
|
516
|
+
redirect_detected= false
|
517
|
+
error_match = nil
|
518
|
+
last_errors_text = ''
|
519
|
+
response = @aws.last_response
|
520
|
+
# log error
|
521
|
+
request_text_data = "#{request[:server]}:#{request[:port]}#{request[:request].path}"
|
522
|
+
# is this a redirect?
|
523
|
+
# yes!
|
524
|
+
if response.is_a?(Net::HTTPRedirection)
|
525
|
+
redirect_detected = true
|
526
|
+
else
|
527
|
+
# no, it's an error ...
|
528
|
+
@aws.logger.warn("##### #{@aws.class.name} returned an error: #{response.code} #{response.message}\n#{response.body} #####")
|
529
|
+
@aws.logger.warn("##### #{@aws.class.name} request: #{request_text_data} ####")
|
530
|
+
end
|
531
|
+
# Check response body: if it is an Amazon XML document or not:
|
532
|
+
if redirect_detected || (response.body && response.body[/<\?xml/]) # ... it is a xml document
|
533
|
+
@aws.class.bench_xml.add! do
|
534
|
+
error_parser = RightErrorResponseParser.new
|
535
|
+
error_parser.parse(response)
|
536
|
+
@aws.last_errors = error_parser.errors
|
537
|
+
@aws.last_request_id = error_parser.requestID
|
538
|
+
last_errors_text = @aws.last_errors.flatten.join("\n")
|
539
|
+
# on redirect :
|
540
|
+
if redirect_detected
|
541
|
+
location = response['location']
|
542
|
+
# ... log information and ...
|
543
|
+
@aws.logger.info("##### #{@aws.class.name} redirect requested: #{response.code} #{response.message} #####")
|
544
|
+
@aws.logger.info("##### New location: #{location} #####")
|
545
|
+
# ... fix the connection data
|
546
|
+
request[:server] = URI.parse(location).host
|
547
|
+
request[:protocol] = URI.parse(location).scheme
|
548
|
+
request[:port] = URI.parse(location).port
|
549
|
+
end
|
550
|
+
end
|
551
|
+
else # ... it is not a xml document(probably just a html page?)
|
552
|
+
@aws.last_errors = [[response.code, "#{response.message} (#{request_text_data})"]]
|
553
|
+
@aws.last_request_id = '-undefined-'
|
554
|
+
last_errors_text = response.message
|
555
|
+
end
|
556
|
+
# now - check the error
|
557
|
+
unless redirect_detected
|
558
|
+
@errors_list.each do |error_to_find|
|
559
|
+
if last_errors_text[/#{error_to_find}/i]
|
560
|
+
error_found = true
|
561
|
+
error_match = error_to_find
|
562
|
+
@aws.logger.warn("##### Retry is needed, error pattern match: #{error_to_find} #####")
|
563
|
+
break
|
564
|
+
end
|
565
|
+
end
|
566
|
+
end
|
567
|
+
# check the time has gone from the first error come
|
568
|
+
if redirect_detected || error_found
|
569
|
+
# Close the connection to the server and recreate a new one.
|
570
|
+
# It may have a chance that one server is a semi-down and reconnection
|
571
|
+
# will help us to connect to the other server
|
572
|
+
if !redirect_detected && @close_on_error
|
573
|
+
@aws.connection.finish "#{self.class.name}: error match to pattern '#{error_match}'"
|
574
|
+
end
|
575
|
+
|
576
|
+
if (Time.now < @stop_at)
|
577
|
+
@retries += 1
|
578
|
+
unless redirect_detected
|
579
|
+
@aws.logger.warn("##### Retry ##{@retries} is being performed. Sleeping for #{@reiteration_delay} sec. Whole time: #{Time.now-@started_at} sec ####")
|
580
|
+
sleep @reiteration_delay
|
581
|
+
@reiteration_delay *= 2
|
582
|
+
|
583
|
+
# Always make sure that the fp is set to point to the beginning(?)
|
584
|
+
# of the File/IO. TODO: it assumes that offset is 0, which is bad.
|
585
|
+
if(request[:request].body_stream && request[:request].body_stream.respond_to?(:pos))
|
586
|
+
begin
|
587
|
+
request[:request].body_stream.pos = 0
|
588
|
+
rescue Exception => e
|
589
|
+
@logger.warn("Retry may fail due to unable to reset the file pointer" +
|
590
|
+
" -- #{self.class.name} : #{e.inspect}")
|
591
|
+
end
|
592
|
+
end
|
593
|
+
else
|
594
|
+
@aws.logger.info("##### Retry ##{@retries} is being performed due to a redirect. ####")
|
595
|
+
end
|
596
|
+
result = @aws.request_info(request, @parser)
|
597
|
+
else
|
598
|
+
@aws.logger.warn("##### Ooops, time is over... ####")
|
599
|
+
end
|
600
|
+
# aha, this is unhandled error:
|
601
|
+
elsif @close_on_error
|
602
|
+
# Is this a 5xx error ?
|
603
|
+
if @aws.last_response.code.to_s[/^5\d\d$/]
|
604
|
+
@aws.connection.finish "#{self.class.name}: code: #{@aws.last_response.code}: '#{@aws.last_response.message}'"
|
605
|
+
# Is this a 4xx error ?
|
606
|
+
elsif @aws.last_response.code.to_s[/^4\d\d$/] && @close_on_4xx_probability > rand(100)
|
607
|
+
@aws.connection.finish "#{self.class.name}: code: #{@aws.last_response.code}: '#{@aws.last_response.message}', " +
|
608
|
+
"probability: #{@close_on_4xx_probability}%"
|
609
|
+
end
|
610
|
+
end
|
611
|
+
result
|
612
|
+
end
|
613
|
+
|
614
|
+
end
|
615
|
+
|
616
|
+
|
617
|
+
#-----------------------------------------------------------------
|
618
|
+
|
619
|
+
class RightSaxParserCallback #:nodoc:
|
620
|
+
def self.include_callback
|
621
|
+
include XML::SaxParser::Callbacks
|
622
|
+
end
|
623
|
+
def initialize(right_aws_parser)
|
624
|
+
@right_aws_parser = right_aws_parser
|
625
|
+
end
|
626
|
+
def on_start_element(name, attr_hash)
|
627
|
+
@right_aws_parser.tag_start(name, attr_hash)
|
628
|
+
end
|
629
|
+
def on_characters(chars)
|
630
|
+
@right_aws_parser.text(chars)
|
631
|
+
end
|
632
|
+
def on_end_element(name)
|
633
|
+
@right_aws_parser.tag_end(name)
|
634
|
+
end
|
635
|
+
def on_start_document; end
|
636
|
+
def on_comment(msg); end
|
637
|
+
def on_processing_instruction(target, data); end
|
638
|
+
def on_cdata_block(cdata); end
|
639
|
+
def on_end_document; end
|
640
|
+
end
|
641
|
+
|
642
|
+
class RightAWSParser #:nodoc:
|
643
|
+
# default parsing library
|
644
|
+
DEFAULT_XML_LIBRARY = 'rexml'
|
645
|
+
# a list of supported parsers
|
646
|
+
@@supported_xml_libs = [DEFAULT_XML_LIBRARY, 'libxml']
|
647
|
+
|
648
|
+
@@xml_lib = DEFAULT_XML_LIBRARY # xml library name: 'rexml' | 'libxml'
|
649
|
+
def self.xml_lib
|
650
|
+
@@xml_lib
|
651
|
+
end
|
652
|
+
def self.xml_lib=(new_lib_name)
|
653
|
+
@@xml_lib = new_lib_name
|
654
|
+
end
|
655
|
+
|
656
|
+
attr_accessor :result
|
657
|
+
attr_reader :xmlpath
|
658
|
+
attr_accessor :xml_lib
|
659
|
+
|
660
|
+
def initialize(params={})
|
661
|
+
@xmlpath = ''
|
662
|
+
@result = false
|
663
|
+
@text = ''
|
664
|
+
@xml_lib = params[:xml_lib] || @@xml_lib
|
665
|
+
@logger = params[:logger]
|
666
|
+
reset
|
667
|
+
end
|
668
|
+
def tag_start(name, attributes)
|
669
|
+
@text = ''
|
670
|
+
tagstart(name, attributes)
|
671
|
+
@xmlpath += @xmlpath.empty? ? name : "/#{name}"
|
672
|
+
end
|
673
|
+
def tag_end(name)
|
674
|
+
if @xmlpath =~ /^(.*?)\/?#{name}$/
|
675
|
+
@xmlpath = $1
|
676
|
+
end
|
677
|
+
tagend(name)
|
678
|
+
end
|
679
|
+
def text(text)
|
680
|
+
@text += text
|
681
|
+
tagtext(text)
|
682
|
+
end
|
683
|
+
# Parser method.
|
684
|
+
# Params:
|
685
|
+
# xml_text - xml message text(String) or Net:HTTPxxx instance (response)
|
686
|
+
# params[:xml_lib] - library name: 'rexml' | 'libxml'
|
687
|
+
def parse(xml_text, params={})
|
688
|
+
# Get response body
|
689
|
+
xml_text = xml_text.body unless xml_text.is_a?(String)
|
690
|
+
@xml_lib = params[:xml_lib] || @xml_lib
|
691
|
+
# check that we had no problems with this library otherwise use default
|
692
|
+
@xml_lib = DEFAULT_XML_LIBRARY unless @@supported_xml_libs.include?(@xml_lib)
|
693
|
+
# load xml library
|
694
|
+
if @xml_lib=='libxml' && !defined?(XML::SaxParser)
|
695
|
+
begin
|
696
|
+
require 'xml/libxml'
|
697
|
+
# is it new ? - Setup SaxParserCallback
|
698
|
+
if XML::Parser::VERSION >= '0.5.1.0'
|
699
|
+
RightSaxParserCallback.include_callback
|
700
|
+
end
|
701
|
+
rescue LoadError => e
|
702
|
+
@@supported_xml_libs.delete(@xml_lib)
|
703
|
+
@xml_lib = DEFAULT_XML_LIBRARY
|
704
|
+
if @logger
|
705
|
+
@logger.error e.inspect
|
706
|
+
@logger.error e.backtrace
|
707
|
+
@logger.info "Can not load 'libxml' library. '#{DEFAULT_XML_LIBRARY}' is used for parsing."
|
708
|
+
end
|
709
|
+
end
|
710
|
+
end
|
711
|
+
# Parse the xml text
|
712
|
+
case @xml_lib
|
713
|
+
when 'libxml'
|
714
|
+
xml = XML::SaxParser.new
|
715
|
+
xml.string = xml_text
|
716
|
+
# check libxml-ruby version
|
717
|
+
if XML::Parser::VERSION >= '0.5.1.0'
|
718
|
+
xml.callbacks = RightSaxParserCallback.new(self)
|
719
|
+
else
|
720
|
+
xml.on_start_element{|name, attr_hash| self.tag_start(name, attr_hash)}
|
721
|
+
xml.on_characters{ |text| self.text(text)}
|
722
|
+
xml.on_end_element{ |name| self.tag_end(name)}
|
723
|
+
end
|
724
|
+
xml.parse
|
725
|
+
else
|
726
|
+
REXML::Document.parse_stream(xml_text, self)
|
727
|
+
end
|
728
|
+
end
|
729
|
+
# Parser must have a lots of methods
|
730
|
+
# (see /usr/lib/ruby/1.8/rexml/parsers/streamparser.rb)
|
731
|
+
# We dont need most of them in RightAWSParser and method_missing helps us
|
732
|
+
# to skip their definition
|
733
|
+
def method_missing(method, *params)
|
734
|
+
# if the method is one of known - just skip it ...
|
735
|
+
return if [:comment, :attlistdecl, :notationdecl, :elementdecl,
|
736
|
+
:entitydecl, :cdata, :xmldecl, :attlistdecl, :instruction,
|
737
|
+
:doctype].include?(method)
|
738
|
+
# ... else - call super to raise an exception
|
739
|
+
super(method, params)
|
740
|
+
end
|
741
|
+
# the functions to be overriden by children (if nessesery)
|
742
|
+
def reset ; end
|
743
|
+
def tagstart(name, attributes); end
|
744
|
+
def tagend(name) ; end
|
745
|
+
def tagtext(text) ; end
|
746
|
+
end
|
747
|
+
|
748
|
+
#-----------------------------------------------------------------
|
749
|
+
# PARSERS: Errors
|
750
|
+
#-----------------------------------------------------------------
|
751
|
+
|
752
|
+
#<Error>
|
753
|
+
# <Code>TemporaryRedirect</Code>
|
754
|
+
# <Message>Please re-send this request to the specified temporary endpoint. Continue to use the original request endpoint for future requests.</Message>
|
755
|
+
# <RequestId>FD8D5026D1C5ABA3</RequestId>
|
756
|
+
# <Endpoint>bucket-for-k.s3-external-3.amazonaws.com</Endpoint>
|
757
|
+
# <HostId>ItJy8xPFPli1fq/JR3DzQd3iDvFCRqi1LTRmunEdM1Uf6ZtW2r2kfGPWhRE1vtaU</HostId>
|
758
|
+
# <Bucket>bucket-for-k</Bucket>
|
759
|
+
#</Error>
|
760
|
+
|
761
|
+
class RightErrorResponseParser < RightAWSParser #:nodoc:
|
762
|
+
attr_accessor :errors # array of hashes: error/message
|
763
|
+
attr_accessor :requestID
|
764
|
+
# attr_accessor :endpoint, :host_id, :bucket
|
765
|
+
def tagend(name)
|
766
|
+
case name
|
767
|
+
when 'RequestID' ; @requestID = @text
|
768
|
+
when 'Code' ; @code = @text
|
769
|
+
when 'Message' ; @message = @text
|
770
|
+
# when 'Endpoint' ; @endpoint = @text
|
771
|
+
# when 'HostId' ; @host_id = @text
|
772
|
+
# when 'Bucket' ; @bucket = @text
|
773
|
+
when 'Error' ; @errors << [ @code, @message ]
|
774
|
+
end
|
775
|
+
end
|
776
|
+
def reset
|
777
|
+
@errors = []
|
778
|
+
end
|
779
|
+
end
|
780
|
+
|
781
|
+
# Dummy parser - does nothing
|
782
|
+
# Returns the original params back
|
783
|
+
class RightDummyParser # :nodoc:
|
784
|
+
attr_accessor :result
|
785
|
+
def parse(response, params={})
|
786
|
+
@result = [response, params]
|
787
|
+
end
|
788
|
+
end
|
789
|
+
|
790
|
+
class RightHttp2xxParser < RightAWSParser # :nodoc:
|
791
|
+
def parse(response)
|
792
|
+
@result = response.is_a?(Net::HTTPSuccess)
|
793
|
+
end
|
794
|
+
end
|
795
|
+
|
796
|
+
end
|
797
|
+
|