aws 2.3.34 → 2.4.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,179 @@
1
+ # encoding: utf-8
2
+ require File.dirname(__FILE__) + '/test_helper.rb'
3
+ require_relative 's3_test_base'
4
+ require File.dirname(__FILE__) + '/../test_credentials.rb'
5
+
6
+ class TestS3Class < S3TestBase
7
+
8
+ #---------------------------
9
+ # Aws::S3 classes
10
+ #---------------------------
11
+
12
+ def test_20_s3
13
+ # create bucket
14
+ bucket = @s.bucket(@bucket, true)
15
+ assert bucket
16
+ # check that the bucket exists
17
+ assert @s.buckets.map { |b| b.name }.include?(@bucket)
18
+ # delete bucket
19
+ assert bucket.clear
20
+ assert bucket.delete
21
+ end
22
+
23
+ def test_21_bucket_create_put_get_key
24
+ bucket = Aws::S3::Bucket.create(@s, @bucket, true)
25
+ # check that the bucket exists
26
+ assert @s.buckets.map { |b| b.name }.include?(@bucket)
27
+ assert bucket.keys.empty?, "keys are not empty: " + bucket.keys.inspect
28
+ # put data
29
+ assert bucket.put(@key3, RIGHT_OBJECT_TEXT, {'family'=>'123456'})
30
+ # get data and compare
31
+ assert_equal RIGHT_OBJECT_TEXT, bucket.get(@key3)
32
+ # get key object
33
+ key = bucket.key(@key3, true)
34
+ assert_equal Aws::S3::Key, key.class
35
+ assert key.exists?
36
+ assert_equal '123456', key.meta_headers['family']
37
+ end
38
+
39
+ def test_22_bucket_put_big_with_multibyte_chars
40
+ bucket = Aws::S3::Bucket.create(@s, @bucket, true)
41
+ super_big_string = ""
42
+ 10000.times { |i| super_big_string << "abcde Café" }
43
+ # this string has multibye values just to mess things up abit.
44
+ puts 'String made, putting...'
45
+ puts "#{super_big_string.size} - #{super_big_string.bytesize}"
46
+ assert bucket.put("super_big", super_big_string), 'Put bucket fail'
47
+
48
+ got = bucket.get("super_big")
49
+ puts 'got.class=' + got.class.name
50
+ assert_equal(super_big_string, got, "not the same yo")
51
+ end
52
+
53
+ def test_23_put_strange_things
54
+ bucket = Aws::S3::Bucket.create(@s, @bucket, true)
55
+
56
+ # this is kinda bad, you put a nil, but get an empty string back
57
+ assert bucket.put("strange", nil), 'Put bucket fail'
58
+ got = bucket.get("strange")
59
+ assert_equal("", got)
60
+
61
+ x = "\xE2\x80\x99s Café"
62
+ puts "#{x.size} - #{x.bytesize}"
63
+ assert bucket.put("multibye", x)
64
+
65
+
66
+
67
+ end
68
+
69
+ def test_30_keys
70
+ bucket = Aws::S3::Bucket.create(@s, @bucket, false)
71
+ # create first key
72
+ key3 = Aws::S3::Key.create(bucket, @key3)
73
+ key3.refresh
74
+ assert key3.exists?
75
+ assert_equal '123456', key3.meta_headers['family']
76
+ # create second key
77
+ key2 = Aws::S3::Key.create(bucket, @key2)
78
+ assert !key2.refresh
79
+ assert !key2.exists?
80
+ assert_raise(Aws::AwsError) { key2.head }
81
+ # store key
82
+ key2.meta_headers = {'family'=>'111222333'}
83
+ assert key2.put(RIGHT_OBJECT_TEXT)
84
+ # make sure that the key exists
85
+ assert key2.refresh
86
+ assert key2.exists?
87
+ assert key2.head
88
+ # get its data
89
+ assert_equal RIGHT_OBJECT_TEXT, key2.get
90
+ # drop key
91
+ assert key2.delete
92
+ assert !key2.exists?
93
+ end
94
+
95
+ def test_31_rename_key
96
+ bucket = Aws::S3::Bucket.create(@s, @bucket, false)
97
+ # -- 1 -- (key based rename)
98
+ # create a key
99
+ key = bucket.key('test/copy/1')
100
+ key.put(RIGHT_OBJECT_TEXT)
101
+ original_key = key.clone
102
+ assert key.exists?, "'test/copy/1' should exist"
103
+ # rename it
104
+ key.rename('test/copy/2')
105
+ assert_equal 'test/copy/2', key.name
106
+ assert key.exists?, "'test/copy/2' should exist"
107
+ # the original key should not exist
108
+ assert !original_key.exists?, "'test/copy/1' should not exist"
109
+ # -- 2 -- (bucket based rename)
110
+ bucket.rename_key('test/copy/2', 'test/copy/3')
111
+ assert bucket.key('test/copy/3').exists?, "'test/copy/3' should exist"
112
+ assert !bucket.key('test/copy/2').exists?, "'test/copy/2' should not exist"
113
+ end
114
+
115
+ def test_32_copy_key
116
+ bucket = Aws::S3::Bucket.create(@s, @bucket, false)
117
+ # -- 1 -- (key based copy)
118
+ # create a key
119
+ key = bucket.key('test/copy/10')
120
+ key.put(RIGHT_OBJECT_TEXT)
121
+ # make copy
122
+ new_key = key.copy('test/copy/11')
123
+ # make sure both the keys exist and have a correct data
124
+ assert key.exists?, "'test/copy/10' should exist"
125
+ assert new_key.exists?, "'test/copy/11' should exist"
126
+ assert_equal RIGHT_OBJECT_TEXT, key.get
127
+ assert_equal RIGHT_OBJECT_TEXT, new_key.get
128
+ # -- 2 -- (bucket based copy)
129
+ bucket.copy_key('test/copy/11', 'test/copy/12')
130
+ assert bucket.key('test/copy/11').exists?, "'test/copy/11' should exist"
131
+ assert bucket.key('test/copy/12').exists?, "'test/copy/12' should exist"
132
+ assert_equal RIGHT_OBJECT_TEXT, bucket.key('test/copy/11').get
133
+ assert_equal RIGHT_OBJECT_TEXT, bucket.key('test/copy/12').get
134
+ end
135
+
136
+ def test_33_move_key
137
+ bucket = Aws::S3::Bucket.create(@s, @bucket, false)
138
+ # -- 1 -- (key based copy)
139
+ # create a key
140
+ key = bucket.key('test/copy/20')
141
+ key.put(RIGHT_OBJECT_TEXT)
142
+ # move
143
+ new_key = key.move('test/copy/21')
144
+ # make sure both the keys exist and have a correct data
145
+ assert !key.exists?, "'test/copy/20' should not exist"
146
+ assert new_key.exists?, "'test/copy/21' should exist"
147
+ assert_equal RIGHT_OBJECT_TEXT, new_key.get
148
+ # -- 2 -- (bucket based copy)
149
+ bucket.copy_key('test/copy/21', 'test/copy/22')
150
+ assert bucket.key('test/copy/21').exists?, "'test/copy/21' should not exist"
151
+ assert bucket.key('test/copy/22').exists?, "'test/copy/22' should exist"
152
+ assert_equal RIGHT_OBJECT_TEXT, bucket.key('test/copy/22').get
153
+ end
154
+
155
+ def test_40_save_meta
156
+ bucket = Aws::S3::Bucket.create(@s, @bucket, false)
157
+ # create a key
158
+ key = bucket.key('test/copy/30')
159
+ key.put(RIGHT_OBJECT_TEXT)
160
+ assert key.meta_headers.blank?
161
+ # store some meta keys
162
+ meta = {'family' => 'oops', 'race' => 'troll'}
163
+ assert_equal meta, key.save_meta(meta)
164
+ # reload meta
165
+ assert_equal meta, key.reload_meta
166
+ end
167
+
168
+ def test_60_clear_delete
169
+ bucket = Aws::S3::Bucket.create(@s, @bucket, false)
170
+ # add another key
171
+ bucket.put(@key2, RIGHT_OBJECT_TEXT)
172
+ # delete 'folder'
173
+ assert_equal 1, bucket.delete_folder(@key1).size
174
+ # delete
175
+ assert_raise(Aws::AwsError) { bucket.delete }
176
+ assert bucket.delete(true)
177
+ end
178
+
179
+ end
@@ -0,0 +1,139 @@
1
+ require File.dirname(__FILE__) + '/test_helper.rb'
2
+ require_relative 's3_test_base'
3
+ require File.dirname(__FILE__) + '/../test_credentials.rb'
4
+
5
+ class TestS3Rights < S3TestBase
6
+ # Grantees
7
+
8
+ def test_30_create_bucket
9
+ bucket = @s.bucket(@bucket, true, 'public-read')
10
+ assert bucket
11
+ end
12
+
13
+ def test_31_list_grantees
14
+ bucket = Aws::S3::Bucket.create(@s, @bucket, false)
15
+ # get grantees list
16
+ grantees = bucket.grantees
17
+ # check that the grantees count equal to 2 (root, AllUsers)
18
+ assert_equal 2, grantees.size
19
+ end
20
+
21
+ def test_32_grant_revoke_drop
22
+ bucket = Aws::S3::Bucket.create(@s, @bucket, false)
23
+ # Take 'AllUsers' grantee
24
+ grantee = Aws::S3::Grantee.new(bucket, 'http://acs.amazonaws.com/groups/global/AllUsers')
25
+ # Check exists?
26
+ assert grantee.exists?
27
+ # Add grant as String
28
+ assert grantee.grant('WRITE')
29
+ # Add grants as Array
30
+ assert grantee.grant(['READ_ACP', 'WRITE_ACP'])
31
+ # Check perms count
32
+ assert_equal 4, grantee.perms.size
33
+ # revoke 'WRITE_ACP'
34
+ assert grantee.revoke('WRITE_ACP')
35
+ # Check manual perm removal method
36
+ grantee.perms -= ['READ_ACP']
37
+ grantee.apply
38
+ assert_equal 2, grantee.perms.size
39
+ # Check grantee removal if it has no permissions
40
+ assert grantee.perms = []
41
+ assert grantee.apply
42
+ assert !grantee.exists?
43
+ # Check multiple perms assignment
44
+ assert grantee.grant('FULL_CONTROL', 'READ', 'WRITE')
45
+ assert_equal ['FULL_CONTROL', 'READ', 'WRITE'].sort, grantee.perms.sort
46
+ # Check multiple perms removal
47
+ assert grantee.revoke('FULL_CONTROL', 'WRITE')
48
+ assert_equal ['READ'], grantee.perms
49
+ # check 'Drop' method
50
+ assert grantee.drop
51
+ assert !grantee.exists?
52
+ assert_equal 1, bucket.grantees.size
53
+ # Delete bucket
54
+ bucket.delete(true)
55
+ end
56
+
57
+ def test_33_key_grantees
58
+ # Create bucket
59
+ bucket = @s.bucket(@bucket, true)
60
+ # Create key
61
+ key = bucket.key(@key1)
62
+ assert key.put(RIGHT_OBJECT_TEXT, 'public-read')
63
+ # Get grantees list (must be == 2)
64
+ grantees = key.grantees
65
+ assert grantees
66
+ assert_equal 2, grantees.size
67
+ # Take one of grantees and give him 'Write' perms
68
+ grantee = grantees[0]
69
+ assert grantee.grant('WRITE')
70
+ # Drop grantee
71
+ assert grantee.drop
72
+ # Drop bucket
73
+ bucket.delete(true)
74
+ end
75
+
76
+ def test_34_bucket_create_put_with_perms
77
+ bucket = Aws::S3::Bucket.create(@s, @bucket, true)
78
+ # check that the bucket exists
79
+ assert @s.buckets.map { |b| b.name }.include?(@bucket)
80
+ assert bucket.keys.empty?
81
+ # put data (with canned ACL)
82
+ assert bucket.put(@key1, RIGHT_OBJECT_TEXT, {'family'=>'123456'}, "public-read")
83
+ # get data and compare
84
+ assert_equal RIGHT_OBJECT_TEXT, bucket.get(@key1)
85
+ # get key object
86
+ key = bucket.key(@key1, true)
87
+ assert_equal Aws::S3::Key, key.class
88
+ assert key.exists?
89
+ assert_equal '123456', key.meta_headers['family']
90
+ end
91
+
92
+ def test_35_key_put_with_perms
93
+ bucket = Aws::S3::Bucket.create(@s, @bucket, false)
94
+ # create first key
95
+ key1 = Aws::S3::Key.create(bucket, @key1)
96
+ key1.refresh
97
+ assert key1.exists?
98
+ assert key1.put(RIGHT_OBJECT_TEXT, "public-read")
99
+ # get its data
100
+ assert_equal RIGHT_OBJECT_TEXT, key1.get
101
+ # drop key
102
+ assert key1.delete
103
+ assert !key1.exists?
104
+ end
105
+
106
+ def test_36_set_amazon_problems
107
+ original_problems = Aws::S3Interface.amazon_problems
108
+ assert(original_problems.length > 0)
109
+ Aws::S3Interface.amazon_problems= original_problems << "A New Problem"
110
+ new_problems = Aws::S3Interface.amazon_problems
111
+ assert_equal(new_problems, original_problems)
112
+
113
+ Aws::S3Interface.amazon_problems= nil
114
+ assert_nil(Aws::S3Interface.amazon_problems)
115
+ end
116
+
117
+ def test_37_access_logging
118
+ bucket = Aws::S3::Bucket.create(@s, @bucket, false)
119
+ targetbucket = Aws::S3::Bucket.create(@s, @bucket2, true)
120
+ # Take 'AllUsers' grantee
121
+ grantee = Aws::S3::Grantee.new(targetbucket, 'http://acs.amazonaws.com/groups/s3/LogDelivery')
122
+
123
+ assert grantee.grant(['READ_ACP', 'WRITE'])
124
+
125
+ assert bucket.enable_logging(:targetbucket => targetbucket, :targetprefix => "loggylogs/")
126
+
127
+ assert_equal(bucket.logging_info, {:enabled => true, :targetbucket => @bucket2, :targetprefix => "loggylogs/"})
128
+
129
+ assert bucket.disable_logging
130
+
131
+ # check 'Drop' method
132
+ assert grantee.drop
133
+
134
+ # Delete bucket
135
+ bucket.delete(true)
136
+ targetbucket.delete(true)
137
+ end
138
+
139
+ end
@@ -202,5 +202,24 @@ class TestSqs < Test::Unit::TestCase
202
202
  newsqs = Aws::SqsInterface.new(TestCredentials.aws_access_key_id, TestCredentials.aws_secret_access_key, {:multi_thread => true})
203
203
  assert(newsqs.multi_thread)
204
204
  end
205
+
206
+ # Note that just like the other tests, this one does not clean up
207
+ # after itself (queue does not get deleted)
208
+ def test_29_change_message_visibility
209
+ queue = Aws::Sqs::Queue.create(@s, "#{@queue_name}_29", true, 10)
210
+ # Be sure the queue is empty. Useful when rerunning the tests.
211
+ queue.clear
212
+ queue.send_message(RIGHT_MESSAGE_TEXT)
213
+ m = nil
214
+ # Wait for the message to be delivered
215
+ while m.nil?
216
+ m = queue.receive
217
+ end
218
+ m.visibility = 60
219
+ sleep(20)
220
+ # Had the visibility timeout been not updated, the message would be available
221
+ # at this point.
222
+ assert queue.receive.nil?
223
+ end
205
224
 
206
225
  end
metadata CHANGED
@@ -4,9 +4,9 @@ version: !ruby/object:Gem::Version
4
4
  prerelease: false
5
5
  segments:
6
6
  - 2
7
- - 3
8
- - 34
9
- version: 2.3.34
7
+ - 4
8
+ - 0
9
+ version: 2.4.0
10
10
  platform: ruby
11
11
  authors:
12
12
  - Travis Reeder
@@ -16,7 +16,7 @@ autorequire:
16
16
  bindir: bin
17
17
  cert_chain: []
18
18
 
19
- date: 2010-12-22 00:00:00 -08:00
19
+ date: 2011-01-13 00:00:00 -08:00
20
20
  default_executable:
21
21
  dependencies:
22
22
  - !ruby/object:Gem::Dependency
@@ -71,7 +71,7 @@ dependencies:
71
71
  version: "0"
72
72
  type: :runtime
73
73
  version_requirements: *id004
74
- description: AWS Ruby Library for interfacing with Amazon Web Services.
74
+ description: AWS Ruby Library for interfacing with Amazon Web Services including EC2, S3, SQS, SimpleDB and most of their other services as well. By http://www.appoxy.com
75
75
  email: travis@appoxy.com
76
76
  executables: []
77
77
 
@@ -80,24 +80,30 @@ extensions: []
80
80
  extra_rdoc_files:
81
81
  - README.markdown
82
82
  files:
83
- - lib/acf/right_acf_interface.rb
83
+ - lib/acf/acf_interface.rb
84
84
  - lib/aws.rb
85
85
  - lib/awsbase/aws_response_array.rb
86
+ - lib/awsbase/awsbase.rb
86
87
  - lib/awsbase/benchmark_fix.rb
87
- - lib/awsbase/right_awsbase.rb
88
+ - lib/awsbase/errors.rb
89
+ - lib/awsbase/parsers.rb
88
90
  - lib/awsbase/support.rb
89
- - lib/ec2/right_ec2.rb
90
- - lib/ec2/right_mon_interface.rb
91
+ - lib/awsbase/utils.rb
92
+ - lib/ec2/ec2.rb
93
+ - lib/ec2/mon_interface.rb
91
94
  - lib/elb/elb_interface.rb
92
95
  - lib/iam/iam.rb
93
96
  - lib/rds/rds.rb
94
97
  - lib/right_aws.rb
95
- - lib/s3/right_s3.rb
96
- - lib/s3/right_s3_interface.rb
98
+ - lib/s3/bucket.rb
99
+ - lib/s3/grantee.rb
100
+ - lib/s3/key.rb
101
+ - lib/s3/s3.rb
102
+ - lib/s3/s3_interface.rb
97
103
  - lib/sdb/active_sdb.rb
98
- - lib/sdb/right_sdb_interface.rb
99
- - lib/sqs/right_sqs.rb
100
- - lib/sqs/right_sqs_interface.rb
104
+ - lib/sdb/sdb_interface.rb
105
+ - lib/sqs/sqs.rb
106
+ - lib/sqs/sqs_interface.rb
101
107
  - README.markdown
102
108
  - test/acf/test_acf.rb
103
109
  - test/acf/test_helper.rb
@@ -108,8 +114,11 @@ files:
108
114
  - test/http_connection.rb
109
115
  - test/iam/test_iam.rb
110
116
  - test/rds/test_rds.rb
117
+ - test/s3/s3_test_base.rb
111
118
  - test/s3/test_helper.rb
112
119
  - test/s3/test_s3.rb
120
+ - test/s3/test_s3_class.rb
121
+ - test/s3/test_s3_rights.rb
113
122
  - test/s3/test_s3_stubbed.rb
114
123
  - test/sdb/test_active_sdb.rb
115
124
  - test/sdb/test_helper.rb
@@ -149,7 +158,7 @@ rubyforge_project:
149
158
  rubygems_version: 1.3.7
150
159
  signing_key:
151
160
  specification_version: 3
152
- summary: AWS Ruby Library for interfacing with Amazon Web Services.
161
+ summary: AWS Ruby Library for interfacing with Amazon Web Services. By http://www.appoxy.com
153
162
  test_files:
154
163
  - test/acf/test_acf.rb
155
164
  - test/acf/test_helper.rb
@@ -160,8 +169,11 @@ test_files:
160
169
  - test/http_connection.rb
161
170
  - test/iam/test_iam.rb
162
171
  - test/rds/test_rds.rb
172
+ - test/s3/s3_test_base.rb
163
173
  - test/s3/test_helper.rb
164
174
  - test/s3/test_s3.rb
175
+ - test/s3/test_s3_class.rb
176
+ - test/s3/test_s3_rights.rb
165
177
  - test/s3/test_s3_stubbed.rb
166
178
  - test/sdb/test_active_sdb.rb
167
179
  - test/sdb/test_helper.rb
@@ -1,1254 +0,0 @@
1
- #
2
- # Copyright (c) 2007-2008 RightScale Inc
3
- #
4
- # Permission is hereby granted, free of charge, to any person obtaining
5
- # a copy of this software and associated documentation files (the
6
- # "Software"), to deal in the Software without restriction, including
7
- # without limitation the rights to use, copy, modify, merge, publish,
8
- # distribute, sublicense, and/or sell copies of the Software, and to
9
- # permit persons to whom the Software is furnished to do so, subject to
10
- # the following conditions:
11
- #
12
- # The above copyright notice and this permission notice shall be
13
- # included in all copies or substantial portions of the Software.
14
- #
15
- # THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
16
- # EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
17
- # MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
18
- # NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
19
- # LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
20
- # OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
21
- # WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
22
- #
23
-
24
- # Test
25
- module Aws
26
- require 'digest/md5'
27
- require 'pp'
28
- require 'cgi'
29
- require 'uri'
30
- require 'xmlsimple'
31
- require 'active_support/core_ext'
32
-
33
- class AwsUtils #:nodoc:
34
- @@digest1 = OpenSSL::Digest::Digest.new("sha1")
35
- @@digest256 = nil
36
- if OpenSSL::OPENSSL_VERSION_NUMBER > 0x00908000
37
- @@digest256 = OpenSSL::Digest::Digest.new("sha256") rescue nil # Some installation may not support sha256
38
- end
39
-
40
- def self.sign(aws_secret_access_key, auth_string)
41
- Base64.encode64(OpenSSL::HMAC.digest(@@digest1, aws_secret_access_key, auth_string)).strip
42
- end
43
-
44
-
45
- # Set a timestamp and a signature version
46
- def self.fix_service_params(service_hash, signature)
47
- service_hash["Timestamp"] ||= Time.now.utc.strftime("%Y-%m-%dT%H:%M:%S.000Z") unless service_hash["Expires"]
48
- service_hash["SignatureVersion"] = signature
49
- service_hash
50
- end
51
-
52
- # Signature Version 0
53
- # A deprecated guy (should work till septemper 2009)
54
- def self.sign_request_v0(aws_secret_access_key, service_hash)
55
- fix_service_params(service_hash, '0')
56
- string_to_sign = "#{service_hash['Action']}#{service_hash['Timestamp'] || service_hash['Expires']}"
57
- service_hash['Signature'] = AwsUtils::sign(aws_secret_access_key, string_to_sign)
58
- service_hash.to_a.collect { |key, val| "#{amz_escape(key)}=#{amz_escape(val.to_s)}" }.join("&")
59
- end
60
-
61
- # Signature Version 1
62
- # Another deprecated guy (should work till septemper 2009)
63
- def self.sign_request_v1(aws_secret_access_key, service_hash)
64
- fix_service_params(service_hash, '1')
65
- string_to_sign = service_hash.sort { |a, b| (a[0].to_s.downcase)<=>(b[0].to_s.downcase) }.to_s
66
- service_hash['Signature'] = AwsUtils::sign(aws_secret_access_key, string_to_sign)
67
- service_hash.to_a.collect { |key, val| "#{amz_escape(key)}=#{amz_escape(val.to_s)}" }.join("&")
68
- end
69
-
70
- # Signature Version 2
71
- # EC2, SQS and SDB requests must be signed by this guy.
72
- # See: http://docs.amazonwebservices.com/AmazonSimpleDB/2007-11-07/DeveloperGuide/index.html?REST_RESTAuth.html
73
- # http://developer.amazonwebservices.com/connect/entry.jspa?externalID=1928
74
- def self.sign_request_v2(aws_secret_access_key, service_hash, http_verb, host, uri)
75
- fix_service_params(service_hash, '2')
76
- # select a signing method (make an old openssl working with sha1)
77
- # make 'HmacSHA256' to be a default one
78
- service_hash['SignatureMethod'] = 'HmacSHA256' unless ['HmacSHA256', 'HmacSHA1'].include?(service_hash['SignatureMethod'])
79
- service_hash['SignatureMethod'] = 'HmacSHA1' unless @@digest256
80
- # select a digest
81
- digest = (service_hash['SignatureMethod'] == 'HmacSHA256' ? @@digest256 : @@digest1)
82
- # form string to sign
83
- canonical_string = service_hash.keys.sort.map do |key|
84
- "#{amz_escape(key)}=#{amz_escape(service_hash[key])}"
85
- end.join('&')
86
- string_to_sign = "#{http_verb.to_s.upcase}\n#{host.downcase}\n#{uri}\n#{canonical_string}"
87
- # sign the string
88
- signature = escape_sig(Base64.encode64(OpenSSL::HMAC.digest(digest, aws_secret_access_key, string_to_sign)).strip)
89
- ret = "#{canonical_string}&Signature=#{signature}"
90
- # puts 'full=' + ret.inspect
91
- ret
92
- end
93
-
94
- HEX = [
95
- "%00", "%01", "%02", "%03", "%04", "%05", "%06", "%07",
96
- "%08", "%09", "%0A", "%0B", "%0C", "%0D", "%0E", "%0F",
97
- "%10", "%11", "%12", "%13", "%14", "%15", "%16", "%17",
98
- "%18", "%19", "%1A", "%1B", "%1C", "%1D", "%1E", "%1F",
99
- "%20", "%21", "%22", "%23", "%24", "%25", "%26", "%27",
100
- "%28", "%29", "%2A", "%2B", "%2C", "%2D", "%2E", "%2F",
101
- "%30", "%31", "%32", "%33", "%34", "%35", "%36", "%37",
102
- "%38", "%39", "%3A", "%3B", "%3C", "%3D", "%3E", "%3F",
103
- "%40", "%41", "%42", "%43", "%44", "%45", "%46", "%47",
104
- "%48", "%49", "%4A", "%4B", "%4C", "%4D", "%4E", "%4F",
105
- "%50", "%51", "%52", "%53", "%54", "%55", "%56", "%57",
106
- "%58", "%59", "%5A", "%5B", "%5C", "%5D", "%5E", "%5F",
107
- "%60", "%61", "%62", "%63", "%64", "%65", "%66", "%67",
108
- "%68", "%69", "%6A", "%6B", "%6C", "%6D", "%6E", "%6F",
109
- "%70", "%71", "%72", "%73", "%74", "%75", "%76", "%77",
110
- "%78", "%79", "%7A", "%7B", "%7C", "%7D", "%7E", "%7F",
111
- "%80", "%81", "%82", "%83", "%84", "%85", "%86", "%87",
112
- "%88", "%89", "%8A", "%8B", "%8C", "%8D", "%8E", "%8F",
113
- "%90", "%91", "%92", "%93", "%94", "%95", "%96", "%97",
114
- "%98", "%99", "%9A", "%9B", "%9C", "%9D", "%9E", "%9F",
115
- "%A0", "%A1", "%A2", "%A3", "%A4", "%A5", "%A6", "%A7",
116
- "%A8", "%A9", "%AA", "%AB", "%AC", "%AD", "%AE", "%AF",
117
- "%B0", "%B1", "%B2", "%B3", "%B4", "%B5", "%B6", "%B7",
118
- "%B8", "%B9", "%BA", "%BB", "%BC", "%BD", "%BE", "%BF",
119
- "%C0", "%C1", "%C2", "%C3", "%C4", "%C5", "%C6", "%C7",
120
- "%C8", "%C9", "%CA", "%CB", "%CC", "%CD", "%CE", "%CF",
121
- "%D0", "%D1", "%D2", "%D3", "%D4", "%D5", "%D6", "%D7",
122
- "%D8", "%D9", "%DA", "%DB", "%DC", "%DD", "%DE", "%DF",
123
- "%E0", "%E1", "%E2", "%E3", "%E4", "%E5", "%E6", "%E7",
124
- "%E8", "%E9", "%EA", "%EB", "%EC", "%ED", "%EE", "%EF",
125
- "%F0", "%F1", "%F2", "%F3", "%F4", "%F5", "%F6", "%F7",
126
- "%F8", "%F9", "%FA", "%FB", "%FC", "%FD", "%FE", "%FF"
127
- ]
128
- TO_REMEMBER = 'AZaz09 -_.!~*\'()'
129
- ASCII = {} # {'A'=>65, 'Z'=>90, 'a'=>97, 'z'=>122, '0'=>48, '9'=>57, ' '=>32, '-'=>45, '_'=>95, '.'=>}
130
- TO_REMEMBER.each_char do |c| #unpack("c*").each do |c|
131
- ASCII[c] = c.unpack("c")[0]
132
- end
133
- # puts 'ascii=' + ASCII.inspect
134
-
135
- # Escape a string accordingly Amazon rulles
136
- # http://docs.amazonwebservices.com/AmazonSimpleDB/2007-11-07/DeveloperGuide/index.html?REST_RESTAuth.html
137
- def self.amz_escape(param)
138
-
139
- param = param.to_s
140
- # param = param.force_encoding("UTF-8")
141
-
142
- e = "x" # escape2(param.to_s)
143
- # puts 'ESCAPED=' + e.inspect
144
-
145
-
146
- #return CGI.escape(param.to_s).gsub("%7E", "~").gsub("+", "%20") # from: http://umlaut.rubyforge.org/svn/trunk/lib/aws_product_sign.rb
147
-
148
- #param.to_s.gsub(/([^a-zA-Z0-9._~-]+)/n) do
149
- # '%' + $1.unpack('H2' * $1.size).join('%').upcase
150
- #end
151
-
152
- # puts 'e in=' + e.inspect
153
- # converter = Iconv.new('ASCII', 'UTF-8')
154
- # e = converter.iconv(e) #.unpack('U*').select{ |cp| cp < 127 }.pack('U*')
155
- # puts 'e out=' + e.inspect
156
-
157
- e2 = CGI.escape(param)
158
- e2 = e2.gsub("%7E", "~")
159
- e2 = e2.gsub("+", "%20")
160
- e2 = e2.gsub("*", "%2A")
161
-
162
- # puts 'E2=' + e2.inspect
163
- # puts e == e2.to_s
164
-
165
- e2
166
-
167
- end
168
-
169
- def self.escape2(s)
170
- # home grown
171
- ret = ""
172
- s.unpack("U*") do |ch|
173
- # puts 'ch=' + ch.inspect
174
- if ASCII['A'] <= ch && ch <= ASCII['Z'] # A to Z
175
- ret << ch
176
- elsif ASCII['a'] <= ch && ch <= ASCII['z'] # a to z
177
- ret << ch
178
- elsif ASCII['0'] <= ch && ch <= ASCII['9'] # 0 to 9
179
- ret << ch
180
- elsif ch == ASCII[' '] # space
181
- ret << "%20" # "+"
182
- elsif ch == ASCII['-'] || ch == ASCII['_'] || ch == ASCII['.'] || ch == ASCII['~']
183
- ret << ch
184
- elsif ch <= 0x007f # other ascii
185
- ret << HEX[ch]
186
- elsif ch <= 0x07FF # non-ascii
187
- ret << HEX[0xc0 | (ch >> 6)]
188
- ret << HEX[0x80 | (ch & 0x3F)]
189
- else
190
- ret << HEX[0xe0 | (ch >> 12)]
191
- ret << HEX[0x80 | ((ch >> 6) & 0x3F)]
192
- ret << HEX[0x80 | (ch & 0x3F)]
193
- end
194
-
195
- end
196
- ret
197
-
198
- end
199
-
200
- def self.escape_sig(raw)
201
- e = CGI.escape(raw)
202
- end
203
-
204
- # From Amazon's SQS Dev Guide, a brief description of how to escape:
205
- # "URL encode the computed signature and other query parameters as specified in
206
- # RFC1738, section 2.2. In addition, because the + character is interpreted as a blank space
207
- # by Sun Java classes that perform URL decoding, make sure to encode the + character
208
- # although it is not required by RFC1738."
209
- # Avoid using CGI::escape to escape URIs.
210
- # CGI::escape will escape characters in the protocol, host, and port
211
- # sections of the URI. Only target chars in the query
212
- # string should be escaped.
213
- def self.URLencode(raw)
214
- e = URI.escape(raw)
215
- e.gsub(/\+/, "%2b")
216
- end
217
-
218
-
219
- def self.allow_only(allowed_keys, params)
220
- bogus_args = []
221
- params.keys.each { |p| bogus_args.push(p) unless allowed_keys.include?(p) }
222
- raise AwsError.new("The following arguments were given but are not legal for the function call #{caller_method}: #{bogus_args.inspect}") if bogus_args.length > 0
223
- end
224
-
225
- def self.mandatory_arguments(required_args, params)
226
- rargs = required_args.dup
227
- params.keys.each { |p| rargs.delete(p) }
228
- raise AwsError.new("The following mandatory arguments were not provided to #{caller_method}: #{rargs.inspect}") if rargs.length > 0
229
- end
230
-
231
- def self.caller_method
232
- caller[1]=~/`(.*?)'/
233
- $1
234
- end
235
-
236
- end
237
-
238
- class AwsBenchmarkingBlock #:nodoc:
239
- attr_accessor :xml, :service
240
-
241
- def initialize
242
- # Benchmark::Tms instance for service (Ec2, S3, or SQS) access benchmarking.
243
- @service = Benchmark::Tms.new()
244
- # Benchmark::Tms instance for XML parsing benchmarking.
245
- @xml = Benchmark::Tms.new()
246
- end
247
- end
248
-
249
- class AwsNoChange < RuntimeError
250
- end
251
-
252
- class AwsBase
253
-
254
- # Amazon HTTP Error handling
255
-
256
- # Text, if found in an error message returned by AWS, indicates that this may be a transient
257
- # error. Transient errors are automatically retried with exponential back-off.
258
- AMAZON_PROBLEMS = ['internal service error',
259
- 'is currently unavailable',
260
- 'no response from',
261
- 'Please try again',
262
- 'InternalError',
263
- 'ServiceUnavailable', #from SQS docs
264
- 'Unavailable',
265
- 'This application is not currently available',
266
- 'InsufficientInstanceCapacity'
267
- ]
268
- @@amazon_problems = AMAZON_PROBLEMS
269
- # Returns a list of Amazon service responses which are known to be transient problems.
270
- # We have to re-request if we get any of them, because the problem will probably disappear.
271
- # By default this method returns the same value as the AMAZON_PROBLEMS const.
272
- def self.amazon_problems
273
- @@amazon_problems
274
- end
275
-
276
- # Sets the list of Amazon side problems. Use in conjunction with the
277
- # getter to append problems.
278
- def self.amazon_problems=(problems_list)
279
- @@amazon_problems = problems_list
280
- end
281
-
282
- end
283
-
284
- module AwsBaseInterface
285
- DEFAULT_SIGNATURE_VERSION = '2'
286
-
287
- @@caching = false
288
-
289
- def self.caching
290
- @@caching
291
- end
292
-
293
- def self.caching=(caching)
294
- @@caching = caching
295
- end
296
-
297
- # Current aws_access_key_id
298
- attr_reader :aws_access_key_id
299
- # Last HTTP request object
300
- attr_reader :last_request
301
- # Last HTTP response object
302
- attr_reader :last_response
303
- # Last AWS errors list (used by AWSErrorHandler)
304
- attr_accessor :last_errors
305
- # Last AWS request id (used by AWSErrorHandler)
306
- attr_accessor :last_request_id
307
- # Logger object
308
- attr_accessor :logger
309
- # Initial params hash
310
- attr_accessor :params
311
- # RightHttpConnection instance
312
- attr_reader :connection
313
- # Cache
314
- attr_reader :cache
315
- # Signature version (all services except s3)
316
- attr_reader :signature_version
317
-
318
- def init(service_info, aws_access_key_id, aws_secret_access_key, params={}) #:nodoc:
319
- @params = params
320
- raise AwsError.new("AWS access keys are required to operate on #{service_info[:name]}") \
321
- if aws_access_key_id.blank? || aws_secret_access_key.blank?
322
- @aws_access_key_id = aws_access_key_id
323
- @aws_secret_access_key = aws_secret_access_key
324
- # if the endpoint was explicitly defined - then use it
325
- if @params[:endpoint_url]
326
- @params[:server] = URI.parse(@params[:endpoint_url]).host
327
- @params[:port] = URI.parse(@params[:endpoint_url]).port
328
- @params[:service] = URI.parse(@params[:endpoint_url]).path
329
- @params[:protocol] = URI.parse(@params[:endpoint_url]).scheme
330
- @params[:region] = nil
331
- else
332
- @params[:server] ||= service_info[:default_host]
333
- @params[:server] = "#{@params[:region]}.#{@params[:server]}" if @params[:region]
334
- @params[:port] ||= service_info[:default_port]
335
- @params[:service] ||= service_info[:default_service]
336
- @params[:protocol] ||= service_info[:default_protocol]
337
- @params[:api_version] ||= service_info[:api_version]
338
- end
339
- if !@params[:multi_thread].nil? && @params[:connection_mode].nil? # user defined this
340
- @params[:connection_mode] = @params[:multi_thread] ? :per_thread : :single
341
- end
342
- # @params[:multi_thread] ||= defined?(AWS_DAEMON)
343
- @params[:connection_mode] ||= :default
344
- @params[:connection_mode] = :per_request if @params[:connection_mode] == :default
345
- @logger = @params[:logger]
346
- @logger = Rails.logger if !@logger && defined?(Rails) && defined?(Rails.logger)
347
- @logger = ::Rails.logger if !@logger && defined?(::Rails.logger)
348
- @logger = Logger.new(STDOUT) if !@logger
349
- @logger.info "New #{self.class.name} using #{@params[:connection_mode].to_s}-connection mode"
350
- @error_handler = nil
351
- @cache = {}
352
- @signature_version = (params[:signature_version] || DEFAULT_SIGNATURE_VERSION).to_s
353
- end
354
-
355
- def signed_service_params(aws_secret_access_key, service_hash, http_verb=nil, host=nil, service=nil)
356
- case signature_version.to_s
357
- when '0' then
358
- AwsUtils::sign_request_v0(aws_secret_access_key, service_hash)
359
- when '1' then
360
- AwsUtils::sign_request_v1(aws_secret_access_key, service_hash)
361
- when '2' then
362
- AwsUtils::sign_request_v2(aws_secret_access_key, service_hash, http_verb, host, service)
363
- else
364
- raise AwsError.new("Unknown signature version (#{signature_version.to_s}) requested")
365
- end
366
- end
367
-
368
-
369
- def generate_request(action, params={})
370
- generate_request2(@aws_access_key_id, @aws_secret_access_key, action, @params[:api_version], @params, params)
371
- end
372
-
373
- # FROM SDB
374
- def generate_request2(aws_access_key, aws_secret_key, action, api_version, lib_params, user_params={}, options={}) #:nodoc:
375
- # remove empty params from request
376
- user_params.delete_if { |key, value| value.nil? }
377
- # user_params.each_pair do |k,v|
378
- # user_params[k] = v.force_encoding("UTF-8")
379
- # end
380
- #params_string = params.to_a.collect{|key,val| key + "=#{CGI::escape(val.to_s)}" }.join("&")
381
- # prepare service data
382
- service = lib_params[:service]
383
- # puts 'service=' + service.to_s
384
- service_hash = {"Action" => action,
385
- "AWSAccessKeyId" => aws_access_key}
386
- service_hash.update("Version" => api_version) if api_version
387
- service_hash.update(user_params)
388
- service_params = signed_service_params(aws_secret_key, service_hash, :get, lib_params[:server], lib_params[:service])
389
- #
390
- # use POST method if the length of the query string is too large
391
- # see http://docs.amazonwebservices.com/AmazonSimpleDB/2007-11-07/DeveloperGuide/MakingRESTRequests.html
392
- if service_params.size > 2000
393
- if signature_version == '2'
394
- # resign the request because HTTP verb is included into signature
395
- service_params = signed_service_params(aws_secret_key, service_hash, :post, lib_params[:server], service)
396
- end
397
- request = Net::HTTP::Post.new(service)
398
- request.body = service_params
399
- request['Content-Type'] = 'application/x-www-form-urlencoded; charset=utf-8'
400
- else
401
- request = Net::HTTP::Get.new("#{service}?#{service_params}")
402
- end
403
-
404
- #puts "\n\n --------------- QUERY REQUEST TO AWS -------------- \n\n"
405
- #puts "#{@params[:service]}?#{service_params}\n\n"
406
-
407
- # prepare output hash
408
- {:request => request,
409
- :server => lib_params[:server],
410
- :port => lib_params[:port],
411
- :protocol => lib_params[:protocol]}
412
- end
413
-
414
- def get_conn(connection_name, lib_params, logger)
415
- # thread = lib_params[:multi_thread] ? Thread.current : Thread.main
416
- # thread[connection_name] ||= Rightscale::HttpConnection.new(:exception => Aws::AwsError, :logger => logger)
417
- # conn = thread[connection_name]
418
- # return conn
419
- http_conn = nil
420
- conn_mode = lib_params[:connection_mode]
421
-
422
- # Slice all parameters accepted by Rightscale::HttpConnection#new
423
- params = lib_params.slice(
424
- :user_agent, :ca_file, :http_connection_retry_count, :http_connection_open_timeout,
425
- :http_connection_read_timeout, :http_connection_retry_delay
426
- )
427
- params.merge!(:exception => AwsError, :logger => logger)
428
-
429
- if conn_mode == :per_request
430
- http_conn = Rightscale::HttpConnection.new(params)
431
-
432
- elsif conn_mode == :per_thread || conn_mode == :single
433
- thread = conn_mode == :per_thread ? Thread.current : Thread.main
434
- thread[connection_name] ||= Rightscale::HttpConnection.new(params)
435
- http_conn = thread[connection_name]
436
- # ret = request_info_impl(http_conn, bench, request, parser, &block)
437
- end
438
- return http_conn
439
-
440
- end
441
-
442
- def close_conn(conn_name)
443
- conn_mode = @params[:connection_mode]
444
- if conn_mode == :per_thread || conn_mode == :single
445
- thread = conn_mode == :per_thread ? Thread.current : Thread.main
446
- if !thread[conn_name].nil?
447
- thread[conn_name].finish
448
- thread[conn_name] = nil
449
- end
450
- end
451
- end
452
-
453
- #
454
- # def request_info2(request, parser, lib_params, connection_name, logger, bench)
455
- # t = get_conn(connection_name, lib_params, logger)
456
- # request_info_impl(t, bench, request, parser)
457
- # end
458
-
459
- # Sends request to Amazon and parses the response
460
- # Raises AwsError if any banana happened
461
- def request_info2(request, parser, lib_params, connection_name, logger, bench, options={}, &block) #:nodoc:
462
- ret = nil
463
- # puts 'OPTIONS=' + options.inspect
464
- http_conn = get_conn(connection_name, lib_params, logger)
465
- begin
466
- retry_count = 1
467
- count = 0
468
- while count <= retry_count
469
- puts 'RETRYING QUERY due to QueryTimeout...' if count > 0
470
- begin
471
- ret = request_info_impl(http_conn, bench, request, parser, options, &block)
472
- break
473
- rescue Aws::AwsError => ex
474
- if !ex.include?(/QueryTimeout/) || count == retry_count
475
- raise ex
476
- end
477
- end
478
- count += 1
479
- end
480
- ensure
481
- http_conn.finish if http_conn && lib_params[:connection_mode] == :per_request
482
- end
483
- ret
484
- end
485
-
486
-
487
- # This is the direction we should head instead of writing our own parsers for everything, much simpler
488
- # params:
489
- # - :group_tags => hash of indirection to eliminate, see: http://xml-simple.rubyforge.org/
490
- # - :force_array => true for all or an array of tag names to force
491
- # - :pull_out_array => an array of levels to dig into when generating return value (see rds.rb for example)
492
- def request_info_xml_simple(connection_name, lib_params, request, logger, params = {})
493
-
494
- @connection = get_conn(connection_name, lib_params, logger)
495
- begin
496
- @last_request = request[:request]
497
- @last_response = nil
498
-
499
- response = @connection.request(request)
500
- # puts "response=" + response.body
501
- # benchblock.service.add!{ response = @connection.request(request) }
502
- # check response for errors...
503
- @last_response = response
504
- if response.is_a?(Net::HTTPSuccess)
505
- @error_handler = nil
506
- # benchblock.xml.add! { parser.parse(response) }
507
- # return parser.result
508
- force_array = params[:force_array] || false
509
- # Force_array and group_tags don't work nice together so going to force array manually
510
- xml_simple_options = {"KeyToSymbol"=>false, 'ForceArray' => false}
511
- xml_simple_options["GroupTags"] = params[:group_tags] if params[:group_tags]
512
-
513
- # { 'GroupTags' => { 'searchpath' => 'dir' }
514
- # 'ForceArray' => %r(_list$)
515
- parsed = XmlSimple.xml_in(response.body, xml_simple_options)
516
- # todo: we may want to consider stripping off a couple of layers when doing this, for instance:
517
- # <DescribeDBInstancesResponse xmlns="http://rds.amazonaws.com/admin/2009-10-16/">
518
- # <DescribeDBInstancesResult>
519
- # <DBInstances>
520
- # <DBInstance>....
521
- # Strip it off and only return an array or hash of <DBInstance>'s (hash by identifier).
522
- # would have to be able to make the RequestId available somehow though, perhaps some special array subclass which included that?
523
- unless force_array.is_a? Array
524
- force_array = []
525
- end
526
- parsed = symbolize(parsed, force_array)
527
- # puts 'parsed=' + parsed.inspect
528
- if params[:pull_out_array]
529
- ret = Aws::AwsResponseArray.new(parsed[:response_metadata])
530
- level_hash = parsed
531
- params[:pull_out_array].each do |x|
532
- level_hash = level_hash[x]
533
- end
534
- if level_hash.is_a? Hash # When there's only one
535
- ret << level_hash
536
- else # should be array
537
- # puts 'level_hash=' + level_hash.inspect
538
- level_hash.each do |x|
539
- ret << x
540
- end
541
- end
542
- elsif params[:pull_out_single]
543
- # returns a single object
544
- ret = AwsResponseObjectHash.new(parsed[:response_metadata])
545
- level_hash = parsed
546
- params[:pull_out_single].each do |x|
547
- level_hash = level_hash[x]
548
- end
549
- ret.merge!(level_hash)
550
- else
551
- ret = parsed
552
- end
553
- return ret
554
-
555
- else
556
- @error_handler = AWSErrorHandler.new(self, nil, :errors_list => self.class.amazon_problems) unless @error_handler
557
- check_result = @error_handler.check(request)
558
- if check_result
559
- @error_handler = nil
560
- return check_result
561
- end
562
- request_text_data = "#{request[:server]}:#{request[:port]}#{request[:request].path}"
563
- raise AwsError2.new(@last_response.code, @last_request_id, request_text_data, @last_response.body)
564
- end
565
- ensure
566
- @connection.finish if @connection && lib_params[:connection_mode] == :per_request
567
- end
568
-
569
- end
570
-
571
- def symbolize(hash, force_array)
572
- ret = {}
573
- hash.keys.each do |key|
574
- val = hash[key]
575
- if val.is_a? Hash
576
- val = symbolize(val, force_array)
577
- if force_array.include? key
578
- val = [val]
579
- end
580
- elsif val.is_a? Array
581
- val = val.collect { |x| symbolize(x, force_array) }
582
- end
583
- ret[key.underscore.to_sym] = val
584
- end
585
- ret
586
- end
587
-
588
- # Returns +true+ if the describe_xxx responses are being cached
589
- def caching?
590
- @params.key?(:cache) ? @params[:cache] : @@caching
591
- end
592
-
593
- # Check if the aws function response hits the cache or not.
594
- # If the cache hits:
595
- # - raises an +AwsNoChange+ exception if +do_raise+ == +:raise+.
596
- # - returnes parsed response from the cache if it exists or +true+ otherwise.
597
- # If the cache miss or the caching is off then returns +false+.
598
- def cache_hits?(function, response, do_raise=:raise)
599
- result = false
600
- if caching?
601
- function = function.to_sym
602
- # get rid of requestId (this bad boy was added for API 2008-08-08+ and it is uniq for every response)
603
- response = response.sub(%r{<requestId>.+?</requestId>}, '')
604
- response_md5 =Digest::MD5.hexdigest(response).to_s
605
- # check for changes
606
- unless @cache[function] && @cache[function][:response_md5] == response_md5
607
- # well, the response is new, reset cache data
608
- update_cache(function, {:response_md5 => response_md5,
609
- :timestamp => Time.now,
610
- :hits => 0,
611
- :parsed => nil})
612
- else
613
- # aha, cache hits, update the data and throw an exception if needed
614
- @cache[function][:hits] += 1
615
- if do_raise == :raise
616
- raise(AwsNoChange, "Cache hit: #{function} response has not changed since "+
617
- "#{@cache[function][:timestamp].strftime('%Y-%m-%d %H:%M:%S')}, "+
618
- "hits: #{@cache[function][:hits]}.")
619
- else
620
- result = @cache[function][:parsed] || true
621
- end
622
- end
623
- end
624
- result
625
- end
626
-
627
- def update_cache(function, hash)
628
- (@cache[function.to_sym] ||= {}).merge!(hash) if caching?
629
- end
630
-
631
- def on_exception(options={:raise=>true, :log=>true}) # :nodoc:
632
- raise if $!.is_a?(AwsNoChange)
633
- AwsError::on_aws_exception(self, options)
634
- end
635
-
636
- # Return +true+ if this instance works in multi_thread mode and +false+ otherwise.
637
- def multi_thread
638
- @params[:multi_thread]
639
- end
640
-
641
-
642
- def request_info_impl(connection, benchblock, request, parser, options={}, &block) #:nodoc:
643
- @connection = connection
644
- @last_request = request[:request]
645
- @last_response = nil
646
- response =nil
647
- blockexception = nil
648
-
649
- # puts 'OPTIONS2=' + options.inspect
650
-
651
- if (block != nil)
652
- # TRB 9/17/07 Careful - because we are passing in blocks, we get a situation where
653
- # an exception may get thrown in the block body (which is high-level
654
- # code either here or in the application) but gets caught in the
655
- # low-level code of HttpConnection. The solution is not to let any
656
- # exception escape the block that we pass to HttpConnection::request.
657
- # Exceptions can originate from code directly in the block, or from user
658
- # code called in the other block which is passed to response.read_body.
659
- benchblock.service.add! do
660
- responsehdr = @connection.request(request) do |response|
661
- #########
662
- begin
663
- @last_response = response
664
- if response.is_a?(Net::HTTPSuccess)
665
- @error_handler = nil
666
- response.read_body(&block)
667
- else
668
- @error_handler = AWSErrorHandler.new(self, parser, :errors_list => self.class.amazon_problems) unless @error_handler
669
- check_result = @error_handler.check(request, options)
670
- if check_result
671
- @error_handler = nil
672
- return check_result
673
- end
674
- request_text_data = "#{request[:server]}:#{request[:port]}#{request[:request].path}"
675
- raise AwsError.new(@last_errors, @last_response.code, @last_request_id, request_text_data)
676
- end
677
- rescue Exception => e
678
- blockexception = e
679
- end
680
- end
681
- #########
682
-
683
- #OK, now we are out of the block passed to the lower level
684
- if (blockexception)
685
- raise blockexception
686
- end
687
- benchblock.xml.add! do
688
- parser.parse(responsehdr)
689
- end
690
- return parser.result
691
- end
692
- else
693
- benchblock.service.add! { response = @connection.request(request) }
694
- # check response for errors...
695
- @last_response = response
696
- if response.is_a?(Net::HTTPSuccess)
697
- @error_handler = nil
698
- benchblock.xml.add! { parser.parse(response) }
699
- return parser.result
700
- else
701
- @error_handler = AWSErrorHandler.new(self, parser, :errors_list => self.class.amazon_problems) unless @error_handler
702
- check_result = @error_handler.check(request, options)
703
- if check_result
704
- @error_handler = nil
705
- return check_result
706
- end
707
- request_text_data = "#{request[:server]}:#{request[:port]}#{request[:request].path}"
708
- raise AwsError.new(@last_errors, @last_response.code, @last_request_id, request_text_data)
709
- end
710
- end
711
- rescue
712
- @error_handler = nil
713
- raise
714
- end
715
-
716
- def request_cache_or_info(method, link, parser_class, benchblock, use_cache=true) #:nodoc:
717
- # We do not want to break the logic of parsing hence will use a dummy parser to process all the standard
718
- # steps (errors checking etc). The dummy parser does nothig - just returns back the params it received.
719
- # If the caching is enabled and hit then throw AwsNoChange.
720
- # P.S. caching works for the whole images list only! (when the list param is blank)
721
- # check cache
722
- response, params = request_info(link, RightDummyParser.new)
723
- cache_hits?(method.to_sym, response.body) if use_cache
724
- parser = parser_class.new(:logger => @logger)
725
- benchblock.xml.add! { parser.parse(response, params) }
726
- result = block_given? ? yield(parser) : parser.result
727
- # update parsed data
728
- update_cache(method.to_sym, :parsed => result) if use_cache
729
- result
730
- end
731
-
732
- # Returns Amazons request ID for the latest request
733
- def last_request_id
734
- @last_response && @last_response.body.to_s[%r{<requestId>(.+?)</requestId>}] && $1
735
- end
736
-
737
- def hash_params(prefix, list) #:nodoc:
738
- groups = {}
739
- list.each_index { |i| groups.update("#{prefix}.#{i+1}"=>list[i]) } if list
740
- return groups
741
- end
742
-
743
- end
744
-
745
-
746
- # Exception class to signal any Amazon errors. All errors occuring during calls to Amazon's
747
- # web services raise this type of error.
748
- # Attribute inherited by RuntimeError:
749
- # message - the text of the error, generally as returned by AWS in its XML response.
750
- class AwsError < RuntimeError
751
-
752
- # either an array of errors where each item is itself an array of [code, message]),
753
- # or an error string if the error was raised manually, as in <tt>AwsError.new('err_text')</tt>
754
- attr_reader :errors
755
-
756
- # Request id (if exists)
757
- attr_reader :request_id
758
-
759
- # Response HTTP error code
760
- attr_reader :http_code
761
-
762
- # Raw request text data to AWS
763
- attr_reader :request_data
764
-
765
- attr_reader :response
766
-
767
- def initialize(errors=nil, http_code=nil, request_id=nil, request_data=nil, response=nil)
768
- @errors = errors
769
- @request_id = request_id
770
- @http_code = http_code
771
- @request_data = request_data
772
- @response = response
773
- msg = @errors.is_a?(Array) ? @errors.map { |code, msg| "#{code}: #{msg}" }.join("; ") : @errors.to_s
774
- msg += "\nREQUEST=#{@request_data} " unless @request_data.nil?
775
- msg += "\nREQUEST ID=#{@request_id} " unless @request_id.nil?
776
- super(msg)
777
- end
778
-
779
- # Does any of the error messages include the regexp +pattern+?
780
- # Used to determine whether to retry request.
781
- def include?(pattern)
782
- if @errors.is_a?(Array)
783
- @errors.each { |code, msg| return true if code =~ pattern }
784
- else
785
- return true if @errors_str =~ pattern
786
- end
787
- false
788
- end
789
-
790
- # Generic handler for AwsErrors. +aws+ is the Aws::S3, Aws::EC2, or Aws::SQS
791
- # object that caused the exception (it must provide last_request and last_response). Supported
792
- # boolean options are:
793
- # * <tt>:log</tt> print a message into the log using aws.logger to access the Logger
794
- # * <tt>:puts</tt> do a "puts" of the error
795
- # * <tt>:raise</tt> re-raise the error after logging
796
- def self.on_aws_exception(aws, options={:raise=>true, :log=>true})
797
- # Only log & notify if not user error
798
- if !options[:raise] || system_error?($!)
799
- error_text = "#{$!.inspect}\n#{$@}.join('\n')}"
800
- puts error_text if options[:puts]
801
- # Log the error
802
- if options[:log]
803
- request = aws.last_request ? aws.last_request.path : '-none-'
804
- response = aws.last_response ? "#{aws.last_response.code} -- #{aws.last_response.message} -- #{aws.last_response.body}" : '-none-'
805
- @response = response
806
- aws.logger.error error_text
807
- aws.logger.error "Request was: #{request}"
808
- aws.logger.error "Response was: #{response}"
809
- end
810
- end
811
- raise if options[:raise] # re-raise an exception
812
- return nil
813
- end
814
-
815
- # True if e is an AWS system error, i.e. something that is for sure not the caller's fault.
816
- # Used to force logging.
817
- def self.system_error?(e)
818
- !e.is_a?(self) || e.message =~ /InternalError|InsufficientInstanceCapacity|Unavailable/
819
- end
820
-
821
- end
822
-
823
- # Simplified version
824
- class AwsError2 < RuntimeError
825
- # Request id (if exists)
826
- attr_reader :request_id
827
-
828
- # Response HTTP error code
829
- attr_reader :http_code
830
-
831
- # Raw request text data to AWS
832
- attr_reader :request_data
833
-
834
- attr_reader :response
835
-
836
- attr_reader :errors
837
-
838
- def initialize(http_code=nil, request_id=nil, request_data=nil, response=nil)
839
-
840
- @request_id = request_id
841
- @http_code = http_code
842
- @request_data = request_data
843
- @response = response
844
- # puts '@response=' + @response.inspect
845
-
846
- if @response
847
- ref = XmlSimple.xml_in(@response, {"ForceArray"=>false})
848
- # puts "refxml=" + ref.inspect
849
- msg = "#{ref['Error']['Code']}: #{ref['Error']['Message']}"
850
- else
851
- msg = "#{@http_code}: REQUEST(#{@request_data})"
852
- end
853
- msg += "\nREQUEST ID=#{@request_id} " unless @request_id.nil?
854
- super(msg)
855
- end
856
-
857
-
858
- end
859
-
860
-
861
- class AWSErrorHandler
862
- # 0-100 (%)
863
- DEFAULT_CLOSE_ON_4XX_PROBABILITY = 10
864
-
865
- @@reiteration_start_delay = 0.2
866
-
867
- def self.reiteration_start_delay
868
- @@reiteration_start_delay
869
- end
870
-
871
- def self.reiteration_start_delay=(reiteration_start_delay)
872
- @@reiteration_start_delay = reiteration_start_delay
873
- end
874
-
875
- @@reiteration_time = 5
876
-
877
- def self.reiteration_time
878
- @@reiteration_time
879
- end
880
-
881
- def self.reiteration_time=(reiteration_time)
882
- @@reiteration_time = reiteration_time
883
- end
884
-
885
- @@close_on_error = true
886
-
887
- def self.close_on_error
888
- @@close_on_error
889
- end
890
-
891
- def self.close_on_error=(close_on_error)
892
- @@close_on_error = close_on_error
893
- end
894
-
895
- @@close_on_4xx_probability = DEFAULT_CLOSE_ON_4XX_PROBABILITY
896
-
897
- def self.close_on_4xx_probability
898
- @@close_on_4xx_probability
899
- end
900
-
901
- def self.close_on_4xx_probability=(close_on_4xx_probability)
902
- @@close_on_4xx_probability = close_on_4xx_probability
903
- end
904
-
905
- # params:
906
- # :reiteration_time
907
- # :errors_list
908
- # :close_on_error = true | false
909
- # :close_on_4xx_probability = 1-100
910
- def initialize(aws, parser, params={}) #:nodoc:
911
- @aws = aws # Link to RightEc2 | RightSqs | RightS3 instance
912
- @parser = parser # parser to parse Amazon response
913
- @started_at = Time.now
914
- @stop_at = @started_at + (params[:reiteration_time] || @@reiteration_time)
915
- @errors_list = params[:errors_list] || []
916
- @reiteration_delay = @@reiteration_start_delay
917
- @retries = 0
918
- # close current HTTP(S) connection on 5xx, errors from list and 4xx errors
919
- @close_on_error = params[:close_on_error].nil? ? @@close_on_error : params[:close_on_error]
920
- @close_on_4xx_probability = params[:close_on_4xx_probability] || @@close_on_4xx_probability
921
- end
922
-
923
- # Returns false if
924
- def check(request, options={}) #:nodoc:
925
- result = false
926
- error_found = false
927
- redirect_detected = false
928
- error_match = nil
929
- last_errors_text = ''
930
- response = @aws.last_response
931
- # log error
932
- request_text_data = "#{request[:server]}:#{request[:port]}#{request[:request].path}"
933
- # is this a redirect?
934
- # yes!
935
- if response.is_a?(Net::HTTPRedirection)
936
- redirect_detected = true
937
- else
938
- # no, it's an error ...
939
- @aws.logger.warn("##### #{@aws.class.name} returned an error: #{response.code} #{response.message}\n#{response.body} #####")
940
- @aws.logger.warn("##### #{@aws.class.name} request: #{request_text_data} ####")
941
- end
942
- # Check response body: if it is an Amazon XML document or not:
943
- if redirect_detected || (response.body && response.body[/<\?xml/]) # ... it is a xml document
944
- @aws.class.bench_xml.add! do
945
- error_parser = RightErrorResponseParser.new
946
- error_parser.parse(response)
947
- @aws.last_errors = error_parser.errors
948
- @aws.last_request_id = error_parser.requestID
949
- last_errors_text = @aws.last_errors.flatten.join("\n")
950
- # on redirect :
951
- if redirect_detected
952
- location = response['location']
953
- # ... log information and ...
954
- @aws.logger.info("##### #{@aws.class.name} redirect requested: #{response.code} #{response.message} #####")
955
- @aws.logger.info("##### New location: #{location} #####")
956
- # ... fix the connection data
957
- request[:server] = URI.parse(location).host
958
- request[:protocol] = URI.parse(location).scheme
959
- request[:port] = URI.parse(location).port
960
- end
961
- end
962
- else # ... it is not a xml document(probably just a html page?)
963
- @aws.last_errors = [[response.code, "#{response.message} (#{request_text_data})"]]
964
- @aws.last_request_id = '-undefined-'
965
- last_errors_text = response.message
966
- end
967
- # now - check the error
968
- unless redirect_detected
969
- @errors_list.each do |error_to_find|
970
- if last_errors_text[/#{error_to_find}/i]
971
- error_found = true
972
- error_match = error_to_find
973
- @aws.logger.warn("##### Retry is needed, error pattern match: #{error_to_find} #####")
974
- break
975
- end
976
- end
977
- end
978
- # check the time has gone from the first error come
979
- if redirect_detected || error_found
980
- # Close the connection to the server and recreate a new one.
981
- # It may have a chance that one server is a semi-down and reconnection
982
- # will help us to connect to the other server
983
- if !redirect_detected && @close_on_error
984
- @aws.connection.finish "#{self.class.name}: error match to pattern '#{error_match}'"
985
- end
986
- # puts 'OPTIONS3=' + options.inspect
987
- if options[:retries].nil? || @retries < options[:retries]
988
- if (Time.now < @stop_at)
989
- @retries += 1
990
- unless redirect_detected
991
- @aws.logger.warn("##### Retry ##{@retries} is being performed. Sleeping for #{@reiteration_delay} sec. Whole time: #{Time.now-@started_at} sec ####")
992
- sleep @reiteration_delay
993
- @reiteration_delay *= 2
994
-
995
- # Always make sure that the fp is set to point to the beginning(?)
996
- # of the File/IO. TODO: it assumes that offset is 0, which is bad.
997
- if (request[:request].body_stream && request[:request].body_stream.respond_to?(:pos))
998
- begin
999
- request[:request].body_stream.pos = 0
1000
- rescue Exception => e
1001
- @logger.warn("Retry may fail due to unable to reset the file pointer" +
1002
- " -- #{self.class.name} : #{e.inspect}")
1003
- end
1004
- end
1005
- else
1006
- @aws.logger.info("##### Retry ##{@retries} is being performed due to a redirect. ####")
1007
- end
1008
- result = @aws.request_info(request, @parser, options)
1009
- else
1010
- @aws.logger.warn("##### Ooops, time is over... ####")
1011
- end
1012
- else
1013
- @aws.logger.info("##### Stopped retrying because retries=#{@retries} and max=#{options[:retries]} ####")
1014
- end
1015
- # aha, this is unhandled error:
1016
- elsif @close_on_error
1017
- # Is this a 5xx error ?
1018
- if @aws.last_response.code.to_s[/^5\d\d$/]
1019
- @aws.connection.finish "#{self.class.name}: code: #{@aws.last_response.code}: '#{@aws.last_response.message}'"
1020
- # Is this a 4xx error ?
1021
- elsif @aws.last_response.code.to_s[/^4\d\d$/] && @close_on_4xx_probability > rand(100)
1022
- @aws.connection.finish "#{self.class.name}: code: #{@aws.last_response.code}: '#{@aws.last_response.message}', " +
1023
- "probability: #{@close_on_4xx_probability}%"
1024
- end
1025
- end
1026
- result
1027
- end
1028
-
1029
- end
1030
-
1031
-
1032
- #-----------------------------------------------------------------
1033
-
1034
- class RightSaxParserCallback #:nodoc:
1035
- def self.include_callback
1036
- include XML::SaxParser::Callbacks
1037
- end
1038
-
1039
- def initialize(right_aws_parser)
1040
- @right_aws_parser = right_aws_parser
1041
- end
1042
-
1043
- def on_start_element(name, attr_hash)
1044
- @right_aws_parser.tag_start(name, attr_hash)
1045
- end
1046
-
1047
- def on_characters(chars)
1048
- @right_aws_parser.text(chars)
1049
- end
1050
-
1051
- def on_end_element(name)
1052
- @right_aws_parser.tag_end(name)
1053
- end
1054
-
1055
- def on_start_document;
1056
- end
1057
-
1058
- def on_comment(msg)
1059
- ;
1060
- end
1061
-
1062
- def on_processing_instruction(target, data)
1063
- ;
1064
- end
1065
-
1066
- def on_cdata_block(cdata)
1067
- ;
1068
- end
1069
-
1070
- def on_end_document;
1071
- end
1072
- end
1073
-
1074
- class AwsParser #:nodoc:
1075
- # default parsing library
1076
- DEFAULT_XML_LIBRARY = 'rexml'
1077
- # a list of supported parsers
1078
- @@supported_xml_libs = [DEFAULT_XML_LIBRARY, 'libxml']
1079
-
1080
- @@xml_lib = DEFAULT_XML_LIBRARY # xml library name: 'rexml' | 'libxml'
1081
- def self.xml_lib
1082
- @@xml_lib
1083
- end
1084
-
1085
- def self.xml_lib=(new_lib_name)
1086
- @@xml_lib = new_lib_name
1087
- end
1088
-
1089
- attr_accessor :result
1090
- attr_reader :xmlpath
1091
- attr_accessor :xml_lib
1092
-
1093
- def initialize(params={})
1094
- @xmlpath = ''
1095
- @result = false
1096
- @text = ''
1097
- @xml_lib = params[:xml_lib] || @@xml_lib
1098
- @logger = params[:logger]
1099
- reset
1100
- end
1101
-
1102
- def tag_start(name, attributes)
1103
- @text = ''
1104
- tagstart(name, attributes)
1105
- @xmlpath += @xmlpath.empty? ? name : "/#{name}"
1106
- end
1107
-
1108
- def tag_end(name)
1109
- if @xmlpath =~ /^(.*?)\/?#{name}$/
1110
- @xmlpath = $1
1111
- end
1112
- tagend(name)
1113
- end
1114
-
1115
- def text(text)
1116
- @text += text
1117
- tagtext(text)
1118
- end
1119
-
1120
- # Parser method.
1121
- # Params:
1122
- # xml_text - xml message text(String) or Net:HTTPxxx instance (response)
1123
- # params[:xml_lib] - library name: 'rexml' | 'libxml'
1124
- def parse(xml_text, params={})
1125
- # Get response body
1126
- unless xml_text.is_a?(String)
1127
- xml_text = xml_text.body.respond_to?(:force_encoding) ? xml_text.body.force_encoding("UTF-8") : xml_text.body
1128
- end
1129
-
1130
- @xml_lib = params[:xml_lib] || @xml_lib
1131
- # check that we had no problems with this library otherwise use default
1132
- @xml_lib = DEFAULT_XML_LIBRARY unless @@supported_xml_libs.include?(@xml_lib)
1133
- # load xml library
1134
- if @xml_lib=='libxml' && !defined?(XML::SaxParser)
1135
- begin
1136
- require 'xml/libxml'
1137
- # is it new ? - Setup SaxParserCallback
1138
- if XML::Parser::VERSION >= '0.5.1.0'
1139
- RightSaxParserCallback.include_callback
1140
- end
1141
- rescue LoadError => e
1142
- @@supported_xml_libs.delete(@xml_lib)
1143
- @xml_lib = DEFAULT_XML_LIBRARY
1144
- if @logger
1145
- @logger.error e.inspect
1146
- @logger.error e.backtrace
1147
- @logger.info "Can not load 'libxml' library. '#{DEFAULT_XML_LIBRARY}' is used for parsing."
1148
- end
1149
- end
1150
- end
1151
- # Parse the xml text
1152
- case @xml_lib
1153
- when 'libxml'
1154
- xml = XML::SaxParser.string(xml_text)
1155
- # check libxml-ruby version
1156
- if XML::Parser::VERSION >= '0.5.1.0'
1157
- xml.callbacks = RightSaxParserCallback.new(self)
1158
- else
1159
- xml.on_start_element { |name, attr_hash| self.tag_start(name, attr_hash) }
1160
- xml.on_characters { |text| self.text(text) }
1161
- xml.on_end_element { |name| self.tag_end(name) }
1162
- end
1163
- xml.parse
1164
- else
1165
- REXML::Document.parse_stream(xml_text, self)
1166
- end
1167
- end
1168
-
1169
- # Parser must have a lots of methods
1170
- # (see /usr/lib/ruby/1.8/rexml/parsers/streamparser.rb)
1171
- # We dont need most of them in AwsParser and method_missing helps us
1172
- # to skip their definition
1173
- def method_missing(method, *params)
1174
- # if the method is one of known - just skip it ...
1175
- return if [:comment, :attlistdecl, :notationdecl, :elementdecl,
1176
- :entitydecl, :cdata, :xmldecl, :attlistdecl, :instruction,
1177
- :doctype].include?(method)
1178
- # ... else - call super to raise an exception
1179
- super(method, params)
1180
- end
1181
-
1182
- # the functions to be overriden by children (if nessesery)
1183
- def reset;
1184
- end
1185
-
1186
- def tagstart(name, attributes)
1187
- ;
1188
- end
1189
-
1190
- def tagend(name)
1191
- ;
1192
- end
1193
-
1194
- def tagtext(text)
1195
- ;
1196
- end
1197
- end
1198
-
1199
- #-----------------------------------------------------------------
1200
- # PARSERS: Errors
1201
- #-----------------------------------------------------------------
1202
-
1203
- #<Error>
1204
- # <Code>TemporaryRedirect</Code>
1205
- # <Message>Please re-send this request to the specified temporary endpoint. Continue to use the original request endpoint for future requests.</Message>
1206
- # <RequestId>FD8D5026D1C5ABA3</RequestId>
1207
- # <Endpoint>bucket-for-k.s3-external-3.amazonaws.com</Endpoint>
1208
- # <HostId>ItJy8xPFPli1fq/JR3DzQd3iDvFCRqi1LTRmunEdM1Uf6ZtW2r2kfGPWhRE1vtaU</HostId>
1209
- # <Bucket>bucket-for-k</Bucket>
1210
- #</Error>
1211
-
1212
- class RightErrorResponseParser < AwsParser #:nodoc:
1213
- attr_accessor :errors # array of hashes: error/message
1214
- attr_accessor :requestID
1215
- # attr_accessor :endpoint, :host_id, :bucket
1216
- def tagend(name)
1217
- case name
1218
- when 'RequestID';
1219
- @requestID = @text
1220
- when 'Code';
1221
- @code = @text
1222
- when 'Message';
1223
- @message = @text
1224
- # when 'Endpoint' ; @endpoint = @text
1225
- # when 'HostId' ; @host_id = @text
1226
- # when 'Bucket' ; @bucket = @text
1227
- when 'Error';
1228
- @errors << [@code, @message]
1229
- end
1230
- end
1231
-
1232
- def reset
1233
- @errors = []
1234
- end
1235
- end
1236
-
1237
- # Dummy parser - does nothing
1238
- # Returns the original params back
1239
- class RightDummyParser # :nodoc:
1240
- attr_accessor :result
1241
-
1242
- def parse(response, params={})
1243
- @result = [response, params]
1244
- end
1245
- end
1246
-
1247
- class RightHttp2xxParser < AwsParser # :nodoc:
1248
- def parse(response)
1249
- @result = response.is_a?(Net::HTTPSuccess)
1250
- end
1251
- end
1252
-
1253
- end
1254
-