fmalamitsas-aws-s3 0.6.2.1254423625
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/CHANGELOG +107 -0
- data/COPYING +19 -0
- data/INSTALL +55 -0
- data/README.erb +58 -0
- data/Rakefile +334 -0
- data/TODO +26 -0
- data/aws-s3.gemspec +42 -0
- data/bin/s3sh +6 -0
- data/bin/setup.rb +10 -0
- data/lib/aws/s3.rb +60 -0
- data/lib/aws/s3/acl.rb +636 -0
- data/lib/aws/s3/authentication.rb +222 -0
- data/lib/aws/s3/base.rb +270 -0
- data/lib/aws/s3/bittorrent.rb +58 -0
- data/lib/aws/s3/bucket.rb +372 -0
- data/lib/aws/s3/connection.rb +288 -0
- data/lib/aws/s3/error.rb +69 -0
- data/lib/aws/s3/exceptions.rb +133 -0
- data/lib/aws/s3/extensions.rb +342 -0
- data/lib/aws/s3/logging.rb +317 -0
- data/lib/aws/s3/object.rb +626 -0
- data/lib/aws/s3/owner.rb +46 -0
- data/lib/aws/s3/parsing.rb +99 -0
- data/lib/aws/s3/response.rb +180 -0
- data/lib/aws/s3/service.rb +51 -0
- data/lib/aws/s3/version.rb +12 -0
- data/site/index.erb +41 -0
- data/site/public/images/box-and-gem.gif +0 -0
- data/site/public/images/favicon.ico +0 -0
- data/site/public/ruby.css +18 -0
- data/site/public/screen.css +99 -0
- data/support/faster-xml-simple/COPYING +18 -0
- data/support/faster-xml-simple/README +8 -0
- data/support/faster-xml-simple/Rakefile +54 -0
- data/support/faster-xml-simple/lib/faster_xml_simple.rb +190 -0
- data/support/faster-xml-simple/test/fixtures/test-1.rails.yml +4 -0
- data/support/faster-xml-simple/test/fixtures/test-1.xml +3 -0
- data/support/faster-xml-simple/test/fixtures/test-1.yml +4 -0
- data/support/faster-xml-simple/test/fixtures/test-2.rails.yml +6 -0
- data/support/faster-xml-simple/test/fixtures/test-2.xml +3 -0
- data/support/faster-xml-simple/test/fixtures/test-2.yml +6 -0
- data/support/faster-xml-simple/test/fixtures/test-3.rails.yml +6 -0
- data/support/faster-xml-simple/test/fixtures/test-3.xml +5 -0
- data/support/faster-xml-simple/test/fixtures/test-3.yml +6 -0
- data/support/faster-xml-simple/test/fixtures/test-4.rails.yml +5 -0
- data/support/faster-xml-simple/test/fixtures/test-4.xml +7 -0
- data/support/faster-xml-simple/test/fixtures/test-4.yml +5 -0
- data/support/faster-xml-simple/test/fixtures/test-5.rails.yml +8 -0
- data/support/faster-xml-simple/test/fixtures/test-5.xml +7 -0
- data/support/faster-xml-simple/test/fixtures/test-5.yml +8 -0
- data/support/faster-xml-simple/test/fixtures/test-6.rails.yml +43 -0
- data/support/faster-xml-simple/test/fixtures/test-6.xml +29 -0
- data/support/faster-xml-simple/test/fixtures/test-6.yml +41 -0
- data/support/faster-xml-simple/test/fixtures/test-7.rails.yml +23 -0
- data/support/faster-xml-simple/test/fixtures/test-7.xml +22 -0
- data/support/faster-xml-simple/test/fixtures/test-7.yml +22 -0
- data/support/faster-xml-simple/test/fixtures/test-8.rails.yml +14 -0
- data/support/faster-xml-simple/test/fixtures/test-8.xml +8 -0
- data/support/faster-xml-simple/test/fixtures/test-8.yml +11 -0
- data/support/faster-xml-simple/test/regression_test.rb +47 -0
- data/support/faster-xml-simple/test/test_helper.rb +17 -0
- data/support/faster-xml-simple/test/xml_simple_comparison_test.rb +46 -0
- data/support/rdoc/code_info.rb +211 -0
- data/test/acl_test.rb +254 -0
- data/test/authentication_test.rb +118 -0
- data/test/base_test.rb +136 -0
- data/test/bucket_test.rb +74 -0
- data/test/connection_test.rb +216 -0
- data/test/error_test.rb +70 -0
- data/test/extensions_test.rb +340 -0
- data/test/fixtures.rb +89 -0
- data/test/fixtures/buckets.yml +133 -0
- data/test/fixtures/errors.yml +34 -0
- data/test/fixtures/headers.yml +3 -0
- data/test/fixtures/logging.yml +15 -0
- data/test/fixtures/loglines.yml +5 -0
- data/test/fixtures/logs.yml +7 -0
- data/test/fixtures/policies.yml +16 -0
- data/test/logging_test.rb +89 -0
- data/test/mocks/fake_response.rb +26 -0
- data/test/object_test.rb +205 -0
- data/test/parsing_test.rb +66 -0
- data/test/remote/acl_test.rb +116 -0
- data/test/remote/bittorrent_test.rb +45 -0
- data/test/remote/bucket_test.rb +146 -0
- data/test/remote/logging_test.rb +82 -0
- data/test/remote/object_test.rb +379 -0
- data/test/remote/test_file.data +0 -0
- data/test/remote/test_helper.rb +33 -0
- data/test/response_test.rb +68 -0
- data/test/service_test.rb +23 -0
- data/test/test_helper.rb +118 -0
- metadata +241 -0
@@ -0,0 +1,58 @@
|
|
1
|
+
module AWS
|
2
|
+
module S3
|
3
|
+
# Objects on S3 can be distributed via the BitTorrent file sharing protocol.
|
4
|
+
#
|
5
|
+
# You can get a torrent file for an object by calling <tt>torrent_for</tt>:
|
6
|
+
#
|
7
|
+
# S3Object.torrent_for 'kiss.jpg', 'marcel'
|
8
|
+
#
|
9
|
+
# Or just call the <tt>torrent</tt> method if you already have the object:
|
10
|
+
#
|
11
|
+
# song = S3Object.find 'kiss.jpg', 'marcel'
|
12
|
+
# song.torrent
|
13
|
+
#
|
14
|
+
# Calling <tt>grant_torrent_access_to</tt> on a object will allow anyone to anonymously
|
15
|
+
# fetch the torrent file for that object:
|
16
|
+
#
|
17
|
+
# S3Object.grant_torrent_access_to 'kiss.jpg', 'marcel'
|
18
|
+
#
|
19
|
+
# Anonymous requests to
|
20
|
+
#
|
21
|
+
# http://s3.amazonaws.com/marcel/kiss.jpg?torrent
|
22
|
+
#
|
23
|
+
# will serve up the torrent file for that object.
|
24
|
+
module BitTorrent
|
25
|
+
def self.included(klass) #:nodoc:
|
26
|
+
klass.extend ClassMethods
|
27
|
+
end
|
28
|
+
|
29
|
+
# Adds methods to S3Object for accessing the torrent of a given object.
|
30
|
+
module ClassMethods
|
31
|
+
# Returns the torrent file for the object with the given <tt>key</tt>.
|
32
|
+
def torrent_for(key, bucket = nil)
|
33
|
+
get(path!(bucket, key) << '?torrent').body
|
34
|
+
end
|
35
|
+
alias_method :torrent, :torrent_for
|
36
|
+
|
37
|
+
# Grants access to the object with the given <tt>key</tt> to be accessible as a torrent.
|
38
|
+
def grant_torrent_access_to(key, bucket = nil)
|
39
|
+
policy = acl(key, bucket)
|
40
|
+
return true if policy.grants.include?(:public_read)
|
41
|
+
policy.grants << ACL::Grant.grant(:public_read)
|
42
|
+
acl(key, bucket, policy)
|
43
|
+
end
|
44
|
+
alias_method :grant_torrent_access, :grant_torrent_access_to
|
45
|
+
end
|
46
|
+
|
47
|
+
# Returns the torrent file for the object.
|
48
|
+
def torrent
|
49
|
+
self.class.torrent_for(key, bucket.name)
|
50
|
+
end
|
51
|
+
|
52
|
+
# Grants torrent access publicly to anyone who requests it on this object.
|
53
|
+
def grant_torrent_access
|
54
|
+
self.class.grant_torrent_access_to(key, bucket.name)
|
55
|
+
end
|
56
|
+
end
|
57
|
+
end
|
58
|
+
end
|
@@ -0,0 +1,372 @@
|
|
1
|
+
module AWS
|
2
|
+
module S3
|
3
|
+
# Buckets are containers for objects (the files you store on S3). To create a new bucket you just specify its name.
|
4
|
+
#
|
5
|
+
# # Pick a unique name, or else you'll get an error
|
6
|
+
# # if the name is already taken.
|
7
|
+
# Bucket.create('jukebox')
|
8
|
+
#
|
9
|
+
# Bucket names must be unique across the entire S3 system, sort of like domain names across the internet. If you try
|
10
|
+
# to create a bucket with a name that is already taken, you will get an error.
|
11
|
+
#
|
12
|
+
# Assuming the name you chose isn't already taken, your new bucket will now appear in the bucket list:
|
13
|
+
#
|
14
|
+
# Service.buckets
|
15
|
+
# # => [#<AWS::S3::Bucket @attributes={"name"=>"jukebox"}>]
|
16
|
+
#
|
17
|
+
# Once you have succesfully created a bucket you can you can fetch it by name using Bucket.find.
|
18
|
+
#
|
19
|
+
# music_bucket = Bucket.find('jukebox')
|
20
|
+
#
|
21
|
+
# The bucket that is returned will contain a listing of all the objects in the bucket.
|
22
|
+
#
|
23
|
+
# music_bucket.objects.size
|
24
|
+
# # => 0
|
25
|
+
#
|
26
|
+
# If all you are interested in is the objects of the bucket, you can get to them directly using Bucket.objects.
|
27
|
+
#
|
28
|
+
# Bucket.objects('jukebox').size
|
29
|
+
# # => 0
|
30
|
+
#
|
31
|
+
# By default all objects will be returned, though there are several options you can use to limit what is returned, such as
|
32
|
+
# specifying that only objects whose name is after a certain place in the alphabet be returned, and etc. Details about these options can
|
33
|
+
# be found in the documentation for Bucket.find.
|
34
|
+
#
|
35
|
+
# To add an object to a bucket you specify the name of the object, its value, and the bucket to put it in.
|
36
|
+
#
|
37
|
+
# file = 'black-flowers.mp3'
|
38
|
+
# S3Object.store(file, open(file), 'jukebox')
|
39
|
+
#
|
40
|
+
# You'll see your file has been added to it:
|
41
|
+
#
|
42
|
+
# music_bucket.objects
|
43
|
+
# # => [#<AWS::S3::S3Object '/jukebox/black-flowers.mp3'>]
|
44
|
+
#
|
45
|
+
# You can treat your bucket like a hash and access objects by name:
|
46
|
+
#
|
47
|
+
# jukebox['black-flowers.mp3']
|
48
|
+
# # => #<AWS::S3::S3Object '/jukebox/black-flowers.mp3'>
|
49
|
+
#
|
50
|
+
# In the event that you want to delete a bucket, you can use Bucket.delete.
|
51
|
+
#
|
52
|
+
# Bucket.delete('jukebox')
|
53
|
+
#
|
54
|
+
# Keep in mind, like unix directories, you can not delete a bucket unless it is empty. Trying to delete a bucket
|
55
|
+
# that contains objects will raise a BucketNotEmpty exception.
|
56
|
+
#
|
57
|
+
# Passing the :force => true option to delete will take care of deleting all the bucket's objects for you.
|
58
|
+
#
|
59
|
+
# Bucket.delete('photos', :force => true)
|
60
|
+
# # => true
|
61
|
+
class Bucket < Base
|
62
|
+
class << self
|
63
|
+
# Creates a bucket named <tt>name</tt>.
|
64
|
+
#
|
65
|
+
# Bucket.create('jukebox')
|
66
|
+
#
|
67
|
+
# Your bucket name must be unique across all of S3. If the name
|
68
|
+
# you request has already been taken, you will get a 409 Conflict response, and a BucketAlreadyExists exception
|
69
|
+
# will be raised.
|
70
|
+
#
|
71
|
+
# By default new buckets have their access level set to private. You can override this using the <tt>:access</tt> option.
|
72
|
+
#
|
73
|
+
# Bucket.create('internet_drop_box', :access => :public_read_write)
|
74
|
+
#
|
75
|
+
# By default new buckets will be created in US. You can override this using the <tt>:location</tt> option.
|
76
|
+
#
|
77
|
+
# Bucket.create('internet_drop_box', :location => :eu)
|
78
|
+
#
|
79
|
+
# The full list of access levels that you can set on Bucket and S3Object creation are listed in the README[link:files/README.html]
|
80
|
+
# in the section called 'Setting access levels'.
|
81
|
+
def create(name, options = {})
|
82
|
+
validate_name!(name)
|
83
|
+
self.current_host = name
|
84
|
+
put("/", options, BucketConfiguration.new(options[:location]).to_s).success?
|
85
|
+
end
|
86
|
+
|
87
|
+
class BucketConfiguration < XmlGenerator
|
88
|
+
|
89
|
+
attr_reader :location
|
90
|
+
|
91
|
+
def initialize(location)
|
92
|
+
@location = location
|
93
|
+
super()
|
94
|
+
end
|
95
|
+
|
96
|
+
def build
|
97
|
+
return nil unless location == :eu
|
98
|
+
xml.tag!('CreateBucketConfiguration') do
|
99
|
+
xml.LocationConstraint 'EU'
|
100
|
+
end
|
101
|
+
end
|
102
|
+
|
103
|
+
end
|
104
|
+
|
105
|
+
# Fetches the bucket named <tt>name</tt>.
|
106
|
+
#
|
107
|
+
# Bucket.find('jukebox')
|
108
|
+
#
|
109
|
+
# If a default bucket is inferable from the current connection's subdomain, or if set explicitly with Base.set_current_bucket,
|
110
|
+
# it will be used if no bucket is specified.
|
111
|
+
#
|
112
|
+
# MusicBucket.current_bucket
|
113
|
+
# => 'jukebox'
|
114
|
+
# MusicBucket.find.name
|
115
|
+
# => 'jukebox'
|
116
|
+
#
|
117
|
+
# By default all objects contained in the bucket will be returned (sans their data) along with the bucket.
|
118
|
+
# You can access your objects using the Bucket#objects method.
|
119
|
+
#
|
120
|
+
# Bucket.find('jukebox').objects
|
121
|
+
#
|
122
|
+
# There are several options which allow you to limit which objects are retrieved. The list of object filtering options
|
123
|
+
# are listed in the documentation for Bucket.objects.
|
124
|
+
def find(name = nil, options = {})
|
125
|
+
new(get(path(name, options)).bucket)
|
126
|
+
end
|
127
|
+
|
128
|
+
# Return just the objects in the bucket named <tt>name</tt>.
|
129
|
+
#
|
130
|
+
# By default all objects of the named bucket will be returned. There are options, though, for filtering
|
131
|
+
# which objects are returned.
|
132
|
+
#
|
133
|
+
# === Object filtering options
|
134
|
+
#
|
135
|
+
# * <tt>:max_keys</tt> - The maximum number of keys you'd like to see in the response body.
|
136
|
+
# The server may return fewer than this many keys, but will not return more.
|
137
|
+
#
|
138
|
+
# Bucket.objects('jukebox').size
|
139
|
+
# # => 3
|
140
|
+
# Bucket.objects('jukebox', :max_keys => 1).size
|
141
|
+
# # => 1
|
142
|
+
#
|
143
|
+
# * <tt>:prefix</tt> - Restricts the response to only contain results that begin with the specified prefix.
|
144
|
+
#
|
145
|
+
# Bucket.objects('jukebox')
|
146
|
+
# # => [<AWS::S3::S3Object '/jazz/miles.mp3'>, <AWS::S3::S3Object '/jazz/dolphy.mp3'>, <AWS::S3::S3Object '/classical/malher.mp3'>]
|
147
|
+
# Bucket.objects('jukebox', :prefix => 'classical')
|
148
|
+
# # => [<AWS::S3::S3Object '/classical/malher.mp3'>]
|
149
|
+
#
|
150
|
+
# * <tt>:marker</tt> - Marker specifies where in the result set to resume listing. It restricts the response
|
151
|
+
# to only contain results that occur alphabetically _after_ the value of marker. To retrieve the next set of results,
|
152
|
+
# use the last key from the current page of results as the marker in your next request.
|
153
|
+
#
|
154
|
+
# # Skip 'mahler'
|
155
|
+
# Bucket.objects('jukebox', :marker => 'mb')
|
156
|
+
# # => [<AWS::S3::S3Object '/jazz/miles.mp3'>]
|
157
|
+
#
|
158
|
+
# === Examples
|
159
|
+
#
|
160
|
+
# # Return no more than 2 objects whose key's are listed alphabetically after the letter 'm'.
|
161
|
+
# Bucket.objects('jukebox', :marker => 'm', :max_keys => 2)
|
162
|
+
# # => [<AWS::S3::S3Object '/jazz/miles.mp3'>, <AWS::S3::S3Object '/classical/malher.mp3'>]
|
163
|
+
#
|
164
|
+
# # Return no more than 2 objects whose key's are listed alphabetically after the letter 'm' and have the 'jazz' prefix.
|
165
|
+
# Bucket.objects('jukebox', :marker => 'm', :max_keys => 2, :prefix => 'jazz')
|
166
|
+
# # => [<AWS::S3::S3Object '/jazz/miles.mp3'>]
|
167
|
+
def objects(name = nil, options = {})
|
168
|
+
find(name, options).object_cache
|
169
|
+
end
|
170
|
+
|
171
|
+
def common_prefixes(name= nil, options = {})
|
172
|
+
find(name, options).common_prefix_cache
|
173
|
+
end
|
174
|
+
|
175
|
+
# Deletes the bucket named <tt>name</tt>.
|
176
|
+
#
|
177
|
+
# All objects in the bucket must be deleted before the bucket can be deleted. If the bucket is not empty,
|
178
|
+
# BucketNotEmpty will be raised.
|
179
|
+
#
|
180
|
+
# You can side step this issue by passing the :force => true option to delete which will take care of
|
181
|
+
# emptying the bucket before deleting it.
|
182
|
+
#
|
183
|
+
# Bucket.delete('photos', :force => true)
|
184
|
+
#
|
185
|
+
# Only the owner of a bucket can delete a bucket, regardless of the bucket's access control policy.
|
186
|
+
def delete(name = nil, options = {})
|
187
|
+
find(name).delete_all if options[:force]
|
188
|
+
|
189
|
+
name = path(name)
|
190
|
+
Base.delete(name).success?
|
191
|
+
end
|
192
|
+
|
193
|
+
# List all your buckets. This is a convenient wrapper around AWS::S3::Service.buckets.
|
194
|
+
def list(reload = false)
|
195
|
+
self.current_host = nil
|
196
|
+
Service.buckets(reload)
|
197
|
+
end
|
198
|
+
|
199
|
+
private
|
200
|
+
def validate_name!(name)
|
201
|
+
raise InvalidBucketName.new(name) unless name =~ /^[-\w.]{3,255}$/
|
202
|
+
end
|
203
|
+
|
204
|
+
def path(name, options = {})
|
205
|
+
if name.is_a?(Hash)
|
206
|
+
options = name
|
207
|
+
name = nil
|
208
|
+
end
|
209
|
+
self.current_host = bucket_name(name)
|
210
|
+
"/#{RequestOptions.process(options).to_query_string}"
|
211
|
+
end
|
212
|
+
end
|
213
|
+
|
214
|
+
attr_reader :object_cache, :common_prefix_cache #:nodoc:
|
215
|
+
|
216
|
+
include Enumerable
|
217
|
+
|
218
|
+
def initialize(attributes = {}) #:nodoc:
|
219
|
+
super
|
220
|
+
@object_cache = []
|
221
|
+
@common_prefix_cache = []
|
222
|
+
build_contents!
|
223
|
+
end
|
224
|
+
|
225
|
+
# Fetches the object named <tt>object_key</tt>, or nil if the bucket does not contain an object with the
|
226
|
+
# specified key.
|
227
|
+
#
|
228
|
+
# bucket.objects
|
229
|
+
# => [#<AWS::S3::S3Object '/marcel_molina/beluga_baby.jpg'>,
|
230
|
+
# #<AWS::S3::S3Object '/marcel_molina/tongue_overload.jpg'>]
|
231
|
+
# bucket['beluga_baby.jpg']
|
232
|
+
# => #<AWS::S3::S3Object '/marcel_molina/beluga_baby.jpg'>
|
233
|
+
def [](object_key)
|
234
|
+
detect {|file| file.key == object_key.to_s}
|
235
|
+
end
|
236
|
+
|
237
|
+
# Initializes a new S3Object belonging to the current bucket.
|
238
|
+
#
|
239
|
+
# object = bucket.new_object
|
240
|
+
# object.value = data
|
241
|
+
# object.key = 'classical/mahler.mp3'
|
242
|
+
# object.store
|
243
|
+
# bucket.objects.include?(object)
|
244
|
+
# => true
|
245
|
+
def new_object(attributes = {})
|
246
|
+
object = S3Object.new(attributes)
|
247
|
+
register(object)
|
248
|
+
object
|
249
|
+
end
|
250
|
+
|
251
|
+
# List S3Object's of the bucket.
|
252
|
+
#
|
253
|
+
# Once fetched the objects will be cached. You can reload the objects by passing <tt>:reload</tt>.
|
254
|
+
#
|
255
|
+
# bucket.objects(:reload)
|
256
|
+
#
|
257
|
+
# You can also filter the objects using the same options listed in Bucket.objects.
|
258
|
+
#
|
259
|
+
# bucket.objects(:prefix => 'jazz')
|
260
|
+
#
|
261
|
+
# Using these filtering options will implictly reload the objects.
|
262
|
+
#
|
263
|
+
# To reclaim all the objects for the bucket you can pass in :reload again.
|
264
|
+
def objects(options = {})
|
265
|
+
if options.is_a?(Hash)
|
266
|
+
reload = !options.empty?
|
267
|
+
else
|
268
|
+
reload = options
|
269
|
+
options = {}
|
270
|
+
end
|
271
|
+
|
272
|
+
reload!(options) if reload || object_cache.empty?
|
273
|
+
object_cache
|
274
|
+
end
|
275
|
+
|
276
|
+
def common_prefixes(options = {})
|
277
|
+
if options.is_a?(Hash)
|
278
|
+
reload = !options.empty?
|
279
|
+
else
|
280
|
+
reload = options
|
281
|
+
options = {}
|
282
|
+
end
|
283
|
+
|
284
|
+
reload!(options) if reload || common_prefix_cache.empty?
|
285
|
+
common_prefix_cache
|
286
|
+
end
|
287
|
+
|
288
|
+
# Iterates over the objects in the bucket.
|
289
|
+
#
|
290
|
+
# bucket.each do |object|
|
291
|
+
# # Do something with the object ...
|
292
|
+
# end
|
293
|
+
def each(&block)
|
294
|
+
# Dup the collection since we might be destructively modifying the object_cache during the iteration.
|
295
|
+
objects.dup.each(&block)
|
296
|
+
end
|
297
|
+
|
298
|
+
# Returns true if there are no objects in the bucket.
|
299
|
+
def empty?
|
300
|
+
objects.empty? && common_prefixes.empty?
|
301
|
+
end
|
302
|
+
|
303
|
+
# Returns the number of objects in the bucket.
|
304
|
+
def size
|
305
|
+
objects.size
|
306
|
+
end
|
307
|
+
|
308
|
+
# Deletes the bucket. See its class method counter part Bucket.delete for caveats about bucket deletion and how to ensure
|
309
|
+
# a bucket is deleted no matter what.
|
310
|
+
def delete(options = {})
|
311
|
+
self.class.delete(name, options)
|
312
|
+
end
|
313
|
+
|
314
|
+
# Delete all files in the bucket. Use with caution. Can not be undone.
|
315
|
+
def delete_all
|
316
|
+
each do |object|
|
317
|
+
object.delete
|
318
|
+
end
|
319
|
+
self
|
320
|
+
end
|
321
|
+
alias_method :clear, :delete_all
|
322
|
+
|
323
|
+
# Buckets observe their objects and have this method called when one of their objects
|
324
|
+
# is either stored or deleted.
|
325
|
+
def update(action, object) #:nodoc:
|
326
|
+
case action
|
327
|
+
when :stored then add object unless objects.include?(object)
|
328
|
+
when :deleted then object_cache.delete(object)
|
329
|
+
end
|
330
|
+
end
|
331
|
+
|
332
|
+
private
|
333
|
+
def build_contents!
|
334
|
+
return unless has_contents?
|
335
|
+
attributes.delete('contents').each do |content|
|
336
|
+
add new_object(content)
|
337
|
+
end
|
338
|
+
|
339
|
+
if attributes['common_prefixes']
|
340
|
+
attributes.delete('common_prefixes').each do |common_prefix|
|
341
|
+
common_prefix_cache << common_prefix['prefix']
|
342
|
+
end
|
343
|
+
end
|
344
|
+
end
|
345
|
+
|
346
|
+
def has_contents?
|
347
|
+
attributes.has_key?('contents')
|
348
|
+
end
|
349
|
+
|
350
|
+
def add(object)
|
351
|
+
register(object)
|
352
|
+
object_cache << object
|
353
|
+
end
|
354
|
+
|
355
|
+
def register(object)
|
356
|
+
object.bucket = self
|
357
|
+
end
|
358
|
+
|
359
|
+
def reload!(options = {})
|
360
|
+
object_cache.clear
|
361
|
+
self.class.objects(name, options).each do |object|
|
362
|
+
add object
|
363
|
+
end
|
364
|
+
|
365
|
+
common_prefix_cache.clear
|
366
|
+
self.class.common_prefixes(name, options).each do |common_prefix|
|
367
|
+
common_prefix_cache << common_prefix
|
368
|
+
end
|
369
|
+
end
|
370
|
+
end
|
371
|
+
end
|
372
|
+
end
|
@@ -0,0 +1,288 @@
|
|
1
|
+
module AWS
|
2
|
+
module S3
|
3
|
+
class Connection #:nodoc:
|
4
|
+
class << self
|
5
|
+
def connect(options = {})
|
6
|
+
new(options)
|
7
|
+
end
|
8
|
+
|
9
|
+
def prepare_path(path)
|
10
|
+
path = path.remove_extended unless path.valid_utf8?
|
11
|
+
URI.escape(path)
|
12
|
+
end
|
13
|
+
end
|
14
|
+
|
15
|
+
attr_reader :access_key_id, :secret_access_key, :http, :options
|
16
|
+
|
17
|
+
# Creates a new connection. Connections make the actual requests to S3, though these requests are usually
|
18
|
+
# called from subclasses of Base.
|
19
|
+
#
|
20
|
+
# For details on establishing connections, check the Connection::Management::ClassMethods.
|
21
|
+
def initialize(options = {})
|
22
|
+
@options = Options.new(options)
|
23
|
+
connect
|
24
|
+
end
|
25
|
+
|
26
|
+
def request(verb, path, headers = {}, body = nil, attempts = 0, current_host = nil, &block)
|
27
|
+
body.rewind if body.respond_to?(:rewind) unless attempts.zero?
|
28
|
+
|
29
|
+
requester = Proc.new do
|
30
|
+
path = self.class.prepare_path(path) if attempts.zero? # Only escape the path once
|
31
|
+
request = request_method(verb).new(path, headers)
|
32
|
+
ensure_content_type!(request)
|
33
|
+
add_user_agent!(request)
|
34
|
+
set_host!(request, current_host)
|
35
|
+
authenticate!(request, current_host)
|
36
|
+
if body
|
37
|
+
if body.respond_to?(:read)
|
38
|
+
request.body_stream = body
|
39
|
+
else
|
40
|
+
request.body = body
|
41
|
+
end
|
42
|
+
request.content_length = body.respond_to?(:lstat) ? body.stat.size : body.size
|
43
|
+
else
|
44
|
+
request.content_length = 0
|
45
|
+
end
|
46
|
+
http.request(request, &block)
|
47
|
+
end
|
48
|
+
|
49
|
+
if persistent?
|
50
|
+
http.start unless http.started?
|
51
|
+
requester.call
|
52
|
+
else
|
53
|
+
http.start(&requester)
|
54
|
+
end
|
55
|
+
rescue Errno::EPIPE, Timeout::Error, Errno::EINVAL, EOFError
|
56
|
+
@http = create_connection
|
57
|
+
attempts == 3 ? raise : (attempts += 1; retry)
|
58
|
+
end
|
59
|
+
|
60
|
+
def url_for(path, current_host, options = {})
|
61
|
+
authenticate = options.delete(:authenticated)
|
62
|
+
# Default to true unless explicitly false
|
63
|
+
authenticate = true if authenticate.nil?
|
64
|
+
path = self.class.prepare_path(path)
|
65
|
+
request = request_method(:get).new(path, {})
|
66
|
+
query_string = query_string_authentication(request, current_host, options)
|
67
|
+
returning "#{protocol(options)}#{get_host(current_host)}#{port_string}#{path}" do |url|
|
68
|
+
url << "?#{query_string}" if authenticate
|
69
|
+
end
|
70
|
+
end
|
71
|
+
|
72
|
+
def subdomain
|
73
|
+
subdomain = http.address.gsub("#{DEFAULT_HOST}", "").gsub(/.$/,'')
|
74
|
+
subdomain.empty? ? nil : subdomain
|
75
|
+
end
|
76
|
+
|
77
|
+
def persistent?
|
78
|
+
options[:persistent]
|
79
|
+
end
|
80
|
+
|
81
|
+
def protocol(options = {})
|
82
|
+
# This always trumps http.use_ssl?
|
83
|
+
if options[:use_ssl] == false
|
84
|
+
'http://'
|
85
|
+
elsif options[:use_ssl] || http.use_ssl?
|
86
|
+
'https://'
|
87
|
+
else
|
88
|
+
'http://'
|
89
|
+
end
|
90
|
+
end
|
91
|
+
|
92
|
+
private
|
93
|
+
def extract_keys!
|
94
|
+
missing_keys = []
|
95
|
+
extract_key = Proc.new {|key| options[key] || (missing_keys.push(key); nil)}
|
96
|
+
@access_key_id = extract_key[:access_key_id]
|
97
|
+
@secret_access_key = extract_key[:secret_access_key]
|
98
|
+
raise MissingAccessKey.new(missing_keys) unless missing_keys.empty?
|
99
|
+
end
|
100
|
+
|
101
|
+
def create_connection
|
102
|
+
http = http_class.new(options[:server], options[:port])
|
103
|
+
http.use_ssl = !options[:use_ssl].nil? || options[:port] == 443
|
104
|
+
http.verify_mode = OpenSSL::SSL::VERIFY_NONE
|
105
|
+
http
|
106
|
+
end
|
107
|
+
|
108
|
+
def http_class
|
109
|
+
if options.connecting_through_proxy?
|
110
|
+
Net::HTTP::Proxy(*options.proxy_settings)
|
111
|
+
else
|
112
|
+
Net::HTTP
|
113
|
+
end
|
114
|
+
end
|
115
|
+
|
116
|
+
def connect
|
117
|
+
extract_keys!
|
118
|
+
@http = create_connection
|
119
|
+
end
|
120
|
+
|
121
|
+
def port_string
|
122
|
+
default_port = options[:use_ssl] ? 443 : 80
|
123
|
+
http.port == default_port ? '' : ":#{http.port}"
|
124
|
+
end
|
125
|
+
|
126
|
+
def ensure_content_type!(request)
|
127
|
+
request['Content-Type'] ||= 'binary/octet-stream'
|
128
|
+
end
|
129
|
+
|
130
|
+
# Just do Header authentication for now
|
131
|
+
def authenticate!(request, current_host)
|
132
|
+
request['Authorization'] = Authentication::Header.new(request, access_key_id, secret_access_key, current_host)
|
133
|
+
end
|
134
|
+
|
135
|
+
def add_user_agent!(request)
|
136
|
+
request['User-Agent'] ||= "AWS::S3/#{Version}"
|
137
|
+
end
|
138
|
+
|
139
|
+
def set_host!(request, host)
|
140
|
+
request['Host'] = get_host(host)
|
141
|
+
end
|
142
|
+
|
143
|
+
def get_host(host)
|
144
|
+
(host && http.address.match(host)) ? http.address : host.nil? ? http.address : host.match(/amazonaws.com/) ? host : "#{host}.#{http.address}"
|
145
|
+
end
|
146
|
+
|
147
|
+
def query_string_authentication(request, current_host, options = {})
|
148
|
+
Authentication::QueryString.new(request, access_key_id, secret_access_key, current_host, options)
|
149
|
+
end
|
150
|
+
|
151
|
+
def request_method(verb)
|
152
|
+
Net::HTTP.const_get(verb.to_s.capitalize)
|
153
|
+
end
|
154
|
+
|
155
|
+
def method_missing(method, *args, &block)
|
156
|
+
options[method] || super
|
157
|
+
end
|
158
|
+
|
159
|
+
module Management #:nodoc:
|
160
|
+
def self.included(base)
|
161
|
+
base.cattr_accessor :connections
|
162
|
+
base.connections = {}
|
163
|
+
base.extend ClassMethods
|
164
|
+
end
|
165
|
+
|
166
|
+
# Manage the creation and destruction of connections for AWS::S3::Base and its subclasses. Connections are
|
167
|
+
# created with establish_connection!.
|
168
|
+
module ClassMethods
|
169
|
+
# Creates a new connection with which to make requests to the S3 servers for the calling class.
|
170
|
+
#
|
171
|
+
# AWS::S3::Base.establish_connection!(:access_key_id => '...', :secret_access_key => '...')
|
172
|
+
#
|
173
|
+
# You can set connections for every subclass of AWS::S3::Base. Once the initial connection is made on
|
174
|
+
# Base, all subsequent connections will inherit whatever values you don't specify explictly. This allows you to
|
175
|
+
# customize details of the connection, such as what server the requests are made to, by just specifying one
|
176
|
+
# option.
|
177
|
+
#
|
178
|
+
# AWS::S3::Bucket.established_connection!(:use_ssl => true)
|
179
|
+
#
|
180
|
+
# The Bucket connection would inherit the <tt>:access_key_id</tt> and the <tt>:secret_access_key</tt> from
|
181
|
+
# Base's connection. Unlike the Base connection, all Bucket requests would be made over SSL.
|
182
|
+
#
|
183
|
+
# == Required arguments
|
184
|
+
#
|
185
|
+
# * <tt>:access_key_id</tt> - The access key id for your S3 account. Provided by Amazon.
|
186
|
+
# * <tt>:secret_access_key</tt> - The secret access key for your S3 account. Provided by Amazon.
|
187
|
+
#
|
188
|
+
# If any of these required arguments is missing, a MissingAccessKey exception will be raised.
|
189
|
+
#
|
190
|
+
# == Optional arguments
|
191
|
+
#
|
192
|
+
# * <tt>:server</tt> - The server to make requests to. You can use this to specify your bucket in the subdomain,
|
193
|
+
# or your own domain's cname if you are using virtual hosted buckets. Defaults to <tt>s3.amazonaws.com</tt>.
|
194
|
+
# * <tt>:port</tt> - The port to the requests should be made on. Defaults to 80 or 443 if the <tt>:use_ssl</tt>
|
195
|
+
# argument is set.
|
196
|
+
# * <tt>:use_ssl</tt> - Whether requests should be made over SSL. If set to true, the <tt>:port</tt> argument
|
197
|
+
# will be implicitly set to 443, unless specified otherwise. Defaults to false.
|
198
|
+
# * <tt>:persistent</tt> - Whether to use a persistent connection to the server. Having this on provides around a two fold
|
199
|
+
# performance increase but for long running processes some firewalls may find the long lived connection suspicious and close the connection.
|
200
|
+
# If you run into connection errors, try setting <tt>:persistent</tt> to false. Defaults to false.
|
201
|
+
# * <tt>:proxy</tt> - If you need to connect through a proxy, you can specify your proxy settings by specifying a <tt>:host</tt>, <tt>:port</tt>, <tt>:user</tt>, and <tt>:password</tt>
|
202
|
+
# with the <tt>:proxy</tt> option.
|
203
|
+
# The <tt>:host</tt> setting is required if specifying a <tt>:proxy</tt>.
|
204
|
+
#
|
205
|
+
# AWS::S3::Bucket.established_connection!(:proxy => {
|
206
|
+
# :host => '...', :port => 8080, :user => 'marcel', :password => 'secret'
|
207
|
+
# })
|
208
|
+
def establish_connection!(options = {})
|
209
|
+
# After you've already established the default connection, just specify
|
210
|
+
# the difference for subsequent connections
|
211
|
+
options = default_connection.options.merge(options) if connected?
|
212
|
+
connections[connection_name] = Connection.connect(options)
|
213
|
+
end
|
214
|
+
|
215
|
+
# Returns the connection for the current class, or Base's default connection if the current class does not
|
216
|
+
# have its own connection.
|
217
|
+
#
|
218
|
+
# If not connection has been established yet, NoConnectionEstablished will be raised.
|
219
|
+
def connection
|
220
|
+
if connected?
|
221
|
+
connections[connection_name] || default_connection
|
222
|
+
else
|
223
|
+
raise NoConnectionEstablished
|
224
|
+
end
|
225
|
+
end
|
226
|
+
|
227
|
+
# Returns true if a connection has been made yet.
|
228
|
+
def connected?
|
229
|
+
!connections.empty?
|
230
|
+
end
|
231
|
+
|
232
|
+
# Removes the connection for the current class. If there is no connection for the current class, the default
|
233
|
+
# connection will be removed.
|
234
|
+
def disconnect(name = connection_name)
|
235
|
+
name = default_connection unless connections.has_key?(name)
|
236
|
+
connection = connections[name]
|
237
|
+
connection.http.finish if connection.persistent?
|
238
|
+
connections.delete(name)
|
239
|
+
end
|
240
|
+
|
241
|
+
# Clears *all* connections, from all classes, with prejudice.
|
242
|
+
def disconnect!
|
243
|
+
connections.each_key {|connection| disconnect(connection)}
|
244
|
+
end
|
245
|
+
|
246
|
+
private
|
247
|
+
def connection_name
|
248
|
+
name
|
249
|
+
end
|
250
|
+
|
251
|
+
def default_connection_name
|
252
|
+
'AWS::S3::Base'
|
253
|
+
end
|
254
|
+
|
255
|
+
def default_connection
|
256
|
+
connections[default_connection_name]
|
257
|
+
end
|
258
|
+
end
|
259
|
+
end
|
260
|
+
|
261
|
+
class Options < Hash #:nodoc:
|
262
|
+
VALID_OPTIONS = [:access_key_id, :secret_access_key, :server, :port, :use_ssl, :persistent, :proxy].freeze
|
263
|
+
|
264
|
+
def initialize(options = {})
|
265
|
+
super()
|
266
|
+
validate(options)
|
267
|
+
replace(:server => DEFAULT_HOST, :port => (options[:use_ssl] ? 443 : 80))
|
268
|
+
merge!(options)
|
269
|
+
end
|
270
|
+
|
271
|
+
def connecting_through_proxy?
|
272
|
+
!self[:proxy].nil?
|
273
|
+
end
|
274
|
+
|
275
|
+
def proxy_settings
|
276
|
+
self[:proxy].values_at(:host, :port, :user, :password)
|
277
|
+
end
|
278
|
+
|
279
|
+
private
|
280
|
+
def validate(options)
|
281
|
+
invalid_options = options.keys - VALID_OPTIONS
|
282
|
+
raise InvalidConnectionOption.new(invalid_options) unless invalid_options.empty?
|
283
|
+
raise ArgumentError, "Missing proxy settings. Must specify at least :host." if options[:proxy] && !options[:proxy][:host]
|
284
|
+
end
|
285
|
+
end
|
286
|
+
end
|
287
|
+
end
|
288
|
+
end
|