s33r 0.3.1 → 0.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- data/examples/cli/acl_x.rb +41 -0
- data/examples/cli/logging_x.rb +20 -0
- data/{bin → examples/cli}/s3cli.rb +13 -20
- data/examples/fores33r/README +183 -0
- data/examples/fores33r/Rakefile +10 -0
- data/examples/fores33r/app/controllers/application.rb +4 -0
- data/examples/fores33r/app/controllers/browser_controller.rb +68 -0
- data/examples/fores33r/app/helpers/application_helper.rb +8 -0
- data/examples/fores33r/app/views/browser/index.rhtml +14 -0
- data/examples/fores33r/app/views/browser/show_bucket.rhtml +14 -0
- data/examples/fores33r/app/views/layouts/application.rhtml +29 -0
- data/examples/fores33r/config/boot.rb +44 -0
- data/examples/fores33r/config/database.yml +35 -0
- data/examples/fores33r/config/environment.rb +64 -0
- data/examples/fores33r/config/environments/development.rb +21 -0
- data/examples/fores33r/config/environments/production.rb +18 -0
- data/examples/fores33r/config/environments/test.rb +19 -0
- data/examples/fores33r/config/routes.rb +23 -0
- data/examples/fores33r/doc/README_FOR_APP +2 -0
- data/examples/fores33r/log/development.log +5507 -0
- data/examples/fores33r/log/production.log +0 -0
- data/examples/fores33r/log/server.log +0 -0
- data/examples/fores33r/log/test.log +0 -0
- data/examples/fores33r/public/404.html +8 -0
- data/examples/fores33r/public/500.html +8 -0
- data/examples/fores33r/public/dispatch.cgi +10 -0
- data/examples/fores33r/public/dispatch.fcgi +24 -0
- data/examples/fores33r/public/dispatch.rb +10 -0
- data/examples/fores33r/public/favicon.ico +0 -0
- data/examples/fores33r/public/images/rails.png +0 -0
- data/examples/fores33r/public/javascripts/application.js +2 -0
- data/examples/fores33r/public/javascripts/controls.js +815 -0
- data/examples/fores33r/public/javascripts/dragdrop.js +913 -0
- data/examples/fores33r/public/javascripts/effects.js +958 -0
- data/examples/fores33r/public/javascripts/prototype.js +2006 -0
- data/examples/fores33r/public/robots.txt +1 -0
- data/examples/fores33r/public/stylesheets/core.css +37 -0
- data/examples/fores33r/script/about +3 -0
- data/examples/fores33r/script/breakpointer +3 -0
- data/examples/fores33r/script/console +3 -0
- data/examples/fores33r/script/destroy +3 -0
- data/examples/fores33r/script/generate +3 -0
- data/examples/fores33r/script/performance/benchmarker +3 -0
- data/examples/fores33r/script/performance/profiler +3 -0
- data/examples/fores33r/script/plugin +3 -0
- data/examples/fores33r/script/process/reaper +3 -0
- data/examples/fores33r/script/process/spawner +3 -0
- data/examples/fores33r/script/runner +3 -0
- data/examples/fores33r/script/server +3 -0
- data/examples/fores33r/test/test_helper.rb +28 -0
- data/examples/fores33r/tmp/sessions/ruby_sess.39d37e054d21d545 +0 -0
- data/examples/fores33r/tmp/sessions/ruby_sess.acf71fc73aa74983 +0 -0
- data/examples/fores33r/tmp/sessions/ruby_sess.c1697b7d6670f3cd +0 -0
- data/examples/s3.yaml +11 -0
- data/html/classes/Net/HTTPGenericRequest.html +32 -32
- data/html/classes/Net/HTTPResponse.html +20 -19
- data/html/classes/S33r.html +422 -190
- data/html/classes/S33r/BucketListing.html +107 -70
- data/html/classes/S33r/Client.html +888 -414
- data/html/classes/S33r/LoggingResource.html +222 -0
- data/html/classes/S33r/NamedBucket.html +149 -150
- data/html/classes/S33r/OrderlyXmlMarkup.html +165 -0
- data/html/classes/S33r/S33rException.html +3 -0
- data/html/classes/S33r/S33rException/BucketNotLogTargetable.html +119 -0
- data/html/classes/S33r/S33rException/InvalidPermission.html +111 -0
- data/html/classes/S33r/S33rException/InvalidS3GroupType.html +111 -0
- data/html/classes/S33r/S3ACL.html +125 -0
- data/html/classes/S33r/S3ACL/ACLDoc.html +521 -0
- data/html/classes/S33r/{S3User.html → S3ACL/AmazonCustomer.html} +27 -30
- data/html/classes/S33r/S3ACL/CanonicalUser.html +212 -0
- data/html/classes/S33r/S3ACL/Grant.html +403 -0
- data/html/classes/S33r/S3ACL/Grantee.html +239 -0
- data/html/classes/S33r/S3ACL/Group.html +178 -0
- data/html/classes/S33r/S3Object.html +53 -50
- data/html/classes/S33r/Sync.html +6 -6
- data/html/classes/XML.html +4 -2
- data/html/created.rid +1 -1
- data/html/files/README_txt.html +82 -28
- data/html/files/lib/s33r/bucket_listing_rb.html +1 -8
- data/html/files/lib/s33r/builder_rb.html +108 -0
- data/html/files/lib/s33r/client_rb.html +2 -1
- data/html/files/lib/s33r/core_rb.html +2 -1
- data/html/files/lib/s33r/libxml_extensions_rb.html +1 -1
- data/html/files/lib/s33r/logging_rb.html +109 -0
- data/html/files/lib/s33r/named_bucket_rb.html +1 -1
- data/html/files/lib/s33r/s33r_exception_rb.html +1 -1
- data/html/files/lib/s33r/s33r_http_rb.html +1 -1
- data/html/files/lib/s33r/s3_acl_rb.html +109 -0
- data/html/fr_class_index.html +12 -1
- data/html/fr_file_index.html +3 -0
- data/html/fr_method_index.html +101 -57
- data/lib/s33r/bucket_listing.rb +21 -22
- data/lib/s33r/builder.rb +20 -0
- data/lib/s33r/client.rb +240 -42
- data/lib/s33r/core.rb +106 -36
- data/lib/s33r/libxml_extensions.rb +2 -2
- data/lib/s33r/logging.rb +43 -0
- data/lib/s33r/named_bucket.rb +16 -17
- data/lib/s33r/s33r_exception.rb +11 -0
- data/lib/s33r/s33r_http.rb +2 -1
- data/lib/s33r/s3_acl.rb +337 -0
- data/test/cases/spec_acl.rb +146 -0
- data/test/cases/spec_all_buckets.rb +28 -0
- data/test/cases/spec_bucket_listing.rb +2 -2
- data/test/cases/spec_client.rb +45 -18
- data/test/cases/spec_core.rb +0 -9
- data/test/cases/spec_namedbucket.rb +3 -3
- data/test/files/acl.xml +47 -0
- data/test/files/acl_grant1.xml +7 -0
- data/test/files/acl_grant2.xml +6 -0
- data/test/files/acl_grant3.xml +6 -0
- data/test/files/acl_grant4.xml +6 -0
- data/test/files/all_buckets.xml +21 -0
- data/test/files/bucket_listing.xml +138 -1
- data/test/files/client_config.yml +0 -1
- data/test/files/logging_acl.xml +34 -0
- data/test/files/namedbucket_config.yml +1 -5
- data/test/files/namedbucket_config2.yml +1 -5
- data/test/test_setup.rb +1 -0
- metadata +132 -7
data/lib/s33r/core.rb
CHANGED
|
@@ -4,6 +4,7 @@ require 'net/http'
|
|
|
4
4
|
require 'net/https'
|
|
5
5
|
require 'openssl'
|
|
6
6
|
require 'xml/libxml'
|
|
7
|
+
require 'parsedate'
|
|
7
8
|
|
|
8
9
|
# Module to handle S3 operations which don't require an internet connection,
|
|
9
10
|
# i.e. data validation and request-building operations;
|
|
@@ -24,13 +25,14 @@ module S33r
|
|
|
24
25
|
PORT = 443
|
|
25
26
|
NON_SSL_PORT = 80
|
|
26
27
|
METADATA_PREFIX = 'x-amz-meta-'
|
|
27
|
-
# Size of each chunk (in bytes) to be sent per request when putting files.
|
|
28
|
+
# Size of each chunk (in bytes) to be sent per request when putting files (1Mb).
|
|
28
29
|
DEFAULT_CHUNK_SIZE = 1048576
|
|
29
30
|
AWS_HEADER_PREFIX = 'x-amz-'
|
|
30
31
|
AWS_AUTH_HEADER_VALUE = "AWS %s:%s"
|
|
31
32
|
INTERESTING_HEADERS = ['content-md5', 'content-type', 'date']
|
|
32
33
|
# Headers which must be included with every request to S3.
|
|
33
34
|
REQUIRED_HEADERS = ['Content-Type', 'Date']
|
|
35
|
+
# Canned ACLs made available by S3.
|
|
34
36
|
CANNED_ACLS = ['private', 'public-read', 'public-read-write', 'authenticated-read']
|
|
35
37
|
# HTTP methods which S3 will respond to.
|
|
36
38
|
METHOD_VERBS = ['GET', 'PUT', 'HEAD', 'POST', 'DELETE']
|
|
@@ -38,6 +40,35 @@ module S33r
|
|
|
38
40
|
BUCKET_LIST_MAX_MAX_KEYS = 1000
|
|
39
41
|
# Default number of seconds an authenticated URL will last for (15 minutes).
|
|
40
42
|
DEFAULT_EXPIRY_SECS = 60 * 15
|
|
43
|
+
# The namespace used for response body XML documents.
|
|
44
|
+
RESPONSE_NAMESPACE_URI = "http://s3.amazonaws.com/doc/2006-03-01/"
|
|
45
|
+
|
|
46
|
+
# Permissions which can be set within a <Grant>
|
|
47
|
+
# (see http://docs.amazonwebservices.com/AmazonS3/2006-03-01/UsingPermissions.html).
|
|
48
|
+
#
|
|
49
|
+
# NB I've missed out the WRITE_ACP permission as this is functionally
|
|
50
|
+
# equivalent to FULL_CONTROL.
|
|
51
|
+
PERMISSIONS = {
|
|
52
|
+
:read => 'READ', # permission to read
|
|
53
|
+
:write => 'WRITE', # permission to write
|
|
54
|
+
:read_acl => 'READ_ACP', # permission to read ACL settings
|
|
55
|
+
:all => 'FULL_CONTROL' # do anything
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
# Used for generating ACL XML documents.
|
|
59
|
+
NAMESPACE = 'xsi'
|
|
60
|
+
NAMESPACE_URI = 'http://www.w3.org/2001/XMLSchema-instance'
|
|
61
|
+
GRANTEE_TYPES = {
|
|
62
|
+
:amazon_customer => 'AmazonCustomerByEmail',
|
|
63
|
+
:canonical_user => 'CanonicalUser',
|
|
64
|
+
:group => 'Group'
|
|
65
|
+
}
|
|
66
|
+
S3_GROUP_TYPES = {
|
|
67
|
+
:all_users => 'global/AllUsers',
|
|
68
|
+
:authenticated_users => 'global/AuthenticatedUsers',
|
|
69
|
+
:log_delivery => 's3/LogDelivery'
|
|
70
|
+
}
|
|
71
|
+
GROUP_ACL_URI_BASE = 'http://acs.amazonaws.com/groups/'
|
|
41
72
|
|
|
42
73
|
# Build canonical string for signing;
|
|
43
74
|
# modified (slightly) from the Amazon sample code.
|
|
@@ -50,11 +81,11 @@ module S33r
|
|
|
50
81
|
end
|
|
51
82
|
end
|
|
52
83
|
|
|
53
|
-
#
|
|
84
|
+
# These fields get empty strings if they don't exist.
|
|
54
85
|
interesting_headers['content-type'] ||= ''
|
|
55
86
|
interesting_headers['content-md5'] ||= ''
|
|
56
87
|
|
|
57
|
-
#
|
|
88
|
+
# If you're using expires for query string auth, then it trumps date.
|
|
58
89
|
if not expires.nil?
|
|
59
90
|
interesting_headers['date'] = expires
|
|
60
91
|
end
|
|
@@ -68,17 +99,19 @@ module S33r
|
|
|
68
99
|
end
|
|
69
100
|
end
|
|
70
101
|
|
|
71
|
-
#
|
|
102
|
+
# Ignore everything after the question mark...
|
|
72
103
|
buf << path.gsub(/\?.*$/, '')
|
|
73
104
|
|
|
74
|
-
# ...unless there is an acl or torrent parameter
|
|
105
|
+
# ...unless there is an acl, logging or torrent parameter
|
|
75
106
|
if path =~ /[&?]acl($|&|=)/
|
|
76
107
|
buf << '?acl'
|
|
77
108
|
elsif path =~ /[&?]torrent($|&|=)/
|
|
78
109
|
buf << '?torrent'
|
|
110
|
+
elsif path =~ /[&?]logging($|&|=)/
|
|
111
|
+
buf << '?logging'
|
|
79
112
|
end
|
|
80
113
|
|
|
81
|
-
|
|
114
|
+
buf
|
|
82
115
|
end
|
|
83
116
|
|
|
84
117
|
# Get the value for the AWS authentication header.
|
|
@@ -106,8 +139,8 @@ module S33r
|
|
|
106
139
|
end
|
|
107
140
|
|
|
108
141
|
# Build the headers required with every S3 request (Date and Content-Type);
|
|
109
|
-
# options hash can contain extra header settings
|
|
110
|
-
#
|
|
142
|
+
# options hash can contain extra header settings;
|
|
143
|
+
# +:date+ and +:content_type+ are required headers, and set to defaults if not supplied
|
|
111
144
|
def add_default_headers(headers, options={})
|
|
112
145
|
# set the default headers required by AWS
|
|
113
146
|
missing_headers = REQUIRED_HEADERS - headers.keys
|
|
@@ -125,6 +158,7 @@ module S33r
|
|
|
125
158
|
end
|
|
126
159
|
|
|
127
160
|
# Add metadata headers, correctly prefixing them first.
|
|
161
|
+
#
|
|
128
162
|
# Returns headers with the metadata headers appended.
|
|
129
163
|
def metadata_headers(headers, metadata={})
|
|
130
164
|
unless metadata.empty?
|
|
@@ -168,38 +202,43 @@ module S33r
|
|
|
168
202
|
end
|
|
169
203
|
str
|
|
170
204
|
end
|
|
171
|
-
|
|
172
|
-
#
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
179
|
-
|
|
180
|
-
scheme_ends_at = (url_end =~ /:\/\//)
|
|
181
|
-
unless scheme_ends_at.nil?
|
|
182
|
-
scheme_ends_at = scheme_ends_at + 1
|
|
183
|
-
url_start = url_end[0..scheme_ends_at]
|
|
184
|
-
url_end = url_end[(scheme_ends_at + 1)..-1]
|
|
185
|
-
end
|
|
186
|
-
|
|
187
|
-
# replace any multiple forward slashes (except those in the scheme)
|
|
188
|
-
url_end = url_end.gsub(/\/{2,}/, '/')
|
|
189
|
-
|
|
190
|
-
url_start + url_end
|
|
205
|
+
|
|
206
|
+
# Return the location of the ACL for a resource.
|
|
207
|
+
def s3_acl_path(bucket_name, resource_key)
|
|
208
|
+
s3_path(bucket_name, resource_key) + "?acl"
|
|
209
|
+
end
|
|
210
|
+
|
|
211
|
+
# Return the location of the logging definition for a resource.
|
|
212
|
+
def s3_logging_path(bucket_name, resource_key)
|
|
213
|
+
s3_path(bucket_name, resource_key) + "?logging"
|
|
191
214
|
end
|
|
192
215
|
|
|
193
|
-
#
|
|
194
|
-
def
|
|
195
|
-
"http://" + HOST +
|
|
216
|
+
# Prepend scheme and HOST to path.
|
|
217
|
+
def s3_url(path)
|
|
218
|
+
"http://" + HOST + path
|
|
219
|
+
end
|
|
220
|
+
|
|
221
|
+
# Public readable URL for a bucket and resource.
|
|
222
|
+
def s3_public_url(bucket_name, resource_key='')
|
|
223
|
+
path = s3_path(bucket_name, resource_key)
|
|
224
|
+
s3_url(path)
|
|
225
|
+
end
|
|
226
|
+
|
|
227
|
+
# Returns the path for this bucket and key.
|
|
228
|
+
def s3_path(bucket_name, resource_key)
|
|
229
|
+
path = '/' + bucket_name
|
|
230
|
+
path += '/' + resource_key unless '' == resource_key or resource_key.nil?
|
|
231
|
+
path
|
|
196
232
|
end
|
|
197
233
|
|
|
198
234
|
# Generate a get-able URL for an S3 resource key which passes authentication in querystring.
|
|
199
|
-
#
|
|
235
|
+
#
|
|
236
|
+
# int +expires+: when the URL expires (seconds since the epoch); NB you can use S33r.parse_expiry
|
|
237
|
+
# to generate a suitable value from a string.
|
|
200
238
|
def s3_authenticated_url(aws_access_key, aws_secret_access_key, bucket_name, resource_key,
|
|
201
|
-
expires)
|
|
202
|
-
path =
|
|
239
|
+
expires=nil)
|
|
240
|
+
path = s3_path(bucket_name, resource_key)
|
|
241
|
+
expires = S33r.parse_expiry(expires)
|
|
203
242
|
|
|
204
243
|
canonical_string = generate_canonical_string('GET', path, {}, expires)
|
|
205
244
|
signature = generate_signature(aws_secret_access_key, canonical_string)
|
|
@@ -207,10 +246,11 @@ module S33r
|
|
|
207
246
|
querystring = generate_querystring({ 'Signature' => signature, 'Expires' => expires,
|
|
208
247
|
'AWSAccessKeyId' => aws_access_key })
|
|
209
248
|
|
|
210
|
-
return
|
|
249
|
+
return s3_url(path) + querystring
|
|
211
250
|
end
|
|
212
251
|
|
|
213
|
-
# Turn keys in a hash hsh into symbols.
|
|
252
|
+
# Turn keys in a hash +hsh+ into symbols.
|
|
253
|
+
#
|
|
214
254
|
# Returns a hash with 'symbolised' keys.
|
|
215
255
|
def S33r.keys_to_symbols(hsh)
|
|
216
256
|
symbolised = hsh.inject({}) do |symbolised, key_value|
|
|
@@ -218,4 +258,34 @@ module S33r
|
|
|
218
258
|
end
|
|
219
259
|
symbolised
|
|
220
260
|
end
|
|
261
|
+
|
|
262
|
+
# Parse an expiry date to seconds since the epoch.
|
|
263
|
+
#
|
|
264
|
+
# +expires+ can be set to 'forever' to get a time 20 years in the future;
|
|
265
|
+
# 'default' to use the current time + DEFAULT_EXPIRY_SECS;
|
|
266
|
+
# or to a specific date (parseable by ParseDate); or to an integer
|
|
267
|
+
# representing seconds since the epoch.
|
|
268
|
+
#
|
|
269
|
+
# TODO: testing for this
|
|
270
|
+
def S33r.parse_expiry(expires)
|
|
271
|
+
base_expires = Time.now.to_i
|
|
272
|
+
if 'forever' == expires
|
|
273
|
+
# 20 years (same as forever in computer terms)
|
|
274
|
+
expires = base_expires + (60 * 60 * 24 * 365.25 * 20).to_i
|
|
275
|
+
elsif 'default' == expires
|
|
276
|
+
# default to DEFAULT_EXPIRY_SECS seconds from now if expires not set
|
|
277
|
+
expires = base_expires + DEFAULT_EXPIRY_SECS
|
|
278
|
+
elsif expires.is_a?(String)
|
|
279
|
+
datetime_parts = ParseDate.parsedate(expires)
|
|
280
|
+
expires = Time.gm(*datetime_parts).to_i
|
|
281
|
+
end
|
|
282
|
+
expires
|
|
283
|
+
end
|
|
284
|
+
|
|
285
|
+
# Remove the namespace declaration from S3 XML response bodies (libxml
|
|
286
|
+
# isn't fond of it).
|
|
287
|
+
def S33r.remove_namespace(xml_in)
|
|
288
|
+
namespace = RESPONSE_NAMESPACE_URI.gsub('/', '\/')
|
|
289
|
+
xml_in.gsub(/ xmlns="#{namespace}"/, '')
|
|
290
|
+
end
|
|
221
291
|
end
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
# Convenience methods for libxml classes.
|
|
2
2
|
module XML
|
|
3
|
-
# Find first matching
|
|
4
|
-
# (intended for use as a mixed-in method for Document instances).
|
|
3
|
+
# Find first element matching +xpath+ and return its content
|
|
4
|
+
# (intended for use as a mixed-in method for Document/Node instances).
|
|
5
5
|
#
|
|
6
6
|
# +xpath+: XPath query to run against self.
|
|
7
7
|
#
|
data/lib/s33r/logging.rb
ADDED
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
require 'rubygems'
|
|
2
|
+
require 'xml/libxml'
|
|
3
|
+
require_gem 'builder'
|
|
4
|
+
|
|
5
|
+
module S33r
|
|
6
|
+
# For manipulating logging directives on resources
|
|
7
|
+
# (see http://docs.amazonwebservices.com/AmazonS3/2006-03-01/LoggingHowTo.html).
|
|
8
|
+
#
|
|
9
|
+
# Calling LoggingResource.new (no arguments) will generate a blank instance
|
|
10
|
+
# which can be used to remove logging from a resource.
|
|
11
|
+
class LoggingResource
|
|
12
|
+
attr_reader :log_target, :log_prefix
|
|
13
|
+
|
|
14
|
+
def initialize(log_target=nil, log_prefix=nil)
|
|
15
|
+
@log_target = log_target
|
|
16
|
+
@log_prefix = log_prefix
|
|
17
|
+
end
|
|
18
|
+
|
|
19
|
+
# Generate a BucketLoggingStatus XML document for putting to the ?logging
|
|
20
|
+
# URL for a resource.
|
|
21
|
+
#
|
|
22
|
+
#-- TODO: test generates correct XML
|
|
23
|
+
def to_xml
|
|
24
|
+
xml_str = ""
|
|
25
|
+
xml = Builder::XmlMarkup.new(:target => xml_str, :indent => 0)
|
|
26
|
+
|
|
27
|
+
xml.instruct!
|
|
28
|
+
|
|
29
|
+
# BucketLoggingStatus XML.
|
|
30
|
+
xml.BucketLoggingStatus({"xmlns" => RESPONSE_NAMESPACE_URI}) {
|
|
31
|
+
unless @log_target.nil? and @log_prefix.nil?
|
|
32
|
+
xml.LoggingEnabled {
|
|
33
|
+
xml.TargetBucket @log_target
|
|
34
|
+
xml.TargetPrefix @log_prefix
|
|
35
|
+
}
|
|
36
|
+
end
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
xml_str
|
|
40
|
+
end
|
|
41
|
+
|
|
42
|
+
end
|
|
43
|
+
end
|
data/lib/s33r/named_bucket.rb
CHANGED
|
@@ -6,14 +6,14 @@ require File.join(File.dirname(__FILE__), 'client')
|
|
|
6
6
|
module S33r
|
|
7
7
|
# Wraps the S33r::Client class to make it more convenient for use with a single bucket.
|
|
8
8
|
class NamedBucket < Client
|
|
9
|
-
attr_accessor :
|
|
9
|
+
attr_accessor :name, :strict, :public_contents, :dump_requests
|
|
10
10
|
|
|
11
11
|
# Initialize an instance from a config_file. The config. file can include a separate
|
|
12
12
|
# +options+ section specifying options specific to NamedBucket instances (see the initialize method
|
|
13
13
|
# for more details).
|
|
14
14
|
# Other options are as for S33r::Client.init.
|
|
15
15
|
def NamedBucket.init(config_file)
|
|
16
|
-
aws_access_key, aws_secret_access_key, options
|
|
16
|
+
aws_access_key, aws_secret_access_key, options = super.class.load_config(config_file)
|
|
17
17
|
NamedBucket.new(aws_access_key, aws_secret_access_key, options)
|
|
18
18
|
end
|
|
19
19
|
|
|
@@ -27,21 +27,21 @@ module S33r
|
|
|
27
27
|
def initialize(aws_access_key, aws_secret_access_key, options={}, &block)
|
|
28
28
|
super(aws_access_key, aws_secret_access_key, options)
|
|
29
29
|
|
|
30
|
-
@
|
|
31
|
-
if @
|
|
30
|
+
@name = options[:default_bucket]
|
|
31
|
+
if @name.nil?
|
|
32
32
|
raise S33rException::MissingBucketName, "NamedBucket cannot be initialised without specifying\
|
|
33
33
|
a :default_bucket option"
|
|
34
34
|
end
|
|
35
35
|
|
|
36
36
|
# holds a BucketListing instance
|
|
37
|
-
@
|
|
37
|
+
@listing = nil
|
|
38
38
|
|
|
39
39
|
# all content should be created as public-read
|
|
40
40
|
@public_contents = (true == options[:public_contents])
|
|
41
41
|
@client_headers.merge!(canned_acl_header('public-read')) if @public_contents
|
|
42
42
|
|
|
43
43
|
@strict = (true == options[:strict])
|
|
44
|
-
if @strict && !bucket_exists?(@
|
|
44
|
+
if @strict && !bucket_exists?(@name)
|
|
45
45
|
raise S33rException::MissingResource, "Non-existent bucket #{@bucket_name} specified"
|
|
46
46
|
end
|
|
47
47
|
|
|
@@ -60,24 +60,23 @@ module S33r
|
|
|
60
60
|
|
|
61
61
|
# Get a single object from a bucket as a blob.
|
|
62
62
|
def [](key)
|
|
63
|
-
get_resource(@
|
|
63
|
+
get_resource(@name, key).body
|
|
64
64
|
end
|
|
65
65
|
|
|
66
66
|
# Get a BucketListing instance for the content of this bucket.
|
|
67
67
|
def listing
|
|
68
|
-
|
|
69
|
-
@bucket_listing
|
|
68
|
+
list_bucket(@name)
|
|
70
69
|
end
|
|
71
70
|
|
|
72
71
|
# Does this bucket exist?
|
|
73
72
|
# Returns true if the bucket this NamedBucket is mapped to exists.
|
|
74
73
|
def exists?
|
|
75
|
-
bucket_exists?(@
|
|
74
|
+
bucket_exists?(@name)
|
|
76
75
|
end
|
|
77
76
|
|
|
78
77
|
# Delete the bucket.
|
|
79
78
|
def destroy(headers={}, options={})
|
|
80
|
-
delete_bucket(@
|
|
79
|
+
delete_bucket(@name, headers, options)
|
|
81
80
|
end
|
|
82
81
|
|
|
83
82
|
# Get a pretty list of the keys in the bucket.
|
|
@@ -93,22 +92,22 @@ module S33r
|
|
|
93
92
|
# Does the given key exist in the bucket?
|
|
94
93
|
# Returns boolean
|
|
95
94
|
def key_exists?(key)
|
|
96
|
-
|
|
95
|
+
resource_exists?(@name, key)
|
|
97
96
|
end
|
|
98
97
|
|
|
99
98
|
# Put a string into a key inside the bucket.
|
|
100
99
|
def put_text(string, resource_key, headers={})
|
|
101
|
-
super(string, @
|
|
100
|
+
super(string, @name, resource_key, headers)
|
|
102
101
|
end
|
|
103
102
|
|
|
104
103
|
# Put a file into the bucket.
|
|
105
104
|
def put_file(filename, resource_key=nil, headers={}, options={})
|
|
106
|
-
super(filename, @
|
|
105
|
+
super(filename, @name, resource_key, headers, options)
|
|
107
106
|
end
|
|
108
107
|
|
|
109
108
|
# Put a generic stream (e.g. from a file handle) into the bucket.
|
|
110
109
|
def put_stream(data, resource_key, headers={})
|
|
111
|
-
put_resource(@
|
|
110
|
+
put_resource(@name, resource_key, data, headers)
|
|
112
111
|
end
|
|
113
112
|
|
|
114
113
|
# Delete an object from the bucket.
|
|
@@ -116,7 +115,7 @@ module S33r
|
|
|
116
115
|
# and trying to delete a non-existent key (both return a 204).
|
|
117
116
|
# If you want to test for existence first, use key_exists?.
|
|
118
117
|
def delete_resource(resource_key, headers={})
|
|
119
|
-
super(@
|
|
118
|
+
super(@name, resource_key, headers)
|
|
120
119
|
listing
|
|
121
120
|
end
|
|
122
121
|
|
|
@@ -125,7 +124,7 @@ module S33r
|
|
|
125
124
|
#
|
|
126
125
|
# +expires+: time in secs since the epoch when the link should become invalid.
|
|
127
126
|
def s3_authenticated_url(resource_key, expires=(Time.now.to_i + DEFAULT_EXPIRY_SECS))
|
|
128
|
-
super(@aws_access_key, @aws_secret_access_key, @
|
|
127
|
+
super(@aws_access_key, @aws_secret_access_key, @name, resource_key, expires)
|
|
129
128
|
end
|
|
130
129
|
end
|
|
131
130
|
end
|
data/lib/s33r/s33r_exception.rb
CHANGED
|
@@ -30,6 +30,17 @@ module S33r
|
|
|
30
30
|
|
|
31
31
|
class S3FallenOver < Exception
|
|
32
32
|
end
|
|
33
|
+
|
|
34
|
+
class InvalidS3GroupType < Exception
|
|
35
|
+
end
|
|
36
|
+
|
|
37
|
+
class InvalidPermission < Exception
|
|
38
|
+
end
|
|
39
|
+
|
|
40
|
+
# Raised if a bucket cannot be used as a log target
|
|
41
|
+
# (i.e. if LogDelivery permissions not set - see S33r::S3ACL::ACLDoc.add_log_target_grants)
|
|
42
|
+
class BucketNotLogTargetable < Exception
|
|
43
|
+
end
|
|
33
44
|
|
|
34
45
|
end
|
|
35
46
|
end
|
data/lib/s33r/s33r_http.rb
CHANGED
|
@@ -9,8 +9,9 @@ class Net::HTTPResponse
|
|
|
9
9
|
attr_writer :body
|
|
10
10
|
|
|
11
11
|
def check_s3_availability
|
|
12
|
-
raise S33rException::S3FallenOver, "S3 has fallen over! (got a #{code} code in response)" \
|
|
13
12
|
if (500..503).include? code.to_i
|
|
13
|
+
raise S33rException::S3FallenOver, "S3 has fallen over! (got a #{code} code in response)"
|
|
14
|
+
end
|
|
14
15
|
end
|
|
15
16
|
|
|
16
17
|
def ok?
|
data/lib/s33r/s3_acl.rb
ADDED
|
@@ -0,0 +1,337 @@
|
|
|
1
|
+
require 'rubygems'
|
|
2
|
+
require 'xml/libxml'
|
|
3
|
+
require_gem 'builder'
|
|
4
|
+
require File.join(File.dirname(__FILE__), 's33r_exception')
|
|
5
|
+
|
|
6
|
+
module S33r
|
|
7
|
+
# S3 ACL handling.
|
|
8
|
+
#
|
|
9
|
+
# NB an individual ACL for an object can only contain <= 100 grants.
|
|
10
|
+
module S3ACL
|
|
11
|
+
|
|
12
|
+
# An S3 ACL document, incorporating one or more Grants
|
|
13
|
+
# (see http://docs.amazonwebservices.com/AmazonS3/2006-03-01/UsingACL.html).
|
|
14
|
+
#
|
|
15
|
+
# Represents both retrieved ACL XML or can be built up
|
|
16
|
+
# using objects and converted to XML.
|
|
17
|
+
# NB the ACLDoc is oblivious to the resource it is going
|
|
18
|
+
# to be applied to.
|
|
19
|
+
class ACLDoc
|
|
20
|
+
# List of grants to be applied.
|
|
21
|
+
attr_accessor :grants, :owner
|
|
22
|
+
|
|
23
|
+
# +owner+: S33r::S3ACL::CanonicalUser instance
|
|
24
|
+
def initialize(owner, grants=[])
|
|
25
|
+
@grants = grants
|
|
26
|
+
@owner = owner
|
|
27
|
+
end
|
|
28
|
+
|
|
29
|
+
# Create an ACLDoc instance from a raw Access Control Policy XML document.
|
|
30
|
+
#
|
|
31
|
+
# +acl_xml+ is a raw Access Control Policy XML string (NOT libxml Document or Node).
|
|
32
|
+
#
|
|
33
|
+
# Returns nil if the ACL XML is nil.
|
|
34
|
+
def self.from_xml(acl_xml)
|
|
35
|
+
return nil if acl_xml.nil?
|
|
36
|
+
|
|
37
|
+
acl_xml = S33r.remove_namespace(acl_xml)
|
|
38
|
+
doc = XML.get_xml_doc(acl_xml)
|
|
39
|
+
|
|
40
|
+
owner_xml = doc.find('//Owner').to_a.first
|
|
41
|
+
owner = CanonicalUser.from_xml(owner_xml)
|
|
42
|
+
|
|
43
|
+
grants = []
|
|
44
|
+
doc.find('//AccessControlList/Grant').to_a.each do |g|
|
|
45
|
+
grantee_xml = g.find('Grantee').to_a.first
|
|
46
|
+
grantee = Grantee.from_xml(grantee_xml)
|
|
47
|
+
permission = g.xget('Permission')
|
|
48
|
+
|
|
49
|
+
grants << Grant.new(grantee, permission)
|
|
50
|
+
end
|
|
51
|
+
|
|
52
|
+
ACLDoc.new(owner, grants)
|
|
53
|
+
end
|
|
54
|
+
|
|
55
|
+
# Generate AccessControlPolicy XML document.
|
|
56
|
+
def to_xml
|
|
57
|
+
xml_str = ""
|
|
58
|
+
xml = Builder::XmlMarkup.new(:target => xml_str, :indent => 0)
|
|
59
|
+
|
|
60
|
+
xml.instruct!
|
|
61
|
+
|
|
62
|
+
# Access control policy XML.
|
|
63
|
+
xml.AccessControlPolicy({"xmlns" => RESPONSE_NAMESPACE_URI}) {
|
|
64
|
+
xml.Owner {
|
|
65
|
+
xml.ID owner.user_id
|
|
66
|
+
xml.DisplayName owner.display_name
|
|
67
|
+
}
|
|
68
|
+
xml.AccessControlList {
|
|
69
|
+
grants.each do |grant|
|
|
70
|
+
xml << grant.to_xml
|
|
71
|
+
end
|
|
72
|
+
}
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
xml_str
|
|
76
|
+
end
|
|
77
|
+
|
|
78
|
+
# Add a grant to the ACL document.
|
|
79
|
+
#
|
|
80
|
+
# Returns true if grant was added;
|
|
81
|
+
# false otherwise (grant already exists).
|
|
82
|
+
def add_grant(grant)
|
|
83
|
+
if @grants.include?(grant)
|
|
84
|
+
return false
|
|
85
|
+
else
|
|
86
|
+
@grants << grant
|
|
87
|
+
return true
|
|
88
|
+
end
|
|
89
|
+
end
|
|
90
|
+
|
|
91
|
+
# Remove a grant from the ACL document. Note that if you
|
|
92
|
+
# set a grant for an AmazonCustomer, you want be able to remove it by
|
|
93
|
+
# specifying the same grant. This is because grants set by AmazonCustomer
|
|
94
|
+
# are converted at the S3 end into CanonicalUser grants - so you will need
|
|
95
|
+
# to remove a CanonicalUser grant instead. See Grant.for_amazon_customer
|
|
96
|
+
# for a few more details.
|
|
97
|
+
#
|
|
98
|
+
# Returns true if grant was removed;
|
|
99
|
+
# false if it wasn't in the document.
|
|
100
|
+
def remove_grant(grant)
|
|
101
|
+
@grants.delete_if { |g| grant == g }
|
|
102
|
+
end
|
|
103
|
+
|
|
104
|
+
# Does the ACL contain a grant for public reads?
|
|
105
|
+
# (i.e. grants holds a Grant object for :all_users with :read permission)
|
|
106
|
+
def public_readable?
|
|
107
|
+
pr_grant = Grant.public_read_grant
|
|
108
|
+
grants.each do |g|
|
|
109
|
+
return true if pr_grant == g
|
|
110
|
+
end
|
|
111
|
+
return false
|
|
112
|
+
end
|
|
113
|
+
|
|
114
|
+
# Add a public READ permission to this instance.
|
|
115
|
+
def add_public_read_grants
|
|
116
|
+
add_grant(Grant.public_read_grant)
|
|
117
|
+
end
|
|
118
|
+
|
|
119
|
+
# Does the ACL make the associated resource available as a log target?
|
|
120
|
+
def log_targetable?
|
|
121
|
+
log_target_grants = Grant.log_target_grants
|
|
122
|
+
log_target_grants.each { |g| return false if !grants.include?(g) }
|
|
123
|
+
return true
|
|
124
|
+
end
|
|
125
|
+
|
|
126
|
+
# Add permissions to an instances which give READ_ACL
|
|
127
|
+
# and WRITE permissions to the LogDelivery group. Used
|
|
128
|
+
# to enable a bucket as a logging destination.
|
|
129
|
+
#
|
|
130
|
+
# Returns true if grants added, false otherwise
|
|
131
|
+
# (if already a log target).
|
|
132
|
+
def add_log_target_grants
|
|
133
|
+
if log_targetable?
|
|
134
|
+
return false
|
|
135
|
+
else
|
|
136
|
+
Grant.log_target_grants.each { |g| add_grant(g) }
|
|
137
|
+
return true
|
|
138
|
+
end
|
|
139
|
+
end
|
|
140
|
+
|
|
141
|
+
# Remove log target ACLs from the document.
|
|
142
|
+
#
|
|
143
|
+
# Returns true if all log target grants were removed;
|
|
144
|
+
# false otherwise.
|
|
145
|
+
#
|
|
146
|
+
# NB even if this method returns false, that doesn't mean
|
|
147
|
+
# the bucket is still a log target. Use log_targetable? to check
|
|
148
|
+
# whether a bucket can be used as a log target.
|
|
149
|
+
def remove_log_target_grants
|
|
150
|
+
ok = true
|
|
151
|
+
Grant.log_target_grants.each { |g| ok = ok and remove_grant(g) }
|
|
152
|
+
ok
|
|
153
|
+
end
|
|
154
|
+
|
|
155
|
+
end
|
|
156
|
+
|
|
157
|
+
# Representation of an S3 Grant
|
|
158
|
+
# (see http://docs.amazonwebservices.com/AmazonS3/2006-03-01/UsingGrantees.html).
|
|
159
|
+
#
|
|
160
|
+
# A Grant consists of a Grantee and a permission they are to be
|
|
161
|
+
# assigned.
|
|
162
|
+
class Grant
|
|
163
|
+
attr_accessor :grantee, :permission
|
|
164
|
+
|
|
165
|
+
# permission: one of the keys in the PERMISSIONS hash or a raw permission string
|
|
166
|
+
def initialize(grantee, permission)
|
|
167
|
+
@grantee = grantee
|
|
168
|
+
if permission.is_a? String
|
|
169
|
+
@permission = permission
|
|
170
|
+
else
|
|
171
|
+
@permission = PERMISSIONS[permission]
|
|
172
|
+
end
|
|
173
|
+
raise InvalidPermission, "Permission #{permission.to_s} is not a valid permission specifier" if @permission.nil?
|
|
174
|
+
end
|
|
175
|
+
|
|
176
|
+
# Note that setting a grant for an Amazon customer is the
|
|
177
|
+
# same as setting a grant for the CanonicalUser who owns the
|
|
178
|
+
# specified email address. So when you get the ACL back, it will
|
|
179
|
+
# actually contain a CanonicalUser grant.
|
|
180
|
+
def Grant.for_amazon_customer(email_address, permission)
|
|
181
|
+
Grant.new(AmazonCustomer.new(email_address), permission)
|
|
182
|
+
end
|
|
183
|
+
|
|
184
|
+
def Grant.for_canonical_user(id, display_name, permission)
|
|
185
|
+
Grant.new(CanonicalUser.new(id, display_name), permission)
|
|
186
|
+
end
|
|
187
|
+
|
|
188
|
+
def Grant.for_group(group_type, permission)
|
|
189
|
+
Grant.new(Group.new(group_type), permission)
|
|
190
|
+
end
|
|
191
|
+
|
|
192
|
+
# Generator for a Grant which gives READ permissions to the AllUsers
|
|
193
|
+
# group type.
|
|
194
|
+
def Grant.public_read_grant
|
|
195
|
+
Grant.new(Group.new(:all_users), :read)
|
|
196
|
+
end
|
|
197
|
+
|
|
198
|
+
# Generator for a grant which gives the LogDelivery group
|
|
199
|
+
# write and read_acl permissions on a bucket.
|
|
200
|
+
#
|
|
201
|
+
# Returns an array with the two required Grant instances.
|
|
202
|
+
def Grant.log_target_grants
|
|
203
|
+
log_delivery_group = Group.new(:log_delivery)
|
|
204
|
+
[Grant.new(log_delivery_group, :read_acl), Grant.new(log_delivery_group, :write)]
|
|
205
|
+
end
|
|
206
|
+
|
|
207
|
+
# Convert a Grant object into an XML fragment.
|
|
208
|
+
def to_xml
|
|
209
|
+
xml_str = ""
|
|
210
|
+
xml = S33r::OrderlyXmlMarkup.new(:target => xml_str, :indent => 0)
|
|
211
|
+
|
|
212
|
+
# <Grant> element.
|
|
213
|
+
xml.Grant {
|
|
214
|
+
xml.Grantee({"xmlns:#{NAMESPACE}" => NAMESPACE_URI, "xsi:type" => @grantee.grantee_type}) {
|
|
215
|
+
case @grantee.grantee_type
|
|
216
|
+
when GRANTEE_TYPES[:amazon_customer]
|
|
217
|
+
xml.EmailAddress @grantee.email_address
|
|
218
|
+
when GRANTEE_TYPES[:canonical_user]
|
|
219
|
+
xml.ID @grantee.user_id
|
|
220
|
+
xml.DisplayName @grantee.display_name
|
|
221
|
+
when GRANTEE_TYPES[:group]
|
|
222
|
+
xml.URI GROUP_ACL_URI_BASE + @grantee.group_type
|
|
223
|
+
end
|
|
224
|
+
}
|
|
225
|
+
xml.Permission @permission
|
|
226
|
+
}
|
|
227
|
+
|
|
228
|
+
xml_str
|
|
229
|
+
end
|
|
230
|
+
|
|
231
|
+
def ==(obj)
|
|
232
|
+
if !obj.is_a?(Grant)
|
|
233
|
+
return false
|
|
234
|
+
end
|
|
235
|
+
if obj.permission != self.permission or obj.grantee != self.grantee
|
|
236
|
+
return false
|
|
237
|
+
end
|
|
238
|
+
return true
|
|
239
|
+
end
|
|
240
|
+
|
|
241
|
+
end
|
|
242
|
+
|
|
243
|
+
# Abstract representation of an S3 Grantee.
|
|
244
|
+
class Grantee
|
|
245
|
+
attr_reader :grantee_type
|
|
246
|
+
|
|
247
|
+
def ==(obj)
|
|
248
|
+
if !obj.is_a?(Grantee)
|
|
249
|
+
return false
|
|
250
|
+
end
|
|
251
|
+
instance_variables.each do |var|
|
|
252
|
+
method_name = var.gsub(/^@/, '')
|
|
253
|
+
if self.send(method_name) != obj.send(method_name)
|
|
254
|
+
return false
|
|
255
|
+
end
|
|
256
|
+
end
|
|
257
|
+
return true
|
|
258
|
+
end
|
|
259
|
+
|
|
260
|
+
def method_missing(*args)
|
|
261
|
+
nil
|
|
262
|
+
end
|
|
263
|
+
|
|
264
|
+
def self.from_xml(grantee_xml)
|
|
265
|
+
grantee_type = grantee_xml['type']
|
|
266
|
+
|
|
267
|
+
case grantee_type
|
|
268
|
+
when GRANTEE_TYPES[:amazon_customer]
|
|
269
|
+
AmazonCustomer.new(grantee_xml.xget('EmailAddress'))
|
|
270
|
+
when GRANTEE_TYPES[:canonical_user]
|
|
271
|
+
CanonicalUser.from_xml(grantee_xml)
|
|
272
|
+
when GRANTEE_TYPES[:group]
|
|
273
|
+
uri = grantee_xml.xget('URI')
|
|
274
|
+
# last part of the path is the group type
|
|
275
|
+
path = uri.gsub(/#{GROUP_ACL_URI_BASE}/, '')
|
|
276
|
+
|
|
277
|
+
group_type = :all_users
|
|
278
|
+
S3_GROUP_TYPES.each { |k,v| group_type = k if v == path }
|
|
279
|
+
Group.new(group_type)
|
|
280
|
+
end
|
|
281
|
+
end
|
|
282
|
+
|
|
283
|
+
end
|
|
284
|
+
|
|
285
|
+
# An Amazon customer for the purposes of assigning permissions.
|
|
286
|
+
class AmazonCustomer < Grantee
|
|
287
|
+
attr_accessor :email_address
|
|
288
|
+
|
|
289
|
+
def initialize(email_address)
|
|
290
|
+
@grantee_type = GRANTEE_TYPES[:amazon_customer]
|
|
291
|
+
@email_address = email_address
|
|
292
|
+
end
|
|
293
|
+
end
|
|
294
|
+
|
|
295
|
+
# An S3 user.
|
|
296
|
+
class CanonicalUser < Grantee
|
|
297
|
+
attr_accessor :user_id, :display_name
|
|
298
|
+
|
|
299
|
+
# +owner_xml_doc+: XML::Document or Node instance, representing an <Owner> node from
|
|
300
|
+
# inside a ListBucketResult <Contents> element
|
|
301
|
+
# (see http://docs.amazonwebservices.com/AmazonS3/2006-03-01/)
|
|
302
|
+
# or an ACL <Grantee> element.
|
|
303
|
+
def initialize(user_id, display_name)
|
|
304
|
+
@grantee_type = GRANTEE_TYPES[:canonical_user]
|
|
305
|
+
@user_id = user_id
|
|
306
|
+
@display_name = display_name
|
|
307
|
+
end
|
|
308
|
+
|
|
309
|
+
# Create a user object from an XML fragment with
|
|
310
|
+
# ID and DisplayName elements. (Both ACL documents and
|
|
311
|
+
# bucket listings represent users this way.)
|
|
312
|
+
def self.from_xml(user_xml_doc)
|
|
313
|
+
user_id = user_xml_doc.xget('ID')
|
|
314
|
+
display_name = user_xml_doc.xget('DisplayName')
|
|
315
|
+
new(user_id, display_name)
|
|
316
|
+
end
|
|
317
|
+
end
|
|
318
|
+
|
|
319
|
+
# One of the predefined S3 groups.
|
|
320
|
+
#
|
|
321
|
+
# A group must have a type (AllUsers or AuthenticatedUsers).
|
|
322
|
+
class Group < Grantee
|
|
323
|
+
attr_accessor :group_type
|
|
324
|
+
|
|
325
|
+
# The type of group. A key from S3_GROUP_TYPES to
|
|
326
|
+
# one of the pre-defined Amazon group types.
|
|
327
|
+
def initialize(group_type)
|
|
328
|
+
unless S3_GROUP_TYPES.has_key?(group_type)
|
|
329
|
+
raise InvalidS3GroupType, 'No such group type #{group_type}'
|
|
330
|
+
end
|
|
331
|
+
@group_type = S3_GROUP_TYPES[group_type]
|
|
332
|
+
@grantee_type = GRANTEE_TYPES[:group]
|
|
333
|
+
end
|
|
334
|
+
end
|
|
335
|
+
|
|
336
|
+
end
|
|
337
|
+
end
|