stevequinlan-aws-s3 0.6.3

Sign up to get free protection for your applications and to get access to all the features.
Files changed (58) hide show
  1. data/COPYING +19 -0
  2. data/INSTALL +55 -0
  3. data/README.rdoc +545 -0
  4. data/Rakefile +120 -0
  5. data/bin/s3sh +6 -0
  6. data/bin/setup.rb +10 -0
  7. data/lib/aws/s3.rb +60 -0
  8. data/lib/aws/s3/acl.rb +636 -0
  9. data/lib/aws/s3/authentication.rb +221 -0
  10. data/lib/aws/s3/base.rb +240 -0
  11. data/lib/aws/s3/bittorrent.rb +58 -0
  12. data/lib/aws/s3/bucket.rb +319 -0
  13. data/lib/aws/s3/connection.rb +287 -0
  14. data/lib/aws/s3/error.rb +69 -0
  15. data/lib/aws/s3/exceptions.rb +133 -0
  16. data/lib/aws/s3/extensions.rb +356 -0
  17. data/lib/aws/s3/logging.rb +314 -0
  18. data/lib/aws/s3/object.rb +612 -0
  19. data/lib/aws/s3/owner.rb +44 -0
  20. data/lib/aws/s3/parsing.rb +99 -0
  21. data/lib/aws/s3/response.rb +180 -0
  22. data/lib/aws/s3/service.rb +51 -0
  23. data/lib/aws/s3/version.rb +12 -0
  24. data/support/faster-xml-simple/lib/faster_xml_simple.rb +187 -0
  25. data/support/faster-xml-simple/test/regression_test.rb +47 -0
  26. data/support/faster-xml-simple/test/test_helper.rb +17 -0
  27. data/support/faster-xml-simple/test/xml_simple_comparison_test.rb +46 -0
  28. data/support/rdoc/code_info.rb +211 -0
  29. data/test/acl_test.rb +254 -0
  30. data/test/authentication_test.rb +114 -0
  31. data/test/base_test.rb +136 -0
  32. data/test/bucket_test.rb +74 -0
  33. data/test/connection_test.rb +215 -0
  34. data/test/error_test.rb +70 -0
  35. data/test/extensions_test.rb +341 -0
  36. data/test/fixtures.rb +89 -0
  37. data/test/fixtures/buckets.yml +133 -0
  38. data/test/fixtures/errors.yml +34 -0
  39. data/test/fixtures/headers.yml +3 -0
  40. data/test/fixtures/logging.yml +15 -0
  41. data/test/fixtures/loglines.yml +5 -0
  42. data/test/fixtures/logs.yml +7 -0
  43. data/test/fixtures/policies.yml +16 -0
  44. data/test/logging_test.rb +89 -0
  45. data/test/mocks/fake_response.rb +26 -0
  46. data/test/object_test.rb +205 -0
  47. data/test/parsing_test.rb +66 -0
  48. data/test/remote/acl_test.rb +117 -0
  49. data/test/remote/bittorrent_test.rb +45 -0
  50. data/test/remote/bucket_test.rb +146 -0
  51. data/test/remote/logging_test.rb +82 -0
  52. data/test/remote/object_test.rb +371 -0
  53. data/test/remote/test_file.data +0 -0
  54. data/test/remote/test_helper.rb +33 -0
  55. data/test/response_test.rb +68 -0
  56. data/test/service_test.rb +23 -0
  57. data/test/test_helper.rb +110 -0
  58. metadata +159 -0
@@ -0,0 +1,69 @@
1
+ module AWS
2
+ module S3
3
+ # Anything you do that makes a request to S3 could result in an error. If it does, the AWS::S3 library will raise an exception
4
+ # specific to the error. All exception that are raised as a result of a request returning an error response inherit from the
5
+ # ResponseError exception. So should you choose to rescue any such exception, you can simple rescue ResponseError.
6
+ #
7
+ # Say you go to delete a bucket, but the bucket turns out to not be empty. This results in a BucketNotEmpty error (one of the many
8
+ # errors listed at http://docs.amazonwebservices.com/AmazonS3/2006-03-01/ErrorCodeList.html):
9
+ #
10
+ # begin
11
+ # Bucket.delete('jukebox')
12
+ # rescue ResponseError => error
13
+ # # ...
14
+ # end
15
+ #
16
+ # Once you've captured the exception, you can extract the error message from S3, as well as the full error response, which includes
17
+ # things like the HTTP response code:
18
+ #
19
+ # error
20
+ # # => #<AWS::S3::BucketNotEmpty The bucket you tried to delete is not empty>
21
+ # error.message
22
+ # # => "The bucket you tried to delete is not empty"
23
+ # error.response.code
24
+ # # => 409
25
+ #
26
+ # You could use this information to redisplay the error in a way you see fit, or just to log the error and continue on.
27
+ class Error
28
+ #:stopdoc:
29
+ attr_accessor :response
30
+ def initialize(error, response = nil)
31
+ @error = error
32
+ @response = response
33
+ @container = AWS::S3
34
+ find_or_create_exception!
35
+ end
36
+
37
+ def raise
38
+ Kernel.raise exception.new(message, response)
39
+ end
40
+
41
+ private
42
+ attr_reader :error, :exception, :container
43
+
44
+ def find_or_create_exception!
45
+ @exception = container.const_defined?(code) ? find_exception : create_exception
46
+ end
47
+
48
+ def find_exception
49
+ exception_class = container.const_get(code)
50
+ Kernel.raise ExceptionClassClash.new(exception_class) unless exception_class.ancestors.include?(ResponseError)
51
+ exception_class
52
+ end
53
+
54
+ def create_exception
55
+ container.const_set(code, Class.new(ResponseError))
56
+ end
57
+
58
+ def method_missing(method, *args, &block)
59
+ # We actually want nil if the attribute is nil. So we use has_key? rather than [] + ||.
60
+ if error.has_key?(method.to_s)
61
+ error[method.to_s]
62
+ else
63
+ super
64
+ end
65
+ end
66
+ end
67
+ end
68
+ end
69
+ #:startdoc:
@@ -0,0 +1,133 @@
1
+ module AWS
2
+ module S3
3
+
4
+ # Abstract super class of all AWS::S3 exceptions
5
+ class S3Exception < StandardError
6
+ end
7
+
8
+ # All responses with a code between 300 and 599 that contain an <Error></Error> body are wrapped in an
9
+ # ErrorResponse which contains an Error object. This Error class generates a custom exception with the name
10
+ # of the xml Error and its message. All such runtime generated exception classes descend from ResponseError
11
+ # and contain the ErrorResponse object so that all code that makes a request can rescue ResponseError and get
12
+ # access to the ErrorResponse.
13
+ class ResponseError < S3Exception
14
+ attr_reader :response
15
+ def initialize(message, response)
16
+ @response = response
17
+ super(message)
18
+ end
19
+ end
20
+
21
+ #:stopdoc:
22
+
23
+ # Most ResponseError's are created just time on a need to have basis, but we explicitly define the
24
+ # InternalError exception because we want to explicitly rescue InternalError in some cases.
25
+ class InternalError < ResponseError
26
+ end
27
+
28
+ class NoSuchKey < ResponseError
29
+ end
30
+
31
+ class RequestTimeout < ResponseError
32
+ end
33
+
34
+ # Abstract super class for all invalid options.
35
+ class InvalidOption < S3Exception
36
+ end
37
+
38
+ # Raised if an invalid value is passed to the <tt>:access</tt> option when creating a Bucket or an S3Object.
39
+ class InvalidAccessControlLevel < InvalidOption
40
+ def initialize(valid_levels, access_level)
41
+ super("Valid access control levels are #{valid_levels.inspect}. You specified `#{access_level}'.")
42
+ end
43
+ end
44
+
45
+ # Raised if either the access key id or secret access key arguments are missing when establishing a connection.
46
+ class MissingAccessKey < InvalidOption
47
+ def initialize(missing_keys)
48
+ key_list = missing_keys.map {|key| key.to_s}.join(' and the ')
49
+ super("You did not provide both required access keys. Please provide the #{key_list}.")
50
+ end
51
+ end
52
+
53
+ # Raised if a request is attempted before any connections have been established.
54
+ class NoConnectionEstablished < S3Exception
55
+ end
56
+
57
+ # Raised if an unrecognized option is passed when establishing a connection.
58
+ class InvalidConnectionOption < InvalidOption
59
+ def initialize(invalid_options)
60
+ message = "The following connection options are invalid: #{invalid_options.join(', ')}. " +
61
+ "The valid connection options are: #{Connection::Options::VALID_OPTIONS.join(', ')}."
62
+ super(message)
63
+ end
64
+ end
65
+
66
+ # Raised if an invalid bucket name is passed when creating a new Bucket.
67
+ class InvalidBucketName < S3Exception
68
+ def initialize(invalid_name)
69
+ message = "`#{invalid_name}' is not a valid bucket name. " +
70
+ "Bucket names must be between 3 and 255 bytes and " +
71
+ "can contain letters, numbers, dashes and underscores."
72
+ super(message)
73
+ end
74
+ end
75
+
76
+ # Raised if an invalid key name is passed when creating an S3Object.
77
+ class InvalidKeyName < S3Exception
78
+ def initialize(invalid_name)
79
+ message = "`#{invalid_name}' is not a valid key name. " +
80
+ "Key names must be no more than 1024 bytes long."
81
+ super(message)
82
+ end
83
+ end
84
+
85
+ # Raised if an invalid value is assigned to an S3Object's specific metadata name.
86
+ class InvalidMetadataValue < S3Exception
87
+ def initialize(invalid_names)
88
+ message = "The following metadata names have invalid values: #{invalid_names.join(', ')}. " +
89
+ "Metadata can not be larger than 2kilobytes."
90
+ super(message)
91
+ end
92
+ end
93
+
94
+ # Raised if the current bucket can not be inferred when not explicitly specifying the target bucket in the calling
95
+ # method's arguments.
96
+ class CurrentBucketNotSpecified < S3Exception
97
+ def initialize(address)
98
+ message = "No bucket name can be inferred from your current connection's address (`#{address}')"
99
+ super(message)
100
+ end
101
+ end
102
+
103
+ # Raised when an orphaned S3Object belonging to no bucket tries to access its (non-existant) bucket.
104
+ class NoBucketSpecified < S3Exception
105
+ def initialize
106
+ super('The current object must have its bucket set')
107
+ end
108
+ end
109
+
110
+ # Raised if an attempt is made to save an S3Object that does not have a key set.
111
+ class NoKeySpecified < S3Exception
112
+ def initialize
113
+ super('The current object must have its key set')
114
+ end
115
+ end
116
+
117
+ # Raised if you try to save a deleted object.
118
+ class DeletedObject < S3Exception
119
+ def initialize
120
+ super('You can not save a deleted object')
121
+ end
122
+ end
123
+
124
+ class ExceptionClassClash < S3Exception #:nodoc:
125
+ def initialize(klass)
126
+ message = "The exception class you tried to create (`#{klass}') exists and is not an exception"
127
+ super(message)
128
+ end
129
+ end
130
+
131
+ #:startdoc:
132
+ end
133
+ end
@@ -0,0 +1,356 @@
1
+ #:stopdoc:
2
+
3
+ class Hash
4
+ def to_query_string(include_question_mark = true)
5
+ query_string = ''
6
+ unless empty?
7
+ query_string << '?' if include_question_mark
8
+ query_string << inject([]) do |params, (key, value)|
9
+ params << "#{key}=#{value}"
10
+ end.join('&')
11
+ end
12
+ query_string
13
+ end
14
+
15
+ def to_normalized_options
16
+ # Convert all option names to downcased strings, and replace underscores with hyphens
17
+ inject({}) do |normalized_options, (name, value)|
18
+ normalized_options[name.to_header] = value.to_s
19
+ normalized_options
20
+ end
21
+ end
22
+
23
+ def to_normalized_options!
24
+ replace(to_normalized_options)
25
+ end
26
+ end
27
+
28
+ class String
29
+ if RUBY_VERSION <= '1.9'
30
+ def previous!
31
+ self[-1] -= 1
32
+ self
33
+ end
34
+ else
35
+ def previous!
36
+ self[-1] = (self[-1].ord - 1).chr
37
+ self
38
+ end
39
+ end
40
+
41
+ def previous
42
+ dup.previous!
43
+ end
44
+
45
+ def to_header
46
+ downcase.tr('_', '-')
47
+ end
48
+
49
+ # ActiveSupport adds an underscore method to String so let's just use that one if
50
+ # we find that the method is already defined
51
+ def underscore
52
+ gsub(/::/, '/').
53
+ gsub(/([A-Z]+)([A-Z][a-z])/,'\1_\2').
54
+ gsub(/([a-z\d])([A-Z])/,'\1_\2').
55
+ tr("-", "_").downcase
56
+ end unless public_method_defined? :underscore
57
+
58
+ if RUBY_VERSION >= '1.9'
59
+ def valid_utf8?
60
+ dup.force_encoding('UTF-8').valid_encoding?
61
+ end
62
+ else
63
+ def valid_utf8?
64
+ scan(Regexp.new('[^\x00-\xa0]', nil, 'u')) { |s| s.unpack('U') }
65
+ true
66
+ rescue ArgumentError
67
+ false
68
+ end
69
+ end
70
+
71
+ # All paths in in S3 have to be valid unicode so this takes care of
72
+ # cleaning up any strings that aren't valid utf-8 according to String#valid_utf8?
73
+ if RUBY_VERSION >= '1.9'
74
+ def remove_extended!
75
+ sanitized_string = ''
76
+ each_byte do |byte|
77
+ character = byte.chr
78
+ sanitized_string << character if character.ascii_only?
79
+ end
80
+ sanitized_string
81
+ end
82
+ else
83
+ def remove_extended!
84
+ gsub!(/[\x80-\xFF]/) { "%02X" % $&[0] }
85
+ end
86
+ end
87
+
88
+ def remove_extended
89
+ dup.remove_extended!
90
+ end
91
+ end
92
+
93
+ class CoercibleString < String
94
+ class << self
95
+ def coerce(string)
96
+ new(string).coerce
97
+ end
98
+ end
99
+
100
+ def coerce
101
+ case self
102
+ when 'true'; true
103
+ when 'false'; false
104
+ # Don't coerce numbers that start with zero
105
+ when /^[1-9]+\d*$/; Integer(self)
106
+ when datetime_format; Time.parse(self)
107
+ else
108
+ self
109
+ end
110
+ end
111
+
112
+ private
113
+ # Lame hack since Date._parse is so accepting. S3 dates are of the form: '2006-10-29T23:14:47.000Z'
114
+ # so unless the string looks like that, don't even try, otherwise it might convert an object's
115
+ # key from something like '03 1-2-3-Apple-Tree.mp3' to Sat Feb 03 00:00:00 CST 2001.
116
+ def datetime_format
117
+ /^\d{4}-\d{2}-\d{2}\w\d{2}:\d{2}:\d{2}/
118
+ end
119
+ end
120
+
121
+ class Symbol
122
+ def to_header
123
+ to_s.to_header
124
+ end
125
+ end
126
+
127
+ module Kernel
128
+ def __method__(depth = 0)
129
+ caller[depth][/`([^']+)'/, 1]
130
+ end if RUBY_VERSION <= '1.8.7'
131
+
132
+ def __called_from__
133
+ caller[1][/`([^']+)'/, 1]
134
+ end if RUBY_VERSION > '1.8.7'
135
+
136
+ def expirable_memoize(reload = false, storage = nil)
137
+ current_method = RUBY_VERSION > '1.8.7' ? __called_from__ : __method__(1)
138
+ storage = "@#{storage || current_method}"
139
+ if reload
140
+ instance_variable_set(storage, nil)
141
+ else
142
+ if cache = instance_variable_get(storage)
143
+ return cache
144
+ end
145
+ end
146
+ instance_variable_set(storage, yield)
147
+ end
148
+
149
+ def require_library_or_gem(library, gem_name = nil)
150
+ if RUBY_VERSION >= '1.9'
151
+ gem(gem_name || library, '>=0')
152
+ end
153
+ require library
154
+ rescue LoadError => library_not_installed
155
+ begin
156
+ require 'rubygems'
157
+ require library
158
+ rescue LoadError
159
+ raise library_not_installed
160
+ end
161
+ end
162
+ end
163
+
164
+ class Object
165
+ def returning(value)
166
+ yield(value)
167
+ value
168
+ end
169
+ end
170
+
171
+ class Module
172
+ def memoized(method_name)
173
+ original_method = "unmemoized_#{method_name}_#{Time.now.to_i}"
174
+ alias_method original_method, method_name
175
+ module_eval(<<-EVAL, __FILE__, __LINE__)
176
+ def #{method_name}(reload = false, *args, &block)
177
+ expirable_memoize(reload) do
178
+ send(:#{original_method}, *args, &block)
179
+ end
180
+ end
181
+ EVAL
182
+ end
183
+
184
+ def constant(name, value)
185
+ unless const_defined?(name)
186
+ const_set(name, value)
187
+ module_eval(<<-EVAL, __FILE__, __LINE__)
188
+ def self.#{name.to_s.downcase}
189
+ #{name.to_s}
190
+ end
191
+ EVAL
192
+ end
193
+ end
194
+
195
+ # Transforms MarcelBucket into
196
+ #
197
+ # class MarcelBucket < AWS::S3::Bucket
198
+ # set_current_bucket_to 'marcel'
199
+ # end
200
+ def const_missing_from_s3_library(sym)
201
+ if sym.to_s =~ /^(\w+)(Bucket|S3Object)$/
202
+ const = const_set(sym, Class.new(AWS::S3.const_get($2)))
203
+ const.current_bucket = $1.underscore
204
+ const
205
+ else
206
+ const_missing_not_from_s3_library(sym)
207
+ end
208
+ end
209
+ alias_method :const_missing_not_from_s3_library, :const_missing
210
+ alias_method :const_missing, :const_missing_from_s3_library
211
+ end
212
+
213
+
214
+ class Class # :nodoc:
215
+ def cattr_reader(*syms)
216
+ syms.flatten.each do |sym|
217
+ class_eval(<<-EOS, __FILE__, __LINE__)
218
+ unless defined? @@#{sym}
219
+ @@#{sym} = nil
220
+ end
221
+
222
+ def self.#{sym}
223
+ @@#{sym}
224
+ end
225
+
226
+ def #{sym}
227
+ @@#{sym}
228
+ end
229
+ EOS
230
+ end
231
+ end
232
+
233
+ def cattr_writer(*syms)
234
+ syms.flatten.each do |sym|
235
+ class_eval(<<-EOS, __FILE__, __LINE__)
236
+ unless defined? @@#{sym}
237
+ @@#{sym} = nil
238
+ end
239
+
240
+ def self.#{sym}=(obj)
241
+ @@#{sym} = obj
242
+ end
243
+
244
+ def #{sym}=(obj)
245
+ @@#{sym} = obj
246
+ end
247
+ EOS
248
+ end
249
+ end
250
+
251
+ def cattr_accessor(*syms)
252
+ cattr_reader(*syms)
253
+ cattr_writer(*syms)
254
+ end
255
+ end if Class.instance_methods(false).grep(/^cattr_(?:reader|writer|accessor)$/).empty?
256
+
257
+ module SelectiveAttributeProxy
258
+ def self.included(klass)
259
+ klass.extend(ClassMethods)
260
+ klass.class_eval(<<-EVAL, __FILE__, __LINE__)
261
+ cattr_accessor :attribute_proxy
262
+ cattr_accessor :attribute_proxy_options
263
+
264
+ # Default name for attribute storage
265
+ self.attribute_proxy = :attributes
266
+ self.attribute_proxy_options = {:exclusively => true}
267
+
268
+ private
269
+ # By default proxy all attributes
270
+ def proxiable_attribute?(name)
271
+ return true unless self.class.attribute_proxy_options[:exclusively]
272
+ send(self.class.attribute_proxy).has_key?(name)
273
+ end
274
+
275
+ def method_missing(method, *args, &block)
276
+ # Autovivify attribute storage
277
+ if method == self.class.attribute_proxy
278
+ ivar = "@\#{method}"
279
+ instance_variable_set(ivar, {}) unless instance_variable_get(ivar).is_a?(Hash)
280
+ instance_variable_get(ivar)
281
+ # Delegate to attribute storage
282
+ elsif method.to_s =~ /^(\\w+)(=?)$/ && proxiable_attribute?($1)
283
+ attributes_hash_name = self.class.attribute_proxy
284
+ $2.empty? ? send(attributes_hash_name)[$1] : send(attributes_hash_name)[$1] = args.first
285
+ else
286
+ super
287
+ end
288
+ end
289
+ EVAL
290
+ end
291
+
292
+ module ClassMethods
293
+ def proxy_to(attribute_name, options = {})
294
+ if attribute_name.is_a?(Hash)
295
+ options = attribute_name
296
+ else
297
+ self.attribute_proxy = attribute_name
298
+ end
299
+ self.attribute_proxy_options = options
300
+ end
301
+ end
302
+ end
303
+
304
+ # When streaming data up, Net::HTTPGenericRequest hard codes a chunk size of 1k. For large files this
305
+ # is an unfortunately low chunk size, so here we make it use a much larger default size and move it into a method
306
+ # so that the implementation of send_request_with_body_stream doesn't need to be changed to change the chunk size (at least not anymore
307
+ # than I've already had to...).
308
+ module Net
309
+ class HTTPGenericRequest
310
+ def send_request_with_body_stream(sock, ver, path, f)
311
+ raise ArgumentError, "Content-Length not given and Transfer-Encoding is not `chunked'" unless content_length() or chunked?
312
+ unless content_type()
313
+ warn 'net/http: warning: Content-Type did not set; using application/x-www-form-urlencoded' if $VERBOSE
314
+ set_content_type 'application/x-www-form-urlencoded'
315
+ end
316
+ write_header sock, ver, path
317
+ if chunked?
318
+ while s = f.read(chunk_size)
319
+ sock.write(sprintf("%x\r\n", s.length) << s << "\r\n")
320
+ end
321
+ sock.write "0\r\n\r\n"
322
+ else
323
+ while s = f.read(chunk_size)
324
+ sock.write s
325
+ end
326
+ end
327
+ end
328
+
329
+ def chunk_size
330
+ 1048576 # 1 megabyte
331
+ end
332
+ end
333
+
334
+ # Net::HTTP before 1.8.4 doesn't have the use_ssl? method or the Delete request type
335
+ class HTTP
336
+ def use_ssl?
337
+ @use_ssl
338
+ end unless public_method_defined? :use_ssl?
339
+
340
+ class Delete < HTTPRequest
341
+ METHOD = 'DELETE'
342
+ REQUEST_HAS_BODY = false
343
+ RESPONSE_HAS_BODY = true
344
+ end unless const_defined? :Delete
345
+ end
346
+ end
347
+
348
+ class XmlGenerator < String #:nodoc:
349
+ attr_reader :xml
350
+ def initialize
351
+ @xml = Builder::XmlMarkup.new(:indent => 2, :target => self)
352
+ super()
353
+ build
354
+ end
355
+ end
356
+ #:startdoc: