le88-aws-s3 0.6.3.1310482014

Sign up to get free protection for your applications and to get access to all the features.
Files changed (58) hide show
  1. data/COPYING +19 -0
  2. data/INSTALL +55 -0
  3. data/README +545 -0
  4. data/Rakefile +334 -0
  5. data/bin/s3sh +6 -0
  6. data/bin/setup.rb +10 -0
  7. data/lib/aws/s3/acl.rb +636 -0
  8. data/lib/aws/s3/authentication.rb +221 -0
  9. data/lib/aws/s3/base.rb +240 -0
  10. data/lib/aws/s3/bittorrent.rb +58 -0
  11. data/lib/aws/s3/bucket.rb +319 -0
  12. data/lib/aws/s3/connection.rb +278 -0
  13. data/lib/aws/s3/error.rb +69 -0
  14. data/lib/aws/s3/exceptions.rb +133 -0
  15. data/lib/aws/s3/extensions.rb +340 -0
  16. data/lib/aws/s3/logging.rb +314 -0
  17. data/lib/aws/s3/object.rb +612 -0
  18. data/lib/aws/s3/owner.rb +44 -0
  19. data/lib/aws/s3/parsing.rb +99 -0
  20. data/lib/aws/s3/response.rb +180 -0
  21. data/lib/aws/s3/service.rb +51 -0
  22. data/lib/aws/s3/version.rb +12 -0
  23. data/lib/aws/s3.rb +60 -0
  24. data/support/faster-xml-simple/lib/faster_xml_simple.rb +187 -0
  25. data/support/faster-xml-simple/test/regression_test.rb +47 -0
  26. data/support/faster-xml-simple/test/test_helper.rb +17 -0
  27. data/support/faster-xml-simple/test/xml_simple_comparison_test.rb +46 -0
  28. data/support/rdoc/code_info.rb +211 -0
  29. data/test/acl_test.rb +254 -0
  30. data/test/authentication_test.rb +114 -0
  31. data/test/base_test.rb +136 -0
  32. data/test/bucket_test.rb +74 -0
  33. data/test/connection_test.rb +215 -0
  34. data/test/error_test.rb +70 -0
  35. data/test/extensions_test.rb +340 -0
  36. data/test/fixtures/buckets.yml +133 -0
  37. data/test/fixtures/errors.yml +34 -0
  38. data/test/fixtures/headers.yml +3 -0
  39. data/test/fixtures/logging.yml +15 -0
  40. data/test/fixtures/loglines.yml +5 -0
  41. data/test/fixtures/logs.yml +7 -0
  42. data/test/fixtures/policies.yml +16 -0
  43. data/test/fixtures.rb +89 -0
  44. data/test/logging_test.rb +89 -0
  45. data/test/mocks/fake_response.rb +26 -0
  46. data/test/object_test.rb +205 -0
  47. data/test/parsing_test.rb +66 -0
  48. data/test/remote/acl_test.rb +117 -0
  49. data/test/remote/bittorrent_test.rb +45 -0
  50. data/test/remote/bucket_test.rb +146 -0
  51. data/test/remote/logging_test.rb +82 -0
  52. data/test/remote/object_test.rb +371 -0
  53. data/test/remote/test_file.data +0 -0
  54. data/test/remote/test_helper.rb +33 -0
  55. data/test/response_test.rb +68 -0
  56. data/test/service_test.rb +23 -0
  57. data/test/test_helper.rb +110 -0
  58. metadata +179 -0
@@ -0,0 +1,221 @@
1
+ module AWS
2
+ module S3
3
+ # All authentication is taken care of for you by the AWS::S3 library. None the less, some details of the two types
4
+ # of authentication and when they are used may be of interest to some.
5
+ #
6
+ # === Header based authentication
7
+ #
8
+ # Header based authentication is achieved by setting a special <tt>Authorization</tt> header whose value
9
+ # is formatted like so:
10
+ #
11
+ # "AWS #{access_key_id}:#{encoded_canonical}"
12
+ #
13
+ # The <tt>access_key_id</tt> is the public key that is assigned by Amazon for a given account which you use when
14
+ # establishing your initial connection. The <tt>encoded_canonical</tt> is computed according to rules layed out
15
+ # by Amazon which we will describe presently.
16
+ #
17
+ # ==== Generating the encoded canonical string
18
+ #
19
+ # The "canonical string", generated by the CanonicalString class, is computed by collecting the current request method,
20
+ # a set of significant headers of the current request, and the current request path into a string.
21
+ # That canonical string is then encrypted with the <tt>secret_access_key</tt> assigned by Amazon. The resulting encrypted canonical
22
+ # string is then base 64 encoded.
23
+ #
24
+ # === Query string based authentication
25
+ #
26
+ # When accessing a restricted object from the browser, you can authenticate via the query string, by setting the following parameters:
27
+ #
28
+ # "AWSAccessKeyId=#{access_key_id}&Expires=#{expires}&Signature=#{encoded_canonical}"
29
+ #
30
+ # The QueryString class is responsible for generating the appropriate parameters for authentication via the
31
+ # query string.
32
+ #
33
+ # The <tt>access_key_id</tt> and <tt>encoded_canonical</tt> are the same as described in the Header based authentication section.
34
+ # The <tt>expires</tt> value dictates for how long the current url is valid (by default, it will expire in 5 minutes). Expiration can be specified
35
+ # either by an absolute time (expressed in seconds since the epoch), or in relative time (in number of seconds from now).
36
+ # Details of how to customize the expiration of the url are provided in the documentation for the QueryString class.
37
+ #
38
+ # All requests made by this library use header authentication. When a query string authenticated url is needed,
39
+ # the S3Object#url method will include the appropriate query string parameters.
40
+ #
41
+ # === Full authentication specification
42
+ #
43
+ # The full specification of the authentication protocol can be found at
44
+ # http://docs.amazonwebservices.com/AmazonS3/2006-03-01/RESTAuthentication.html
45
+ class Authentication
46
+ constant :AMAZON_HEADER_PREFIX, 'x-amz-'
47
+
48
+ # Signature is the abstract super class for the Header and QueryString authentication methods. It does the job
49
+ # of computing the canonical_string using the CanonicalString class as well as encoding the canonical string. The subclasses
50
+ # parameterize these computations and arrange them in a string form appropriate to how they are used, in one case a http request
51
+ # header value, and in the other case key/value query string parameter pairs.
52
+ class Signature < String #:nodoc:
53
+ attr_reader :request, :access_key_id, :secret_access_key, :options
54
+
55
+ def initialize(request, access_key_id, secret_access_key, options = {})
56
+ super()
57
+ @request, @access_key_id, @secret_access_key = request, access_key_id, secret_access_key
58
+ @options = options
59
+ end
60
+
61
+ private
62
+
63
+ def canonical_string
64
+ options = {}
65
+ options[:expires] = expires if expires?
66
+ CanonicalString.new(request, options)
67
+ end
68
+ memoized :canonical_string
69
+
70
+ def encoded_canonical
71
+ digest = OpenSSL::Digest::Digest.new('sha1')
72
+ b64_hmac = [OpenSSL::HMAC.digest(digest, secret_access_key, canonical_string)].pack("m").strip
73
+ url_encode? ? CGI.escape(b64_hmac) : b64_hmac
74
+ end
75
+
76
+ def url_encode?
77
+ !@options[:url_encode].nil?
78
+ end
79
+
80
+ def expires?
81
+ is_a? QueryString
82
+ end
83
+
84
+ def date
85
+ request['date'].to_s.strip.empty? ? Time.now : Time.parse(request['date'])
86
+ end
87
+ end
88
+
89
+ # Provides header authentication by computing the value of the Authorization header. More details about the
90
+ # various authentication schemes can be found in the docs for its containing module, Authentication.
91
+ class Header < Signature #:nodoc:
92
+ def initialize(*args)
93
+ super
94
+ self << "AWS #{access_key_id}:#{encoded_canonical}"
95
+ end
96
+ end
97
+
98
+ # Provides query string authentication by computing the three authorization parameters: AWSAccessKeyId, Expires and Signature.
99
+ # More details about the various authentication schemes can be found in the docs for its containing module, Authentication.
100
+ class QueryString < Signature #:nodoc:
101
+ constant :DEFAULT_EXPIRY, 300 # 5 minutes
102
+ def initialize(*args)
103
+ super
104
+ options[:url_encode] = true
105
+ self << build
106
+ end
107
+
108
+ private
109
+
110
+ # Will return one of three values, in the following order of precedence:
111
+ #
112
+ # 1) Seconds since the epoch explicitly passed in the +:expires+ option
113
+ # 2) The current time in seconds since the epoch plus the number of seconds passed in
114
+ # the +:expires_in+ option
115
+ # 3) The current time in seconds since the epoch plus the default number of seconds (60 seconds)
116
+ def expires
117
+ return options[:expires] if options[:expires]
118
+ date.to_i + expires_in
119
+ end
120
+
121
+ def expires_in
122
+ options.has_key?(:expires_in) ? Integer(options[:expires_in]) : DEFAULT_EXPIRY
123
+ end
124
+
125
+ # Keep in alphabetical order
126
+ def build
127
+ "AWSAccessKeyId=#{access_key_id}&Expires=#{expires}&Signature=#{encoded_canonical}"
128
+ end
129
+ end
130
+
131
+ # The CanonicalString is used to generate an encrypted signature, signed with your secrect access key. It is composed of
132
+ # data related to the given request for which it provides authentication. This data includes the request method, request headers,
133
+ # and the request path. Both Header and QueryString use it to generate their signature.
134
+ class CanonicalString < String #:nodoc:
135
+ class << self
136
+ def default_headers
137
+ %w(content-type content-md5)
138
+ end
139
+
140
+ def interesting_headers
141
+ ['content-md5', 'content-type', 'date', amazon_header_prefix]
142
+ end
143
+
144
+ def amazon_header_prefix
145
+ /^#{AMAZON_HEADER_PREFIX}/io
146
+ end
147
+ end
148
+
149
+ attr_reader :request, :headers
150
+
151
+ def initialize(request, options = {})
152
+ super()
153
+ @request = request
154
+ @headers = {}
155
+ @options = options
156
+ # "For non-authenticated or anonymous requests. A NotImplemented error result code will be returned if
157
+ # an authenticated (signed) request specifies a Host: header other than 's3.amazonaws.com'"
158
+ # (from http://docs.amazonwebservices.com/AmazonS3/2006-03-01/VirtualHosting.html)
159
+ request['Host'] = DEFAULT_HOST
160
+ build
161
+ end
162
+
163
+ private
164
+ def build
165
+ self << "#{request.method}\n"
166
+ ensure_date_is_valid
167
+
168
+ initialize_headers
169
+ set_expiry!
170
+
171
+ headers.sort_by {|k, _| k}.each do |key, value|
172
+ value = value.to_s.strip
173
+ self << (key =~ self.class.amazon_header_prefix ? "#{key}:#{value}" : value)
174
+ self << "\n"
175
+ end
176
+ self << path
177
+ end
178
+
179
+ def initialize_headers
180
+ identify_interesting_headers
181
+ set_default_headers
182
+ end
183
+
184
+ def set_expiry!
185
+ self.headers['date'] = @options[:expires] if @options[:expires]
186
+ end
187
+
188
+ def ensure_date_is_valid
189
+ request['Date'] ||= Time.now.httpdate
190
+ end
191
+
192
+ def identify_interesting_headers
193
+ request.each do |key, value|
194
+ key = key.downcase # Can't modify frozen string so no bang
195
+ if self.class.interesting_headers.any? {|header| header === key}
196
+ self.headers[key] = value.to_s.strip
197
+ end
198
+ end
199
+ end
200
+
201
+ def set_default_headers
202
+ self.class.default_headers.each do |header|
203
+ self.headers[header] ||= ''
204
+ end
205
+ end
206
+
207
+ def path
208
+ [only_path, extract_significant_parameter].compact.join('?')
209
+ end
210
+
211
+ def extract_significant_parameter
212
+ request.path[/[&?](acl|torrent|logging)(?:&|=|$)/, 1]
213
+ end
214
+
215
+ def only_path
216
+ request.path[/^[^?]*/]
217
+ end
218
+ end
219
+ end
220
+ end
221
+ end
@@ -0,0 +1,240 @@
1
+ module AWS #:nodoc:
2
+ # AWS::S3 is a Ruby library for Amazon's Simple Storage Service's REST API (http://aws.amazon.com/s3).
3
+ # Full documentation of the currently supported API can be found at http://docs.amazonwebservices.com/AmazonS3/2006-03-01.
4
+ #
5
+ # == Getting started
6
+ #
7
+ # To get started you need to require 'aws/s3':
8
+ #
9
+ # % irb -rubygems
10
+ # irb(main):001:0> require 'aws/s3'
11
+ # # => true
12
+ #
13
+ # The AWS::S3 library ships with an interactive shell called <tt>s3sh</tt>. From within it, you have access to all the operations the library exposes from the command line.
14
+ #
15
+ # % s3sh
16
+ # >> Version
17
+ #
18
+ # Before you can do anything, you must establish a connection using Base.establish_connection!. A basic connection would look something like this:
19
+ #
20
+ # AWS::S3::Base.establish_connection!(
21
+ # :access_key_id => 'abc',
22
+ # :secret_access_key => '123'
23
+ # )
24
+ #
25
+ # The minimum connection options that you must specify are your access key id and your secret access key.
26
+ #
27
+ # (If you don't already have your access keys, all you need to sign up for the S3 service is an account at Amazon. You can sign up for S3 and get access keys by visiting http://aws.amazon.com/s3.)
28
+ #
29
+ # For convenience, if you set two special environment variables with the value of your access keys, the console will automatically create a default connection for you. For example:
30
+ #
31
+ # % cat .amazon_keys
32
+ # export AMAZON_ACCESS_KEY_ID='abcdefghijklmnop'
33
+ # export AMAZON_SECRET_ACCESS_KEY='1234567891012345'
34
+ #
35
+ # Then load it in your shell's rc file.
36
+ #
37
+ # % cat .zshrc
38
+ # if [[ -f "$HOME/.amazon_keys" ]]; then
39
+ # source "$HOME/.amazon_keys";
40
+ # fi
41
+ #
42
+ # See more connection details at AWS::S3::Connection::Management::ClassMethods.
43
+ module S3
44
+ constant :DEFAULT_HOST, 's3.amazonaws.com'
45
+
46
+ # AWS::S3::Base is the abstract super class of all classes who make requests against S3, such as the built in
47
+ # Service, Bucket and S3Object classes. It provides methods for making requests, inferring or setting response classes,
48
+ # processing request options, and accessing attributes from S3's response data.
49
+ #
50
+ # Establishing a connection with the Base class is the entry point to using the library:
51
+ #
52
+ # AWS::S3::Base.establish_connection!(:access_key_id => '...', :secret_access_key => '...')
53
+ #
54
+ # The <tt>:access_key_id</tt> and <tt>:secret_access_key</tt> are the two required connection options. More
55
+ # details can be found in the docs for Connection::Management::ClassMethods.
56
+ #
57
+ # Extensive examples can be found in the README[link:files/README.html].
58
+ class Base
59
+ class << self
60
+ # Wraps the current connection's request method and picks the appropriate response class to wrap the response in.
61
+ # If the response is an error, it will raise that error as an exception. All such exceptions can be caught by rescuing
62
+ # their superclass, the ResponseError exception class.
63
+ #
64
+ # It is unlikely that you would call this method directly. Subclasses of Base have convenience methods for each http request verb
65
+ # that wrap calls to request.
66
+ def request(verb, path, options = {}, body = nil, attempts = 0, &block)
67
+ Service.response = nil
68
+ process_options!(options, verb)
69
+ response = response_class.new(connection.request(verb, path, options, body, attempts, &block))
70
+ Service.response = response
71
+
72
+ Error::Response.new(response.response).error.raise if response.error?
73
+ response
74
+ # Once in a while, a request to S3 returns an internal error. A glitch in the matrix I presume. Since these
75
+ # errors are few and far between the request method will rescue InternalErrors the first three times they encouter them
76
+ # and will retry the request again. Most of the time the second attempt will work.
77
+ rescue InternalError, RequestTimeout
78
+ if attempts == 3
79
+ raise
80
+ else
81
+ attempts += 1
82
+ retry
83
+ end
84
+ end
85
+
86
+ [:get, :post, :put, :delete, :head].each do |verb|
87
+ class_eval(<<-EVAL, __FILE__, __LINE__)
88
+ def #{verb}(path, headers = {}, body = nil, &block)
89
+ request(:#{verb}, path, headers, body, &block)
90
+ end
91
+ EVAL
92
+ end
93
+
94
+ # Called when a method which requires a bucket name is called without that bucket name specified. It will try to
95
+ # infer the current bucket by looking for it as the subdomain of the current connection's address. If no subdomain
96
+ # is found, CurrentBucketNotSpecified will be raised.
97
+ #
98
+ # MusicBucket.establish_connection! :server => 'jukeboxzero.s3.amazonaws.com'
99
+ # MusicBucket.connection.server
100
+ # => 'jukeboxzero.s3.amazonaws.com'
101
+ # MusicBucket.current_bucket
102
+ # => 'jukeboxzero'
103
+ #
104
+ # Rather than infering the current bucket from the subdomain, the current class' bucket can be explicitly set with
105
+ # set_current_bucket_to.
106
+ def current_bucket
107
+ connection.subdomain or raise CurrentBucketNotSpecified.new(connection.http.address)
108
+ end
109
+
110
+ # If you plan on always using a specific bucket for certain files, you can skip always having to specify the bucket by creating
111
+ # a subclass of Bucket or S3Object and telling it what bucket to use:
112
+ #
113
+ # class JukeBoxSong < AWS::S3::S3Object
114
+ # set_current_bucket_to 'jukebox'
115
+ # end
116
+ #
117
+ # For all methods that take a bucket name as an argument, the current bucket will be used if the bucket name argument is omitted.
118
+ #
119
+ # other_song = 'baby-please-come-home.mp3'
120
+ # JukeBoxSong.store(other_song, open(other_song))
121
+ #
122
+ # This time we didn't have to explicitly pass in the bucket name, as the JukeBoxSong class knows that it will
123
+ # always use the 'jukebox' bucket.
124
+ #
125
+ # "Astute readers", as they say, may have noticed that we used the third parameter to pass in the content type,
126
+ # rather than the fourth parameter as we had the last time we created an object. If the bucket can be inferred, or
127
+ # is explicitly set, as we've done in the JukeBoxSong class, then the third argument can be used to pass in
128
+ # options.
129
+ #
130
+ # Now all operations that would have required a bucket name no longer do.
131
+ #
132
+ # other_song = JukeBoxSong.find('baby-please-come-home.mp3')
133
+ def set_current_bucket_to(name)
134
+ raise ArgumentError, "`#{__method__}' must be called on a subclass of #{self.name}" if self == AWS::S3::Base
135
+ instance_eval(<<-EVAL)
136
+ def current_bucket
137
+ '#{name}'
138
+ end
139
+ EVAL
140
+ end
141
+ alias_method :current_bucket=, :set_current_bucket_to
142
+
143
+ private
144
+
145
+ def response_class
146
+ FindResponseClass.for(self)
147
+ end
148
+
149
+ def process_options!(options, verb)
150
+ options.replace(RequestOptions.process(options, verb))
151
+ end
152
+
153
+ # Using the conventions layed out in the <tt>response_class</tt> works for more than 80% of the time.
154
+ # There are a few edge cases though where we want a given class to wrap its responses in different
155
+ # response classes depending on which method is being called.
156
+ def respond_with(klass)
157
+ eval(<<-EVAL, binding, __FILE__, __LINE__)
158
+ def new_response_class
159
+ #{klass}
160
+ end
161
+
162
+ class << self
163
+ alias_method :old_response_class, :response_class
164
+ alias_method :response_class, :new_response_class
165
+ end
166
+ EVAL
167
+
168
+ yield
169
+ ensure
170
+ # Restore the original version
171
+ eval(<<-EVAL, binding, __FILE__, __LINE__)
172
+ class << self
173
+ alias_method :response_class, :old_response_class
174
+ end
175
+ EVAL
176
+ end
177
+
178
+ def bucket_name(name)
179
+ name || current_bucket
180
+ end
181
+
182
+ class RequestOptions < Hash #:nodoc:
183
+ attr_reader :options, :verb
184
+
185
+ class << self
186
+ def process(*args, &block)
187
+ new(*args, &block).process!
188
+ end
189
+ end
190
+
191
+ def initialize(options, verb = :get)
192
+ @options = options.to_normalized_options
193
+ @verb = verb
194
+ super()
195
+ end
196
+
197
+ def process!
198
+ set_access_controls! if verb == :put
199
+ replace(options)
200
+ end
201
+
202
+ private
203
+ def set_access_controls!
204
+ ACL::OptionProcessor.process!(options)
205
+ end
206
+ end
207
+ end
208
+
209
+ def initialize(attributes = {}) #:nodoc:
210
+ @attributes = attributes
211
+ end
212
+
213
+ private
214
+ attr_reader :attributes
215
+
216
+ def connection
217
+ self.class.connection
218
+ end
219
+
220
+ def http
221
+ connection.http
222
+ end
223
+
224
+ def request(*args, &block)
225
+ self.class.request(*args, &block)
226
+ end
227
+
228
+ def method_missing(method, *args, &block)
229
+ case
230
+ when attributes.has_key?(method.to_s)
231
+ attributes[method.to_s]
232
+ when attributes.has_key?(method)
233
+ attributes[method]
234
+ else
235
+ super
236
+ end
237
+ end
238
+ end
239
+ end
240
+ end
@@ -0,0 +1,58 @@
1
+ module AWS
2
+ module S3
3
+ # Objects on S3 can be distributed via the BitTorrent file sharing protocol.
4
+ #
5
+ # You can get a torrent file for an object by calling <tt>torrent_for</tt>:
6
+ #
7
+ # S3Object.torrent_for 'kiss.jpg', 'marcel'
8
+ #
9
+ # Or just call the <tt>torrent</tt> method if you already have the object:
10
+ #
11
+ # song = S3Object.find 'kiss.jpg', 'marcel'
12
+ # song.torrent
13
+ #
14
+ # Calling <tt>grant_torrent_access_to</tt> on a object will allow anyone to anonymously
15
+ # fetch the torrent file for that object:
16
+ #
17
+ # S3Object.grant_torrent_access_to 'kiss.jpg', 'marcel'
18
+ #
19
+ # Anonymous requests to
20
+ #
21
+ # http://s3.amazonaws.com/marcel/kiss.jpg?torrent
22
+ #
23
+ # will serve up the torrent file for that object.
24
+ module BitTorrent
25
+ def self.included(klass) #:nodoc:
26
+ klass.extend ClassMethods
27
+ end
28
+
29
+ # Adds methods to S3Object for accessing the torrent of a given object.
30
+ module ClassMethods
31
+ # Returns the torrent file for the object with the given <tt>key</tt>.
32
+ def torrent_for(key, bucket = nil)
33
+ get(path!(bucket, key) << '?torrent').body
34
+ end
35
+ alias_method :torrent, :torrent_for
36
+
37
+ # Grants access to the object with the given <tt>key</tt> to be accessible as a torrent.
38
+ def grant_torrent_access_to(key, bucket = nil)
39
+ policy = acl(key, bucket)
40
+ return true if policy.grants.include?(:public_read)
41
+ policy.grants << ACL::Grant.grant(:public_read)
42
+ acl(key, bucket, policy)
43
+ end
44
+ alias_method :grant_torrent_access, :grant_torrent_access_to
45
+ end
46
+
47
+ # Returns the torrent file for the object.
48
+ def torrent
49
+ self.class.torrent_for(key, bucket.name)
50
+ end
51
+
52
+ # Grants torrent access publicly to anyone who requests it on this object.
53
+ def grant_torrent_access
54
+ self.class.grant_torrent_access_to(key, bucket.name)
55
+ end
56
+ end
57
+ end
58
+ end