aws-sdk 1.6.2 → 1.6.3

data/lib/aws/s3.rb

@@ -26,7 +26,7 @@ module AWS
   # * {Amazon S3}[http://aws.amazon.com/s3/]
   # * {Amazon S3 Documentation}[http://aws.amazon.com/documentation/s3/]
   #
-  # == Credentials
+  # = Credentials
   #
   # You can setup default credentials for all AWS services via 
   # AWS.config:
@@ -34,63 +34,76 @@ module AWS
   #   AWS.config(
   #     :access_key_id => 'YOUR_ACCESS_KEY_ID',
   #     :secret_access_key => 'YOUR_SECRET_ACCESS_KEY')
-  # 
+  #
   # Or you can set them directly on the S3 interface:
   #
   #   s3 = AWS::S3.new(
   #     :access_key_id => 'YOUR_ACCESS_KEY_ID',
   #     :secret_access_key => 'YOUR_SECRET_ACCESS_KEY')
   #
-  # == Buckets Keys and Objects
+  # = Buckets
   #
-  # S3 stores objects in buckets.
+  # Before you can upload files to S3, you need to create a bucket.
   #
-  # To create a bucket:
+  #   s3 = AWS::S3.new
+  #   bucket = s3.buckets.create('my-bucket')
   #
-  #   bucket = s3.buckets.create('mybucket')
+  # If a bucket already exists, you can get a reference to the bucket.
   #
-  # To get a bucket:
+  #   bucket = s3.buckets['my-bucket'] # no request made
   #
-  #   bucket = s3.buckets[:mybucket]
-  #   bucket = s3.buckets['mybucket']
+  # You can also enumerate all buckets in your account.
   #
-  # Listing buckets:
-  # 
   #   s3.buckets.each do |bucket|
   #     puts bucket.name
   #   end
   #
-  # See {Bucket} and {BucketCollection} for more information on working
-  # with S3 buckets.
+  # See {BucketCollection} and {Bucket} for more information on working 
+  # with buckets.
   #
-  # == Listing Objects
+  # = Objects
   #
-  # Enumerating objects in a bucket:
+  # Buckets contain objects.  Each object in a bucket has a unique key.
   #
-  #   bucket.objects.each do |object|
-  #     puts object.key #=> no data is fetched from s3, just a list of keys
-  #   end
+  # == Getting an Object
+  #
+  # If the object already exists, you can get a reference to the object.
   #
-  # You can limit the list of objects with a prefix, or explore the objects
-  # in a bucket as a tree.  See {ObjectCollection} for more information.
+  #   # makes no request, returns an AWS::S3::S3Object
+  #   obj = bucket.objects['key']
   #
-  # == Reading and Writing to S3
+  # == Reading and Writing an Object
   #
-  # Each object in a bucket has a unique key.
+  # The example above returns an {S3Object}.  You call {S3Object#write} and 
+  # {S3Object#read} to upload to and download from S3 respectively.
   #
-  #   photo = s3.buckets['mybucket'].objects['photo.jpg']
+  #   # streaming upload a file to S3
+  #   obj.write(Pathname.new('/path/to/file.txt'))
   #
-  # Writing to an S3Object:
+  #   # streaming download from S3 to a file on disk
+  #   File.open('file.txt', 'w') do |file|
+  #     obj.read do |chunk|
+  #        file.write(chunk)
+  #     end
+  #   end
+  #
+  # == Enumerating Objects
   #
-  #   photo.write(File.read('/some/photo.jpg'))
+  # You can enumerate objects in your buckets.
   #
-  # Reading from an S3Object:
+  #   # enumerate ALL objects in the bucket (even if the bucket contains 
+  #   # more than 1k objects)
+  #   bucket.objects.each do |obj|
+  #     puts obj.key
+  #   end
   #
-  #   File.open("/some/path/on/disk.jpg", "w") do |f|
-  #     f.write(photo.read)
+  #   # enumerate at most 20 objects with the given prefix
+  #   bucket.objects.with_prefix('photos/').each(:limit => 20).each do |photo|
+  #     puts photo.key
   #   end
   #
-  # See {S3Object} for more information on reading and writing to S3.
+  # See {ObjectCollection} and {S3Object} for more information on working
+  # with objects.
   #
   class S3
 
@@ -104,6 +117,8 @@ module AWS
       autoload :BucketVersionCollection,      'bucket_version_collection'
       autoload :Client,                       'client'
       autoload :DataOptions,                  'data_options'
+      autoload :EncryptionUtils,              'encryption_utils'
+      autoload :CipherIO,                     'cipher_io'
       autoload :Errors,                       'errors'
       autoload :MultipartUpload,              'multipart_upload'
       autoload :MultipartUploadCollection,    'multipart_upload_collection'

data/lib/aws/s3/bucket.rb

@@ -14,15 +14,180 @@
 module AWS
   class S3
 
-    # Represents a single S3 bucket.  
+    # Represents a bucket in S3.
     #
-    # @example Creating a Bucket
-    # 
-    #   bucket = s3.buckets.create('mybucket')
+    # = Creating Buckets
     #
-    # @example Getting an Existing Bucket
+    # You create a bucket by name.  Bucket names must be globally unique
+    # and must be DNS compatible.
     #
-    #   bucket = s3.buckets['mybucket']
+    #   s3 = AWS::S3.new
+    #   bucket = s3.buckets.create('dns-compat-bucket-name')
+    #
+    # = Getting a Bucket
+    #
+    # You can create a reference to a bucket, given its name.
+    #
+    #   bucket = s3.buckets['bucket-name'] # makes no request
+    #   bucket.exists? #=> returns true/false
+    #
+    # = Enumerating Buckets
+    #
+    # The {BucketCollection} class is enumerable.
+    #
+    #   s3.buckets.each do |bucket|
+    #     puts bucket.name
+    #   end
+    #
+    # = Deleting a Bucket
+    #
+    # You can delete an empty bucket you own.
+    #
+    #   bucket = s3.buckets.create('my-temp-bucket')
+    #   bucket.objects['abc'].write('xyz')
+    #
+    #   bucket.clear! # deletes all object versions in batches
+    #   bucket.delete
+    #
+    # You can alternatively call {#delete!} which will clear
+    # the bucket for your first.
+    #
+    #   bucket.delete!
+    #
+    # = Objects
+    #
+    # Given a bucket you can access its objects, either by key or by
+    # enumeration.
+    #
+    #   bucket.objects['key'] #=> makes no request, returns an S3Object
+    #
+    #   bucket.objects.each do |obj|
+    #     puts obj.key
+    #   end
+    #
+    # See {ObjectCollection} and {S3Object} for more information on working
+    # with objects.
+    #
+    # = Bucket Policies and ACLs
+    #
+    # You can control access to your bucket and its contents a number
+    # of ways.  You can specify a bucket ACL (access control list)
+    # or a bucket policy.
+    #
+    # == ACLs
+    #
+    # ACLs control access to your bucket and its contents via a list of
+    # grants and grantees.
+    #
+    # === Canned ACLs
+    #
+    # The simplest way to specify an ACL is to use one of Amazon's "canned"
+    # ACLs.  Amazon accepts the following canned ACLs:
+    #
+    # * +:private+
+    # * +:public_read+
+    # * +:public_read_write+
+    # * +:authenticated_read+
+    # * +:bucket_owner_read+
+    # * +:bucket_owner_full_control+
+    #
+    # You can specify a the ACL at bucket creation or later update a bucket.
+    #
+    #   # at create time, defaults to :private when not specified
+    #   bucket = s3.buckets.create('name', :acl => :public_read)
+    #
+    #   # replacing an existing bucket ACL
+    #   bucket.acl = :private
+    #
+    # === Grants
+    #
+    # Alternatively you can specify a hash of grants.  Each entry in the
+    # +:grant+ hash has a grant (key) and a list of grantees (values).
+    # Valid grant keys are:
+    #
+    # * +:grant_read+
+    # * +:grant_write+
+    # * +:grant_read_acp+
+    # * +:grant_write_acp+
+    # * +:grant_full_control+
+    #
+    # Each grantee can be a String, Hash or array of strings or hashes.
+    # The following example uses grants to provide public read
+    # to everyone while providing full control to a user by email address
+    # and to another by their account id (cannonical user id).
+    #
+    #   bucket = s3.buckets.create('name', :grants => {
+    #     :grant_read => [
+    #       { :uri => "http://acs.amazonaws.com/groups/global/AllUsers" },
+    #     ],
+    #     :grant_full_control => [
+    #       { :id => 'abc...mno' }               # cannonical user id
+    #       { :email_address => 'foo@bar.com' }, # email address
+    #     ]
+    #   })
+    #
+    # === ACL Object
+    #
+    # Lastly, you can build an ACL object and use a Ruby DSL to specify grants
+    # and grantees.  See {ACLObject} for more information.
+    #
+    #   # updating an existing bucket acl using ACLObject
+    #   bucket.acl.change do |acl|
+    #     acl.grants.reject! do |g|
+    #       g.grantee.canonical_user_id != bucket.owner.id
+    #     end
+    #   end
+    #
+    # == Policies
+    #
+    # You can also work with bucket policies.
+    #
+    #   policy = AWS::S3::Policy.new
+    #   policy.allow(
+    #     :actions => [:put_object, :get_object]
+    #     :resources => [bucket]
+    #     :principals => :any)
+    #
+    #   bucket.policy = policy
+    #
+    # See {Core::Policy} and {S3::Policy} for more information on build
+    # policy objects.
+    #
+    # = Versioned Buckets
+    #
+    # You can enable versioning on a bucket you control.  When versioning
+    # is enabled, S3 will keep track of each version of each object you
+    # write to the bucket (even deletions).
+    #
+    #   bucket.versioning_enabled? #=> false
+    #   bucket.enable_versioning
+    #   # there is also a #disable_versioning method
+    #
+    #   obj = bucket.objects['my-obj']
+    #   obj.write('a')
+    #   obj.write('b')
+    #   obj.delete
+    #   obj.write('c')
+    #
+    #   obj.versions.each do |obj_version|
+    #     if obj_version.delete_marker?
+    #       puts obj_version.read
+    #     else
+    #       puts "- DELETE MARKER"
+    #     end
+    #   end
+    #
+    # Alternatively you can enumerate all versions of all objects in your
+    # bucket.
+    #
+    #   bucket.versions.each do |obj_version|
+    #     puts obj_version.key + " : " + obj_version.version_id
+    #   end
+    #
+    # See {BucketVersionCollection}, {ObjectVersionCollection} and
+    # {ObjectVersion} for more information on working with objects in
+    # a versioned bucket.  Also see the S3 documentation for information
+    # on object versioning.
     #
     class Bucket
 

data/lib/aws/s3/cipher_io.rb

@@ -0,0 +1,119 @@
+# Copyright 2011-2012 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"). You
+# may not use this file except in compliance with the License. A copy of
+# the License is located at
+#
+#     http://aws.amazon.com/apache2.0/
+#
+# or in the "license" file accompanying this file. This file is
+# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
+# ANY KIND, either express or implied. See the License for the specific
+# language governing permissions and limitations under the License.
+
+module AWS
+  class S3
+
+    # @private
+    class CipherIO
+
+      def initialize cipher, stream, stream_size = nil
+
+        @stream = stream
+        @stream_size = stream_size
+        @orig_cipher = cipher.clone
+
+        reset_cipher
+
+        # add a #rewind method if the original stream can be rewound
+        if @stream.respond_to?(:rewind)
+          Core::MetaUtils.extend_method(self, :rewind) do
+            reset_cipher
+            @stream.rewind
+          end
+        end
+
+        # add a #size method if the stream size is known
+        if stream_size
+          Core::MetaUtils.extend_method(self, :size) do
+            EncryptionUtils.get_encrypted_size(@stream_size)
+          end
+        end
+
+      end
+
+      # @return [String] Returns the requested number of bytes.  If no byte
+      #   amount is given, it will return the entire body of encrypted data
+      def read bytes = nil
+        if bytes
+          (@eof) ? nil : read_chunk(bytes)
+        else
+          (@eof) ? ""  : read_all()
+        end
+      end
+
+      # @return [Boolean] Returns +true+ when the entire stream has been read.
+      def eof?
+        @eof
+      end
+
+      private
+
+      attr_reader :cipher
+
+      # Sets the CipherIO in a reset state without having to know anything
+      #  about the cipher
+      def reset_cipher
+        @cipher = @orig_cipher.clone
+        @eof    = false
+        @final  = nil
+      end
+
+      # @return [String] Returns an encrytped chunk
+      def read_chunk bytes
+        unless @final
+          # If given a number of bytes, read it out and work out encryption
+          #  issues
+          chunk = @stream.read(bytes)
+
+          # If there is nothing, finish the encryption
+          if chunk and chunk.length > 0
+            handle_finish(bytes, cipher.update(chunk))
+          else
+            @eof = true
+            cipher.final
+          end
+          # Read as much as possible if not given a byte size
+        else
+          @eof = true
+          @final
+        end
+      end
+
+      # @return [String] Returns the entire encrypted data
+      def read_all
+        @eof = true
+        body = @stream.read()
+        data = (body and body.length > 0) ? cipher.update(body) : ""
+        data << cipher.final
+      end
+
+      # Figures out how much of the final block goes into the current chunk
+      #   and adds it.
+      # @return [String] Returns the encrypted chunk with possible padding.
+      def handle_finish(bytes, chunk)
+        free_space = bytes - chunk.size
+
+        if free_space > 0
+          @final = cipher.final
+          chunk << @final[0..free_space-1]
+          @final = @final[free_space-1..@final.size-1]
+          @eof   = true unless @final and @final.size > 0
+        end
+
+        chunk
+      end
+
+    end
+  end
+end

data/lib/aws/s3/client.rb

@@ -295,7 +295,7 @@ module AWS
 
         process_response do |response|
           regex = />(.*)<\/LocationConstraint>/
-          matches = response.http_response.body.match(regex)
+          matches = response.http_response.body.to_s.match(regex)
           response.data[:location_constraint] = matches ? matches[1] : nil
         end
 
@@ -573,10 +573,14 @@ module AWS
       #     * +:private+
       #     * +:public_read+
       #     * ...
-      #   @option options [Symbol] :storage_class+ (:standard)
+      #   @option options [String] :storage_class+ ('STANDARD')
       #     Controls whether Reduced Redundancy Storage is enabled for
-      #     the object.  Valid values are +:standard+ and
-      #     +:reduced_redundancy+.
+      #     the object.  Valid values are 'STANDARD' and
+      #     'REDUCED_REDUNDANCY'.
+      #   @option options [Symbol,String] :server_side_encryption (nil) The
+      #     algorithm used to encrypt the object on the server side
+      #     (e.g. :aes256).
+      #   object on the server side, e.g. +:aes256+)
       #   @option options [String] :cache_control
       #     Can be used to specify caching behavior.
       #   @option options [String] :content_disposition
@@ -610,19 +614,20 @@ module AWS
         :content_disposition => 'Content-Disposition',
         :content_encoding => 'Content-Encoding',
         :content_type => 'Content-Type',
-        :storage_class => 'x-amz-storage-class',
-        :server_side_encryption => 'x-amz-server-side-encryption',
         :expires => 'Expires'
       }) do
 
-        configure_request do |request, options, block|
-          options[:server_side_encryption] =
-            options[:server_side_encryption].to_s.upcase if
-            options[:server_side_encryption].kind_of?(Symbol)
-          super(request, options)
-          set_request_data(request, options, block)
+        configure_request do |request, options|
+
+          options = compute_write_options(options)
+          set_body_stream_and_content_length(request, options)
+
           request.metadata = options[:metadata]
           request.storage_class = options[:storage_class]
+          request.server_side_encryption = options[:server_side_encryption]
+
+          super(request, options)
+
         end
 
         process_response do |response|
@@ -637,6 +642,7 @@ module AWS
           end
 
           add_sse_to_response(response)
+
         end
 
         simulate_response do |response|
@@ -826,8 +832,13 @@ module AWS
       #   @option options [String] :content_disposition
       #   @option options [String] :content_encoding
       #   @option options [String] :content_type
-      #   @option options [String] :storage_class
-      #   @option options [String] :server_side_encryption
+      #   @option options [String] :storage_class+ ('STANDARD')
+      #     Controls whether Reduced Redundancy Storage is enabled for
+      #     the object.  Valid values are 'STANDARD' and
+      #     'REDUCED_REDUNDANCY'.
+      #   @option options [Symbol,String] :server_side_encryption (nil) The
+      #     algorithm used to encrypt the object on the server side
+      #     (e.g. :aes256).
       #   @option options [String] :expires
       #   @option options [String] :acl A canned ACL (e.g. 'private',
       #     'public-read', etc).  See the S3 API documentation for
@@ -851,23 +862,20 @@ module AWS
                       :content_disposition => 'Content-Disposition',
                       :content_encoding => 'Content-Encoding',
                       :content_type => 'Content-Type',
-                      :storage_class => 'x-amz-storage-class',
-                      :server_side_encryption => 'x-amz-server-side-encryption',
                       :expires => 'Expires'
                     }) do
 
         configure_request do |req, options|
-          options[:server_side_encryption] =
-            options[:server_side_encryption].to_s.upcase if
-            options[:server_side_encryption].kind_of?(Symbol)
-          super(req, options)
           req.metadata = options[:metadata]
           req.storage_class = options[:storage_class]
+          req.server_side_encryption = options[:server_side_encryption]
+          super(req, options)
         end
 
         process_response do |response|
           add_sse_to_response(response)
         end
+
       end
 
       # @overload list_multipart_uploads(options = {})
@@ -933,26 +941,30 @@ module AWS
       #   @param [Hash] options
       #   @option options [required,String] :bucket_name
       #   @option options [required,String] :key
+      #   @option options [required,String] :upload_id
+      #   @option options [required,Integer] :part_number
       #   @option options [required,String,Pathname,File,IO] :data
       #     The data to upload.  This can be provided as a string,
       #     a Pathname object, or any object that responds to
       #     +#read+ and +#eof?+ (e.g. IO, File, Tempfile, StringIO, etc).
-      #   @option options [required,String] :upload_id
-      #   @option options [required,Integer] :part_number
       #   @return [Core::Response]
       object_method(:upload_part, :put,
                     :header_options => {
                       :content_md5 => 'Content-MD5'
                     }) do
-        configure_request do |request, options, block|
-            require_upload_id!(options[:upload_id])
-          validate!("part_number", options[:part_number]) do
-            "must not be blank" if options[:part_number].to_s.empty?
-          end
-          super(request, options)
-          set_request_data(request, options, block)
+        configure_request do |request, options|
+
+          options = compute_write_options(options)
+          set_body_stream_and_content_length(request, options)
+
+          require_upload_id!(options[:upload_id])
           request.add_param('uploadId', options[:upload_id])
+
+          require_part_number!(options[:part_number])
           request.add_param('partNumber', options[:part_number])
+
+          super(request, options)
+
         end
 
         process_response do |response|
@@ -1051,6 +1063,13 @@ module AWS
       #   @option options [String] :acl A canned ACL (e.g. 'private',
       #     'public-read', etc).  See the S3 API documentation for
       #     a complete list of valid values.
+      #   @option options [Symbol,String] :server_side_encryption (nil) The
+      #     algorithm used to encrypt the object on the server side
+      #     (e.g. :aes256).
+      #   @option options [String] :storage_class+ ('STANDARD')
+      #     Controls whether Reduced Redundancy Storage is enabled for
+      #     the object.  Valid values are 'STANDARD' and
+      #     'REDUCED_REDUNDANCY'.
       #   @option options [String] :grant_read
       #   @option options [String] :grant_write
       #   @option options [String] :grant_read_acp
@@ -1067,25 +1086,21 @@ module AWS
         :copy_source => 'x-amz-copy-source',
         :cache_control => 'Cache-Control',
         :metadata_directive => 'x-amz-metadata-directive',
-        :storage_class => 'x-amz-storage-class',
-        :server_side_encryption => 'x-amz-server-side-encryption',
         :content_type => 'Content-Type',
       }) do
 
         configure_request do |req, options|
-          # TODO : validate metadata directive COPY / REPLACE
-          # TODO : validate storage class STANDARD / REDUCED_REDUNDANCY
-          # TODO : add validations for storage class in other places used
+
           validate!(:copy_source, options[:copy_source]) do
             "may not be blank" if options[:copy_source].to_s.empty?
           end
+
           options = options.merge(:copy_source => escape_path(options[:copy_source]))
-          options[:server_side_encryption] =
-            options[:server_side_encryption].to_s.upcase if
-            options[:server_side_encryption].kind_of?(Symbol)
           super(req, options)
           req.metadata = options[:metadata]
           req.storage_class = options[:storage_class]
+          req.server_side_encryption = options[:server_side_encryption]
+
           if options[:version_id]
             req.headers['x-amz-copy-source'] += "?versionId=#{options[:version_id]}"
           end
@@ -1130,20 +1145,15 @@ module AWS
         end
       end
 
-      def should_retry? response
+      def retryable_error? response
         super or
-          response.request_type == :complete_multipart_upload &&
-          extract_error_details(response)
+          (response.request_type == :complete_multipart_upload &&
+          extract_error_details(response))
           # complete multipart upload can return an error inside a
           # 200 level response -- this forces us to parse the
           # response for errors every time
       end
 
-      def set_request_data request, options, block
-        request.body_stream = data_stream_from(options, &block)
-        request.headers['Content-Length'] = content_length_from(options)
-      end
-
       def new_request
         req = S3::Request.new
         req.force_path_style = config.s3_force_path_style?
@@ -1320,12 +1330,32 @@ module AWS
           end
         end
 
+        def set_body_stream_and_content_length request, options
+
+          unless options[:content_length]
+            msg = "S3 requires a content-length header, unable to determine "
+            msg << "the content length of the data provided, please set "
+            msg << ":content_length"
+            raise ArgumentError, msg
+          end
+
+          request.headers['content-length'] = options[:content_length]
+          request.body_stream = options[:data]
+
+        end
+
         def require_upload_id!(upload_id)
           validate!("upload_id", upload_id) do
             "must not be blank" if upload_id.to_s.empty?
           end
         end
 
+        def require_part_number! part_number
+          validate!("part_number", part_number) do
+            "must not be blank" if part_number.to_s.empty?
+          end
+        end
+
         def validate_parts!(parts)
           validate!("parts", parts) do
             if !parts.kind_of?(Array)