aws-sdk 1.6.2 → 1.6.3

data/lib/aws/s3/config.rb

@@ -25,4 +25,10 @@ AWS::Core::Configuration.module_eval do
 
   add_option :s3_server_side_encryption, nil
 
+  add_option :s3_encryption_key, nil
+
+  add_option :s3_encryption_materials_location, :metadata
+
+  add_option :s3_storage_class, 'STANDARD'
+
 end

data/lib/aws/s3/data_options.rb

@@ -16,82 +16,169 @@ require 'pathname'
 module AWS
   class S3
 
+    # Used by S3#S3Object and S3::Client to accept options with
+    # data that should be uploaded (streamed).
     # @private
     module DataOptions
 
       protected
 
-      def data_stream_from options, &block
+      # @return [Hash] Returns a hash of options with a :data option that
+      #   responds to #read and #eof?.
+      def compute_write_options *args, &block
 
-        validate_data!(options, block)
+        options = convert_args_to_options_hash(*args)
 
-        # block format
-        if block_given?
-          buffer = StringIO.new
-          yield(buffer)
-          buffer.rewind
-          return buffer
+        validate_data!(options, &block)
+
+        rename_file_to_data(options)
+
+        convert_data_to_io_obj(options, &block)
+
+        try_to_determine_content_length(options)
+
+        options
+
+      end
+
+      # Converts an argument list into a single hash of options.  Treats
+      # non-hash arguments in the first position as a data option.
+      def convert_args_to_options_hash *args
+        case args.count
+        when 0 then {}
+        when 1 then args[0].is_a?(Hash) ? args[0] : { :data => args[0] }
+        when 2 then args[1].merge(:data => args[0])
+        else
+          msg = "expected 0, 1 or 2 arguments, got #{args.count}"
+          raise ArgumentError, msg
         end
+      end
 
-        # string, pathname, file, io-like object, etc
-        data = options[:data]
-        file_opts = ["rb"]
-        file_opts << { :encoding => "BINARY" } if Object.const_defined?(:Encoding)
-        case
-        when data.is_a?(String)
+      # Moves options[:file] to options[:data].  If this option is a string
+      # then it is treated as a file path and is converted to an open file.
+      def rename_file_to_data options
+        if file = options.delete(:file)
+          options[:data] = file.is_a?(String) ? open_file(file) : file
+        end
+      end
+
+      # Converts the :data option to an IO-like object.  This allows us
+      # to always perform streaming uploads.
+      def convert_data_to_io_obj options, &block
+
+        data = options.delete(:data)
+
+        if block_given?
+          options[:data] = IOProxy.new(block)
+        elsif data.is_a?(String)
           data.force_encoding("BINARY") if data.respond_to?(:force_encoding)
-          StringIO.new(data)
-        when data.is_a?(Pathname) then File.open(data.to_s, *file_opts)
-        when options[:file]       then File.open(options[:file], *file_opts)
-        else data
+          options[:data] = StringIO.new(data)
+        elsif data.is_a?(Pathname)
+          options[:data] = open_file(data.to_s)
+        elsif io_like?(data)
+          options[:data] = data
+        else
+          msg = "invalid :data option, expected a String, Pathname or "
+          msg << "an object that responds to #read and #eof?"
+          raise ArgumentError, msg
         end
 
       end
 
-      def content_length_from options
-        data = options[:data]
-        case
-        when options[:content_length]    then options[:content_length]
-        when options[:file]              then File.size(options[:file])
-        when data.is_a?(Pathname)        then File.size(data.to_s)
-        when data.is_a?(File)            then File.size(data.path)
-        when data.respond_to?(:bytesize) then data.bytesize
-        when data.respond_to?(:size)     then data.size
-        when data.respond_to?(:length)   then data.length
-        else raise ArgumentError, 'content_length was not provided ' +
-            'and could not be determined'
+      # Attempts to determine the content length of the :data option.
+      # This is only done when a content length is not already provided.
+      def try_to_determine_content_length options
+        unless options[:content_length]
+
+          data = options[:data]
+
+          length = case
+            when data.respond_to?(:bytesize) then data.bytesize
+            when data.respond_to?(:size)     then data.size
+            when data.respond_to?(:length)   then data.length
+            when data.respond_to?(:path)     then File.size(data.path)
+            else nil
+          end
+
+          options[:content_length] = length if length
+
         end
       end
 
-      def validate_data! options, block
+      def validate_data! options, &block
 
         data = options[:data]
-        filename = options[:file]
+        file = options[:file]
 
-        raise ArgumentError, 'data passed multiple ways' if
-          [data, filename, block].compact.size > 1
+        raise ArgumentError, 'Object data passed multiple ways.' if
+          [data, file, block].compact.count > 1
 
-        # accepting block format
-        return if block and block.arity == 1
+        data = file if file
 
-        # accepting file path
-        return if filename.kind_of?(String)
-
-        # accepting strings
+        return if block_given?
         return if data.kind_of?(String)
-
-        # accepting pathname
         return if data.kind_of?(Pathname)
+        return if io_like?(data)
+
+        msg = ":data must be provided as a String, Pathname, File, or "
+        msg << "an object that responds to #read and #eof?"
+        raise ArgumentError, msg
 
-        # accepts io-like objects (responds to read and eof?)
-        if data.respond_to?(:read) and 
-            data.method(:read).arity != 0 and
-            data.respond_to?(:eof?) then
-          return true
+      end
+
+      # @return [Boolean] Returns +true+ if the object responds to
+      #   +#read+ and +#eof?+.
+      def io_like? io
+        io.respond_to?(:read) and io.respond_to?(:eof?)
+      end
+
+      # @param [String] path Path to a file on disk.
+      # @return [File] Given a path string, returns an open File.
+      def open_file path
+        file_opts = ['rb']
+        file_opts << { :encoding => "BINARY" } if Object.const_defined?(:Encoding)
+        File.open(path, *file_opts)
+      end
+
+      # A utility class that turns a block (with 2 args) into an
+      # IO object that responds to #read and #eof.
+      # @private
+      class IOProxy
+
+        def initialize write_block
+          unless write_block.arity == 2
+            msg = "a write block must accept 2 yield params: a buffer and "
+            msg << "a number of bytes to write"
+            raise ArgumentError, msg
+          end
+          @write_block = write_block
+          @eof = false
+        end
+
+        def read bytes = nil
+          if bytes
+            buffer = StringIO.new
+            @write_block.call(buffer, bytes)
+            buffer.rewind
+            @eof = true if buffer.size < bytes
+            buffer.size == 0 ? nil : buffer.read
+          else
+            read_all
+          end
+        end
+
+        def eof?
+          @eof
         end
 
-        raise ArgumentError, 'data must be provided as a String, ' +
-          'Pathname, file path, or an object that responds to #read and #eof?'
+        protected
+
+        def read_all
+          buffer = StringIO.new
+          buffer << read(1024 * 1024 * 5) until eof?
+          buffer.rewind
+          buffer.read
+        end
 
       end
 

data/lib/aws/s3/encryption_utils.rb

@@ -0,0 +1,144 @@
+# Copyright 2011-2012 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"). You
+# may not use this file except in compliance with the License. A copy of
+# the License is located at
+#
+#     http://aws.amazon.com/apache2.0/
+#
+# or in the "license" file accompanying this file. This file is
+# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
+# ANY KIND, either express or implied. See the License for the specific
+# language governing permissions and limitations under the License.
+
+require 'openssl'
+
+module AWS
+  class S3
+    # @private
+    module EncryptionUtils
+
+      protected
+
+
+      # @param [OpenSSL::PKey::RSA, String] key Key used to encrypt.
+      #
+      # @param [String] data Data to be encrypted.
+      #
+      # @note Use check_encryption_materials before this method to check
+      #   formatting of keys
+      #
+      # @return [String] Returns the data encrypted with the key given.
+      def encrypt data, key
+        rsa = OpenSSL::PKey::RSA
+        ## Encrypting data key
+        case key
+        when rsa # Asymmetric encryption
+          key.public_encrypt(data)
+        when String             # Symmetric encryption
+          cipher = get_aes_cipher(:encrypt, :ECB, key)
+          cipher.update(data) + cipher.final
+        end
+      end
+
+      # @param [OpenSSL::PKey::RSA, String] key Key used to encrypt.
+      #
+      # @param [String] data Data to be encrypted.
+      #
+      # @note Use check_encryption_materials before this method to check
+      #   formatting of keys
+      #
+      # @return [String] Returns the data decrypted with the key given.
+      def decrypt data, key
+        rsa = OpenSSL::PKey::RSA
+        begin
+          case key
+          when rsa # Asymmetric Decryption
+              key.private_decrypt(data)
+          when String             # Symmetric Decryption
+              cipher = get_aes_cipher(:decrypt, :ECB, key)
+              cipher.update(data) + cipher.final
+          end
+        rescue OpenSSL::Cipher::CipherError
+          raise RuntimeError, "decryption failed, incorrect key?"
+        end
+      end
+
+      # Checks for any formatting problems for keys and initialization vectors
+      #   supported with EncryptionUtils.
+      #
+      # @param [OpenSSL::PKey::RSA, String] key Key used to encrypt.
+      #
+      # @param [String] data Data to be encrypted.
+      #
+      def check_encryption_materials mode, key
+        rsa = OpenSSL::PKey::RSA
+        case key
+        when rsa
+          unless key.private? or mode == :encrypt
+            msg = "invalid key, #{rsa} requires a private key"
+            raise ArgumentError, msg
+          end
+        when String # no problem
+        else
+          msg = "invalid key, must be an #{rsa} or a cipher key string"
+          raise ArgumentError, msg
+        end
+      end
+
+      # @param [OpenSSL::Cipher] cipher The cipher with configured key and iv.
+      #
+      # @yield [String, String] key_iv_pair A randomly generated key, iv pair
+      #   for use with the given cipher.  Sets the key and iv on the cipher.
+      def generate_aes_key cipher, &block
+        key_iv_pair = [cipher.random_key, cipher.random_iv]
+        yield(key_iv_pair) if block_given?
+      end
+
+      # @param [Symbol] mode The encryption/decryption mode.  Valid inputs are
+      #   :encrypt or :decrypt
+      #
+      # @param [String] key Key for the cipher.
+      #
+      # @param [String] iv IV for the cipher.
+      #
+      # @return [OpenSSL::Cipher] Will return a configured +OpenSSL::Cipher+.
+      def get_aes_cipher mode, block_mode, key = nil, iv = nil
+
+        # If no key given, default to 256 bit
+        cipher_size = (key) ? get_cipher_size(key.length) : 256
+
+        cipher = OpenSSL::Cipher.new("AES-#{cipher_size}-#{block_mode}")
+
+        (mode == :encrypt) ? cipher.encrypt : cipher.decrypt
+        cipher.key = key if key
+        cipher.iv  = iv  if iv
+        cipher
+      end
+
+      # @param  [Fixnum] Size of data given.
+      # @return [Fixnum] Returns the AES encrypted size based on a given size.
+      def get_encrypted_size size
+        # The next multiple of 16
+        ((size / 16) + 1) * 16
+      end
+      module_function :get_encrypted_size
+
+      private
+
+      # @param  [Fixnum] key_length Length of the key given.
+      # @return [Fixnum] Returns the cipher size based on the key length.
+      def get_cipher_size(key_length)
+        case key_length
+        when 32 then 256
+        when 24 then 192
+        when 16 then 128
+        else
+          msg = "invalid key, symmetric key required to be 16, 24, or 32 bytes "
+          msg << "in length, saw length #{key_length}"
+          raise ArgumentError, msg
+        end
+      end
+    end
+  end
+end

data/lib/aws/s3/errors.rb

@@ -76,6 +76,20 @@ module AWS
 
       end
 
+      # This error is special, because S3 must first retrieve the client
+      #   side encryption key in it's encrypted form before finding if the
+      #   key is incorrect.
+      class IncorrectClientSideEncryptionKey < AWS::Errors::Base
+
+        include AWS::Errors::ClientError
+
+        def initialize(msg)
+          super("",
+                "",
+                "IncorrectClientSideEncryptionKey",
+                msg)
+        end
+      end
     end
   end
 end

data/lib/aws/s3/multipart_upload.rb

@@ -110,8 +110,10 @@ module AWS
       # storage consumed by all parts.
       # @return [nil]
       def abort
-        client.abort_multipart_upload(base_opts)
-        @aborted = true
+        unless aborted?
+          client.abort_multipart_upload(base_opts)
+          @aborted = true
+        end
         nil
       end
       alias_method :delete, :abort
@@ -257,8 +259,9 @@ module AWS
       #   returns the object.  If no upload was attempted (e.g. if it
       #   was aborted or if no parts were uploaded), returns +nil+.
       def close
-        return if aborted?
-        if completed_parts.empty?
+        if aborted?
+          nil
+        elsif completed_parts.empty?
           abort
         else
           complete

data/lib/aws/s3/object_collection.rb

@@ -81,8 +81,8 @@ module AWS
       #
       # @param [String] key Where in S3 to write the object.
       # @return [S3Object]
-      def create key, *args
-        self[key].write(*args)
+      def create key, *args, &block
+        self[key].write(*args, &block)
       end
 
       # Returns an S3Object given its name.  For example:

data/lib/aws/s3/policy.rb

@@ -49,7 +49,7 @@ module AWS
           :get_bucket_notification => "s3:GetBucketNotification",
           :set_bucket_notification => "s3:PutBucketNotification"
         }
-        
+
         protected
         def resource_arn resource
           prefix = 'arn:aws:s3:::'

data/lib/aws/s3/request.rb

@@ -28,9 +28,6 @@ module AWS
       # @return [String] S3 object key
       attr_accessor :key
 
-      # @private
-      attr_accessor :body_stream
-
       # @private
       attr_accessor :force_path_style
 
@@ -48,6 +45,14 @@ module AWS
         end
       end
 
+      def server_side_encryption= sse
+        if sse.is_a?(Symbol)
+          headers['x-amz-server-side-encryption'] = sse.to_s.upcase
+        elsif sse
+          headers['x-amz-server-side-encryption'] = sse
+        end
+      end
+
       def host
         path_style? ? @host : "#{bucket}.#{@host}"
       end
@@ -76,30 +81,13 @@ module AWS
 
       end
 
-      # @param [String, IO] body The http request body.  This can be a string or
-      #   any object that responds to #read and #eof? (like an IO object).
-      def body= body
-        @body_stream = StringIO.new(body)
-      end
-
-      # @return [String, nil] The http request body.
-      def body
-        if @body_stream
-          string = @body_stream.read
-          @body_stream.rewind
-          string
-        else
-          nil
-        end
-      end
-
       # From the S3 developer guide:
       #
-      # StringToSign = 
-      #   HTTP-Verb + "\n" + 
-      #   content-md5 + "\n" + 
-      #   content-type + "\n" + 
-      #   date + "\n" + 
+      # StringToSign =
+      #   HTTP-Verb + "\n" +
+      #   content-md5 + "\n" +
+      #   content-type + "\n" +
+      #   date + "\n" +
       #   CanonicalizedAmzHeaders + CanonicalizedResource;
       #
       def string_to_sign
@@ -124,11 +112,11 @@ module AWS
 
       # From the S3 developer guide
       #
-      #   CanonicalizedResource = 
-      #     [ "/" + Bucket ] + 
+      #   CanonicalizedResource =
+      #     [ "/" + Bucket ] +
       #     <HTTP-Request-URI, from the protocol name up to the querystring> +
-      #     [ sub-resource, if present. e.g. "?acl", "?location", 
-      #     "?logging", or "?torrent"]; 
+      #     [ sub-resource, if present. e.g. "?acl", "?location",
+      #     "?logging", or "?torrent"];
       #
       def canonicalized_resource
 
@@ -139,11 +127,11 @@ module AWS
         if Client.dns_compatible_bucket_name?(bucket) and !path_style?
           parts << "/#{bucket}"
         end
-  
+
         # all requests require the portion of the un-decoded uri up to
         # but not including the query string
         parts << path
- 
+
         # lastly any sub resource querystring params need to be appened
         # in lexigraphical ordered joined by '&' and prefixed by '?'
         params = (sub_resource_params + query_parameters_for_signature)
@@ -190,8 +178,8 @@ module AWS
       class << self
 
         def sub_resources
-          %w(acl location logging notification partNumber policy 
-             requestPayment torrent uploadId uploads versionId 
+          %w(acl location logging notification partNumber policy
+             requestPayment torrent uploadId uploads versionId
              versioning versions delete lifecycle)
         end