aws-sdk 1.6.2 → 1.6.3

data/lib/aws/core.rb

@@ -69,7 +69,7 @@ require 'aws/core/autoloader'
 module AWS
 
   # Current version of the AWS SDK for Ruby
-  VERSION = "1.6.2"
+  VERSION = "1.6.3"
 
   register_autoloads(self) do
     autoload :Errors, 'errors'
@@ -299,7 +299,7 @@ module AWS
     #   +true+, requests will always use path style.  This can be useful
     #   for testing environments.
     #
-    # @option options [Integer] :s3_multipart_max_parts (1000) The maximum
+    # @option options [Integer] :s3_multipart_max_parts (10000) The maximum
     #   number of parts to split a file into when uploading in parts to S3.
     #
     # @option options [Integer] :s3_multipart_threshold (16777216) When
@@ -328,6 +328,17 @@ module AWS
     #   * {S3::S3Object#presigned_post}
     #   * {S3::Bucket#presigned_post}
     #
+    # @option options [OpenSSL::PKey::RSA, String] :s3_encryption_key (nil)
+    #   If this is set, AWS::S3::S3Object #read and #write methods will always
+    #   perform client-side encryption with this key. The key can be overridden
+    #   at runtime by using the :encryption_key option.  A value of nil
+    #   means that client-side encryption will not be used.
+    #
+    # @option options [Symbol] :s3_encryption_materials_location (:metadata)
+    #   When set to +:instruction_file+, AWS::S3::S3Object will store
+    #   encryption materials in a seperate object, instead of the object
+    #   metadata.
+    #
     # @option options [String] :simple_db_endpoint ('sdb.amazonaws.com')
     #   The service endpoint for Amazon SimpleDB.
     #

data/lib/aws/core/autoloader.rb

@@ -42,7 +42,7 @@ module AWS
 
     # @private
     class Autoloader
-      
+
       def initialize klass, prefix = nil
         @klass = klass
         @prefix = prefix || klass.name.gsub(/::/, '/').downcase

data/lib/aws/core/client.rb

@@ -21,6 +21,11 @@ module AWS
     # Base client class for all of the Amazon AWS service clients.
     class Client
 
+      # Raised when a request failed due to a networking issue (e.g.
+      # EOFError, IOError, Errno::ECONNRESET, Errno::EPIPE,
+      # Timeout::Error, etc)
+      class NetworkError < StandardError; end
+
       extend Naming
 
       # @private
@@ -155,6 +160,19 @@ module AWS
         response
       end
 
+      # Logs the warning to the configured logger, otherwise to stderr.
+      # @param [String] warning
+      # @return [nil]
+      def log_warning warning
+        message = '[aws-sdk-gem-warning] ' + warning
+        if config.logger
+          config.logger.warn(message)
+        else
+          $stderr.puts(message)
+        end
+        nil
+      end
+
       protected
 
       def new_request
@@ -204,13 +222,30 @@ module AWS
 
       end
 
-      def make_sync_request response
+      def make_sync_request response, &read_block
         retry_server_errors do
 
-          response.http_response = http_response =
-            Http::Response.new
+          response.http_response = Http::Response.new
+
+          @http_handler.handle(
+            response.http_request,
+            response.http_response,
+            &read_block)
+
+          if
+            block_given? and
+            response.http_response.status < 300 and
+            response.http_response.body
+          then
+
+            msg = ":http_handler read the entire http response body into "
+            msg << "memory, it should have instead yielded chunks"
+            log_warning(msg)
+
+            # go ahead and yield the body on behalf of the handler
+            yield(response.http_response.body)
 
-          @http_handler.handle(response.http_request, http_response)
+          end
 
           populate_error(response)
           response.signal_success unless response.error
@@ -227,7 +262,6 @@ module AWS
         while should_retry?(response)
           break if sleeps.empty?
           Kernel.sleep(sleeps.shift)
-          # rebuild the request to get a fresh signature
           rebuild_http_request(response)
           response = yield
         end
@@ -256,8 +290,16 @@ module AWS
       end
 
       def should_retry? response
+        if retryable_error?(response)
+          response.safe_to_retry?
+        else
+          false
+        end
+      end
+
+      def retryable_error? response
         expired_credentials?(response) or
-        response.timeout? or
+        response.network_error? or
         response.throttled? or
         response.error.kind_of?(Errors::ServerError)
       end
@@ -303,7 +345,7 @@ module AWS
       end
 
       # Logs the response to the configured logger.
-      # @param [Resposne] response
+      # @param [Response] response
       # @return [nil]
       def log_response response
         if config.logger
@@ -336,10 +378,10 @@ module AWS
         ]
 
         case
-        when response.timeout? then TimeoutError.new
-        when error_code        then error_class(error_code).new(*error_args)
-        when status >= 500     then Errors::ServerError.new(*error_args)
-        when status >= 300     then Errors::ClientError.new(*error_args)
+        when response.network_error? then NetworkError.new
+        when error_code              then error_class(error_code).new(*error_args)
+        when status >= 500           then Errors::ServerError.new(*error_args)
+        when status >= 300           then Errors::ClientError.new(*error_args)
         else nil # no error
         end
 
@@ -371,21 +413,25 @@ module AWS
         AWS.const_get(self.class.to_s[/(\w+)::Client/, 1])::Errors
       end
 
-      def client_request name, options, &block
+      def client_request name, options, &read_block
         return_or_raise(options) do
           log_client_request(options) do
 
             if config.stub_requests?
 
               response = stub_for(name)
-              response.http_request = build_request(name, options, &block)
+              response.http_request = build_request(name, options)
               response.request_options = options
               response
 
             else
 
               client = self
-              response = new_response { client.send(:build_request, name, options, &block) }
+
+              response = new_response do
+                client.send(:build_request, name, options)
+              end
+
               response.request_type = name
               response.request_options = options
 
@@ -399,8 +445,8 @@ module AWS
               else
                 # process the http request
                 options[:async] ?
-                make_async_request(response) :
-                  make_sync_request(response)
+                make_async_request(response, &read_block) :
+                  make_sync_request(response, &read_block)
 
                 # process the http response
                 response.on_success do
@@ -424,7 +470,7 @@ module AWS
         self.class::CACHEABLE_REQUESTS.include?(name)
       end
 
-      def build_request(name, options, &block)
+      def build_request name, options
 
         # we dont want to pass the async option to the configure block
         opts = options.dup
@@ -445,7 +491,7 @@ module AWS
         http_request.ssl_ca_file = config.ssl_ca_file if config.ssl_ca_file
         http_request.ssl_ca_path = config.ssl_ca_path if config.ssl_ca_path
 
-        send("configure_#{name}_request", http_request, opts, &block)
+        send("configure_#{name}_request", http_request, opts)
 
         http_request.headers["user-agent"] = user_agent_string
         http_request.add_authorization!(credential_provider)
@@ -474,16 +520,18 @@ module AWS
       #
       def self.add_client_request_method method_name, options = {}, &block
 
-        self.operations << method_name
+        operations << method_name
 
         ClientRequestMethodBuilder.new(self, method_name, &block)
 
-        module_eval <<-END
+        method_def = <<-METHOD
           def #{method_name}(*args, &block)
             options = args.first ? args.first : {}
             client_request(#{method_name.inspect}, options, &block)
           end
-        END
+        METHOD
+
+        module_eval(method_def)
 
       end
 
@@ -518,15 +566,6 @@ module AWS
         def configure_request options = {}, &block
           name = "configure_#{@method_name}_request"
           MetaUtils.class_extend_method(@client_class, name, &block)
-          if block.arity == 3
-            m = Module.new
-            m.module_eval(<<-END)
-              def #{name}(req, options, &block)
-                super(req, options, block)
-              end
-            END
-            @client_class.send(:include, m)
-          end
         end
 
         def process_response &block

data/lib/aws/core/configuration.rb

@@ -128,7 +128,7 @@ module AWS
     #   +true+, requests will always use path style.  This can be useful
     #   for testing environments.
     #
-    # @attr_reader [Integer] s3_multipart_max_parts (1000)
+    # @attr_reader [Integer] s3_multipart_max_parts (10000)
     #   The maximum number of parts to split a file into when uploading
     #   in parts to S3.
     #
@@ -163,6 +163,17 @@ module AWS
     #
     #     s3 = AWS::S3.new(:s3_server_side_encryption => :aes256)
     #
+    # @attr_reader [OpenSSL::PKey::RSA, String] s3_encryption_key
+    #   If this is set, AWS::S3::S3Object #read and #write methods will always
+    #   perform client-side encryption with this key. The key can be overridden
+    #   at runtime by using the :encryption_key option.  A value of nil
+    #   means that client-side encryption will not be used.
+    #
+    # @attr_reader [Symbol] s3_encryption_materials_location
+    #   When set to +:instruction_file+, AWS::S3::S3Object will store
+    #   encryption materials in a seperate object, instead of the object
+    #   metadata.
+    #
     # @attr_reader [String] simple_db_endpoint ('sdb.amazonaws.com')
     #   The service endpoint for Amazon SimpleDB.
     #

data/lib/aws/core/http/handler.rb

@@ -14,27 +14,39 @@
 module AWS
   module Core
     module Http
-      
+
       # @private
       class Handler
-  
+
         attr_reader :base
-  
+
         def initialize(base, &block)
           @base = base
           if base.respond_to?(:handle)
-  
-            unless block.arity == 2
-              raise ArgumentError, 'passed block must accept 2 arguments'
+
+            unless [2,3].include?(block.arity)
+              raise ArgumentError, 'passed block must accept 2 or 3 arguments'
             end
+
             MetaUtils.extend_method(self, :handle, &block)
-  
+
+            if block.arity == 3
+              m = Module.new do
+                eval(<<-DEF)
+                  def handle req, resp, &read_block
+                    super(req, resp, read_block)
+                  end
+                DEF
+              end
+              self.extend(m)
+            end
+
           elsif base.respond_to?(:handle_async)
-  
+
             unless block.arity == 3
               raise ArgumentError, 'passed block must accept 3 arguments'
             end
-  
+
             MetaUtils.extend_method(self, :handle_async) do |req, resp, handle|
               @base.handle_async(req, resp, handle)
             end
@@ -44,16 +56,16 @@ module AWS
               end
               define_method(:handle_async, &block)
             end
-  
+
           else
             raise ArgumentError, 'base must respond to #handle or #handle_async'
           end
         end
-  
-        def handle(request, http_response)
-          @base.handle(request, http_response)
+
+        def handle(request, http_response, &read_block)
+          @base.handle(request, http_response, &read_block)
         end
-  
+
         def handle_async(request, http_response, handle)
           Thread.new do
             begin
@@ -65,12 +77,12 @@ module AWS
             end
           end
         end
-  
+
         def sleep_with_callback seconds, &block
           Kernel.sleep(seconds)
           yield
         end
-  
+
       end
     end
   end

data/lib/aws/core/http/net_http_handler.rb

@@ -25,6 +25,18 @@ module AWS
       #
       class NetHttpHandler
 
+        # @private
+        NETWORK_ERRORS = [
+          EOFError,
+          IOError,
+          Errno::ECONNABORTED,
+          Errno::ECONNRESET,
+          Errno::EPIPE,
+          Errno::EINVAL,
+          Timeout::Error,
+          Errno::ETIMEDOUT,
+        ]
+
         # (see Net::HTTP::ConnectionPool.new)
         def initialize options = {}
           @pool = Net::HTTP::ConnectionPool.new(options)
@@ -39,7 +51,7 @@ module AWS
         # @param [Request] request
         # @param [Response] response
         # @return [nil]
-        def handle request, response
+        def handle request, response, &read_block
 
           options = {}
           options[:port] = request.port
@@ -49,17 +61,25 @@ module AWS
           options[:ssl_ca_file] = request.ssl_ca_file if request.ssl_ca_file
           options[:ssl_ca_path] = request.ssl_ca_path if request.ssl_ca_path
 
-          connection = pool.connection_for(request.host, options)
-          connection.read_timeout = request.read_timeout
-
           begin
-            http_response = connection.request(build_net_http_request(request))
-            response.body = http_response.body
-            response.status = http_response.code.to_i
-            response.headers = http_response.to_hash
-          rescue Timeout::Error, Errno::ETIMEDOUT => e
-            response.timeout = true
+
+            connection = pool.connection_for(request.host, options)
+            connection.read_timeout = request.read_timeout
+
+            connection.request(build_net_http_request(request)) do |http_resp|
+              response.status = http_resp.code.to_i
+              response.headers = http_resp.to_hash
+              if block_given? and response.status < 300
+                http_resp.read_body(&read_block)
+              else
+                response.body = http_resp.read_body
+              end
+            end
+
+          rescue *NETWORK_ERRORS
+            response.network_error = true
           end
+
           nil
 
         end
@@ -90,7 +110,7 @@ module AWS
             end
 
           net_http_req = request_class.new(request.uri, headers)
-          net_http_req.body = request.body
+          net_http_req.body_stream = request.body_stream
           net_http_req
 
         end

data/lib/aws/core/http/request.rb

@@ -36,7 +36,7 @@ module AWS
         #   60 seconds.
         attr_accessor :default_read_timeout
 
-        # @return [String] Returns hostname of the request.
+        # @return [String] hostname of the request
         attr_accessor :host
 
         # @return [Integer] Returns the port number this request will be
@@ -47,7 +47,7 @@ module AWS
         #   'POST', 'HEAD' or 'DELETE').  Defaults to 'POST'.
         attr_accessor :http_method
 
-        # @return [Hash] Returns a hash of header values.
+        # @return [CaseInsensitiveHash] request headers
         attr_accessor :headers
 
         # @return [String] Returns the request URI (path + querystring).
@@ -57,24 +57,18 @@ module AWS
         #   to be populated for requests against signature v4 endpoints.
         attr_accessor :region
 
-        # @return [String]
+        # @return [String] Returns the AWS access key ID used to authorize the
+        #   request.
         # @private
         attr_accessor :access_key_id
 
-        # @private
         # @return [Array<Param>] Returns an array of request params.  Requests
         #   that use signature version 2 add params to the request and then
         #   sign those before building the {#body}.  Normally the {#body}
         #   should be set directly with the HTTP payload.
+        # @private
         attr_accessor :params
 
-        # @return [String] Returns the HTTP request payload (body).
-        attr_accessor :body
-
-        # @return [String] Returns the AWS access key ID used to authorize the
-        #   request.
-        attr_accessor :access_key_id
-
         # @return [String] The name of the service for Signature v4 signing.
         #   This does not always match the ruby name (e.g.
         #   simple_email_service and ses do not match).
@@ -125,11 +119,6 @@ module AWS
           default_read_timeout
         end
 
-        # @return [String,nil] Returns the request body (payload).
-        def body
-          @body || url_encoded_params
-        end
-
         # @return [String] Returns the HTTP request path.
         def path
           uri.split(/\?/)[0]
@@ -167,6 +156,53 @@ module AWS
           params.empty? ? nil : params.sort.collect(&:encoded).join('&')
         end
 
+        # @param [String] body
+        def body= body
+          @body = body
+          if body
+            headers['content-length'] = body.size if body
+          else
+            headers.delete('content-length')
+          end
+        end
+
+        # @note Calling #body on a request with a #body_stream
+        #   will cause the entire stream to be read into memory.
+        # @return [String,nil] Returns the request body.
+        def body
+          if @body
+            @body
+          elsif @body_stream
+            @body = @body_stream.read
+            if @body_stream.respond_to?(:rewind)
+              @body_stream.rewind
+            else
+              @body_stream = StringIO.new(@body)
+            end
+            @body
+          else
+            nil
+          end
+        end
+
+        # Sets the request body as an IO object that will be streamed.
+        # @note You must also set the #headers['content-length']
+        # @param [IO] stream An object that responds to #read and #eof.
+        def body_stream= stream
+          @body_stream = stream
+        end
+
+        # @return [IO,nil]
+        def body_stream
+          if @body_stream
+            @body_stream
+          elsif @body
+            StringIO.new(@body)
+          else
+            nil
+          end
+        end
+
         # @private
         class CaseInsensitiveHash < Hash