aws-sdk-v1 1.52.0

data/lib/aws/core/http/curb_handler.rb

@@ -0,0 +1,148 @@
+# Copyright 2011-2013 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"). You
+# may not use this file except in compliance with the License. A copy of
+# the License is located at
+#
+#     http://aws.amazon.com/apache2.0/
+#
+# or in the "license" file accompanying this file. This file is
+# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
+# ANY KIND, either express or implied. See the License for the specific
+# language governing permissions and limitations under the License.
+
+require 'thread'
+
+module AWS
+  module Core
+    module Http
+
+      # @api private
+      class CurbHandler
+
+        def initialize
+          @q = []
+          @sem = Mutex.new
+          @multi = Curl::Multi.new
+
+          start_processor
+        end
+
+        def handle request, response, &read_block
+
+          raise "unsupport http reqest method: #{request.http_method}" unless
+            ['GET', 'HEAD', 'PUT', 'POST', 'DELETE'].include? request.http_method
+          @sem.synchronize do
+            @q << [request, response, read_block, Thread.current]
+            begin
+             @processor.wakeup
+            rescue ThreadError
+              start_processor
+            end
+          end
+          Thread.stop
+          nil
+        end
+
+        # fills the Curl::Multi handle with the given array of queue
+        # items, calling make_easy_handle on each one first
+        private
+        def fill_multi(items)
+          items.each do |item|
+            c = make_easy_handle(*item)
+            @multi.add(c)
+          end
+        end
+
+        # starts a background thread that waits for new items and
+        # sends them through the Curl::Multi handle
+        private
+        def start_processor
+          @processor = Thread.new do
+            loop do
+              items = nil
+              @sem.synchronize do
+                items = @q.slice!(0..-1)
+              end
+              unless items.empty?
+                fill_multi(items)
+                @multi.perform do
+                  # curl is idle, so process more items if we can get them
+                  # without blocking
+                  if !@q.empty? && @sem.try_lock
+                    begin
+                      fill_multi(@q.slice!(0..-1))
+                    ensure
+                      @sem.unlock
+                    end
+                  end
+                end
+              end
+
+              # wait for a new item to show up before continuing
+              Thread.stop if @q.empty?
+            end
+          end
+        end
+
+        private
+        def make_easy_handle request, response, read_block, thread = nil
+
+          protocol = request.use_ssl? ? 'https' : 'http'
+          url = "#{protocol}://#{request.host}:#{request.port}#{request.uri}"
+
+          curl = Curl::Easy.new(url)
+          # curl.verbose = true
+          request.headers.each {|k, v| curl.headers[k] = v}
+
+          curl.on_header {|header_data|
+            if header_data =~ /:\s+/
+              name, value = header_data.strip.split(/:\s+/, 2)
+              response.headers[name] = value
+            end
+            header_data.length
+          }
+
+          case request.http_method
+          when 'GET'
+            # ....
+          when 'HEAD'
+            curl.head = true
+          when 'PUT'
+            curl.put_data = request.body || ''
+          when 'POST'
+            curl.headers['Content-Type'] = curl.headers['Content-Type'] || ''
+            curl.post_body = request.body || ''
+          when 'DELETE'
+            curl.delete = true
+          end
+
+          buffer = []
+
+          if read_block
+            curl.on_body do |chunk|
+              read_block.call(chunk)
+              chunk.size
+            end
+          else
+            curl.on_body do |chunk|
+              buffer << chunk
+              chunk.size
+            end
+          end
+
+          curl.on_complete do
+            response.status = curl.response_code
+            unless read_block
+              response.body = buffer.join("")
+            end
+            thread.run if thread
+          end
+
+          curl
+        end
+
+      end
+    end
+  end
+end

data/lib/aws/core/http/handler.rb

@@ -0,0 +1,89 @@
+# Copyright 2011-2013 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"). You
+# may not use this file except in compliance with the License. A copy of
+# the License is located at
+#
+#     http://aws.amazon.com/apache2.0/
+#
+# or in the "license" file accompanying this file. This file is
+# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
+# ANY KIND, either express or implied. See the License for the specific
+# language governing permissions and limitations under the License.
+
+module AWS
+  module Core
+    module Http
+
+      # @api private
+      class Handler
+
+        attr_reader :base
+
+        def initialize(base, &block)
+          @base = base
+          if base.respond_to?(:handle)
+
+            unless [2,3].include?(block.arity)
+              raise ArgumentError, 'passed block must accept 2 or 3 arguments'
+            end
+
+            MetaUtils.extend_method(self, :handle, &block)
+
+            if block.arity == 3
+              m = Module.new do
+                eval(<<-DEF)
+                  def handle req, resp, &read_block
+                    super(req, resp, read_block)
+                  end
+                DEF
+              end
+              self.extend(m)
+            end
+
+          elsif base.respond_to?(:handle_async)
+
+            unless block.arity == 3
+              raise ArgumentError, 'passed block must accept 3 arguments'
+            end
+
+            MetaUtils.extend_method(self, :handle_async) do |req, resp, handle|
+              @base.handle_async(req, resp, handle)
+            end
+            MetaUtils.extend(self) do
+              define_method(:handle) do |req, resp|
+                raise "attempted to call #handle on an async handler"
+              end
+              define_method(:handle_async, &block)
+            end
+
+          else
+            raise ArgumentError, 'base must respond to #handle or #handle_async'
+          end
+        end
+
+        def handle(request, http_response, &read_block)
+          @base.handle(request, http_response, &read_block)
+        end
+
+        def handle_async(request, http_response, handle)
+          Thread.new do
+            begin
+              self.handle(request, http_response)
+            rescue => e
+              handle.signal_failure
+            else
+              handle.signal_success
+            end
+          end
+        end
+
+        def sleep_with_callback seconds, &block
+          Kernel.sleep(seconds)
+          yield
+        end
+
+      end
+    end
+  end
+end

data/lib/aws/core/http/net_http_handler.rb

@@ -0,0 +1,144 @@
+# Copyright 2011-2013 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"). You
+# may not use this file except in compliance with the License. A copy of
+# the License is located at
+#
+#     http://aws.amazon.com/apache2.0/
+#
+# or in the "license" file accompanying this file. This file is
+# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
+# ANY KIND, either express or implied. See the License for the specific
+# language governing permissions and limitations under the License.
+
+module AWS
+  module Core
+    module Http
+
+      # # NetHttpHandler
+      #
+      # This is the default HTTP handler for the aws-sdk gem.  It uses
+      # Ruby's Net::HTTP to make requests.  It uses persistent connections
+      # and a connection pool.
+      #
+      class NetHttpHandler
+
+        class TruncatedBodyError < IOError; end
+
+        # @api private
+        NETWORK_ERRORS = [
+          SocketError, EOFError, IOError, Timeout::Error,
+          Errno::ECONNABORTED, Errno::ECONNRESET, Errno::EPIPE,
+          Errno::EINVAL, Errno::ETIMEDOUT, OpenSSL::SSL::SSLError
+        ]
+
+        # (see ConnectionPool.new)
+        def initialize options = {}
+          @pool = options[:connection_pool] || ConnectionPool.new(options)
+          @verify_content_length = options[:verify_response_body_content_length]
+        end
+
+        # @return [ConnectionPool]
+        attr_reader :pool
+
+        # Given a populated request object and an empty response object,
+        # this method will make the request and them populate the
+        # response.
+        # @param [Request] request
+        # @param [Response] response
+        # @return [nil]
+        def handle request, response, &read_block
+          retry_possible = true
+
+          begin
+
+            @pool.session_for(request.endpoint) do |http|
+
+              http.read_timeout = request.read_timeout
+              http.continue_timeout = request.continue_timeout if
+                http.respond_to?(:continue_timeout=)
+
+              exp_length = nil
+              act_length = 0
+              http.request(build_net_http_request(request)) do |net_http_resp|
+                response.status = net_http_resp.code.to_i
+                response.headers = net_http_resp.to_hash
+                exp_length = determine_expected_content_length(response)
+                if block_given? and response.status < 300
+                  net_http_resp.read_body do |data|
+                    begin
+                      act_length += data.bytesize
+                      yield data unless data.empty?
+                    ensure
+                      retry_possible = false
+                    end
+                  end
+                else
+                  response.body = net_http_resp.read_body
+                  act_length += response.body.bytesize unless response.body.nil?
+                end
+              end
+              run_check = exp_length && request.http_method != "HEAD" && @verify_content_length
+              if run_check && act_length != exp_length
+                raise TruncatedBodyError, 'content-length does not match'
+              end
+            end
+
+          rescue *NETWORK_ERRORS => error
+            raise error unless retry_possible
+            response.network_error = error
+          end
+          nil
+        end
+
+        protected
+
+        def determine_expected_content_length response
+          if header = response.headers['content-length']
+            if header.is_a?(Array)
+              header.first.to_i
+            end
+          end
+        end
+
+        # Given an AWS::Core::HttpRequest, this method translates
+        # it into a Net::HTTPRequest (Get, Put, Post, Head or Delete).
+        # @param [Request] request
+        # @return [Net::HTTPRequest]
+        def build_net_http_request request
+
+          # Net::HTTP adds a content-type (1.8.7+) and accept-encoding (2.0.0+)
+          # to the request if these headers are not set.  Setting a default
+          # empty value defeats this.
+          #
+          # Removing these are necessary for most services to no break request
+          # signatures as well as dynamodb crc32 checks (these fail if the
+          # response is gzipped).
+          headers = { 'content-type' => '', 'accept-encoding' => '' }
+
+          request.headers.each_pair do |key,value|
+            headers[key] = value.to_s
+          end
+
+          request_class = case request.http_method
+            when 'GET'    then Net::HTTP::Get
+            when 'PUT'    then Net::HTTP::Put
+            when 'POST'   then Net::HTTP::Post
+            when 'HEAD'   then Net::HTTP::Head
+            when 'DELETE' then Net::HTTP::Delete
+            else
+              msg = "unsupported http method: #{request.http_method}"
+              raise ArgumentError, msg
+            end
+
+          net_http_req = request_class.new(request.uri, headers)
+          net_http_req.body_stream = request.body_stream
+          net_http_req
+
+        end
+
+      end
+
+    end
+  end
+end

data/lib/aws/core/http/patch.rb

@@ -0,0 +1,102 @@
+# Copyright 2011-2013 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"). You
+# may not use this file except in compliance with the License. A copy of
+# the License is located at
+#
+#     http://aws.amazon.com/apache2.0/
+#
+# or in the "license" file accompanying this file. This file is
+# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
+# ANY KIND, either express or implied. See the License for the specific
+# language governing permissions and limitations under the License.
+
+require 'net/http'
+require 'net/https'
+require 'thread'
+require 'logger'
+
+module AWS
+  module Core
+    module Http
+
+      def self.patch_net_http_100_continue!
+        if RUBY_VERSION >= '2.0'
+          Net::HTTP.send(:include, Expect100Continue20)
+        elsif RUBY_VERSION >= '1.9.3'
+          Net::HTTP.send(:include, Expect100Continue19)
+        else
+          msg = "Expect 100-continue not supported in Ruby < 1.9.3"
+          raise NotImplementedError, msg
+        end
+        Net::HTTP.send(:alias_method, :old_transport_request, :transport_request)
+        Net::HTTP.send(:alias_method, :transport_request, :new_transport_request)
+      end
+
+      # @api private
+      module Expect100Continue19
+        def new_transport_request(req)
+          begin_transport req
+          res = catch(:response) {
+            req.exec @socket, @curr_http_version, edit_path(req.path)
+            begin
+              res = Net::HTTPResponse.read_new(@socket)
+            end while res.kind_of?(Net::HTTPContinue)
+            res
+          }
+          res.reading_body(@socket, req.response_body_permitted?) {
+            yield res if block_given?
+          }
+          end_transport req, res
+          res
+        rescue => exception
+          D "Conn close because of error #{exception}"
+          @socket.close if @socket and not @socket.closed?
+          raise exception
+        end
+      end
+
+      # @api private
+      module Expect100Continue20
+        def new_transport_request(req)
+          count = 0
+          begin
+            begin_transport req
+            res = catch(:response) {
+              req.exec @socket, @curr_http_version, edit_path(req.path)
+              begin
+                res = Net::HTTPResponse.read_new(@socket)
+                res.decode_content = req.decode_content
+              end while res.kind_of?(Net::HTTPContinue)
+
+              res.uri = req.uri
+
+              res
+            }
+            res.reading_body(@socket, req.response_body_permitted?) {
+              yield res if block_given?
+            }
+          rescue Net::OpenTimeout
+            raise
+          rescue Net::ReadTimeout, IOError, EOFError,
+                 Errno::ECONNRESET, Errno::ECONNABORTED, Errno::EPIPE,
+                 # avoid a dependency on OpenSSL
+                 defined?(OpenSSL::SSL) ? OpenSSL::SSL::SSLError : IOError,
+                 Timeout::Error => exception
+            if count == 0 && IDEMPOTENT_METHODS_.include?(req.method)
+              count += 1
+              @socket.close if @socket and not @socket.closed?
+              D "Conn close because of error #{exception}, and retry"
+              retry
+            end
+            D "Conn close because of error #{exception}"
+            @socket.close if @socket and not @socket.closed?
+            raise
+          end
+
+        end
+      end
+
+    end
+  end
+end