excon 0.16.10 → 0.17.0

data/Gemfile.lock

@@ -1,7 +1,7 @@
 PATH
   remote: .
   specs:
-    excon (0.16.10)
+    excon (0.17.0)
 
 GEM
   remote: http://rubygems.org/

data/README.md

@@ -5,6 +5,7 @@ Http(s) EXtended CONnections
 
 [![Build Status](https://secure.travis-ci.org/geemus/excon.png)](http://travis-ci.org/geemus/excon)
 [![Dependency Status](https://gemnasium.com/geemus/excon.png)](https://gemnasium.com/geemus/excon)
+[![Gem Version](https://fury-badge.herokuapp.com/rb/excon.png)](http://badge.fury.io/rb/excon)
 
 Getting Started
 ---------------
@@ -57,9 +58,21 @@ Both one-off and persistent connections support many other options. Here are a f
     # this request can be repeated safely, so retry on errors up to 3 times
     connection.request(:idempotent => true)
 
+    # this request can be repeated safely, retry up to 6 times
+    connection.request(:idempotent => true, :retry_limit => 6)
+
     # opt out of nonblocking operations for performance and/or as a workaround
     connection.request(:nonblock => false)
 
+    # set longer connect_timeout (default is 60 seconds)
+    connection.request(:connect_timeout => 360)
+
+    # set longer read_timeout (default is 60 seconds)
+    connection.request(:read_timeout => 360)
+
+    # set longer write_timeout (default is 60 seconds)
+    connection.request(:write_timeout => 360)
+
 These options can be combined to make pretty much any request you might need.
 
 Excon can also expect one or more HTTP status code in response, raising an exception if the response does not meet the criteria.

data/excon.gemspec

@@ -13,8 +13,8 @@ Gem::Specification.new do |s|
   ## If your rubyforge_project name is different, then edit it and comment out
   ## the sub! line in the Rakefile
   s.name              = 'excon'
-  s.version           = '0.16.10'
-  s.date              = '2012-11-16'
+  s.version           = '0.17.0'
+  s.date              = '2013-02-01'
   s.rubyforge_project = 'excon'
 
   ## Make sure your summary is short. The description may be as long
@@ -96,6 +96,8 @@ Gem::Specification.new do |s|
     lib/excon/connection.rb
     lib/excon/constants.rb
     lib/excon/errors.rb
+    lib/excon/middlewares/expects.rb
+    lib/excon/middlewares/instrumentor.rb
     lib/excon/response.rb
     lib/excon/socket.rb
     lib/excon/ssl_socket.rb
@@ -106,7 +108,7 @@ Gem::Specification.new do |s|
     tests/data/xs
     tests/header_tests.rb
     tests/idempotent_tests.rb
-    tests/instrumentation_tests.rb
+    tests/middlewares/instrumentation_tests.rb
     tests/proxy_tests.rb
     tests/query_string_tests.rb
     tests/rackups/basic.rb
@@ -114,11 +116,13 @@ Gem::Specification.new do |s|
     tests/rackups/basic_auth.ru
     tests/rackups/proxy.ru
     tests/rackups/query_string.ru
+    tests/rackups/request_headers.ru
     tests/rackups/request_methods.ru
     tests/rackups/response_header.ru
     tests/rackups/ssl.ru
     tests/rackups/thread_safety.ru
     tests/rackups/timeout.ru
+    tests/request_headers_tests.rb
     tests/request_method_tests.rb
     tests/servers/bad.rb
     tests/servers/eof.rb

data/lib/excon.rb

@@ -19,6 +19,7 @@ module Excon
         :chunk_size         => CHUNK_SIZE || DEFAULT_CHUNK_SIZE,
         :connect_timeout    => 60,
         :headers            => {},
+        :idempotent         => false,
         :instrumentor_name  => 'excon',
         :mock               => false,
         :nonblock           => DEFAULT_NONBLOCK,
@@ -42,6 +43,8 @@ end
 require 'excon/constants'
 require 'excon/connection'
 require 'excon/errors'
+require 'excon/middlewares/expects'
+require 'excon/middlewares/instrumentor'
 require 'excon/response'
 require 'excon/socket'
 require 'excon/ssl_socket'
@@ -108,7 +111,7 @@ module Excon
         request_params.update(
           :host              => uri.host,
           :path              => uri.path,
-          :port              => uri.port.to_s,
+          :port              => uri.port,
           :query             => uri.query,
           :scheme            => uri.scheme
         )

data/lib/excon/connection.rb

@@ -1,6 +1,25 @@
 module Excon
   class Connection
-    attr_reader :connection, :proxy
+
+    attr_reader :data
+
+    def params
+      $stderr.puts("Excon::Connection#params is deprecated use Excon::Connection#data instead (#{caller.first})")
+      @data
+    end
+    def params=(new_params)
+      $stderr.puts("Excon::Connection#params= is deprecated use Excon::Connection#data= instead (#{caller.first})")
+      @data = new_params
+    end
+
+    def proxy
+      $stderr.puts("Excon::Connection#proxy is deprecated use Excon::Connection#data[:proxy] instead (#{caller.first})")
+      @data[:proxy]
+    end
+    def proxy=(new_proxy)
+      $stderr.puts("Excon::Connection#proxy= is deprecated use Excon::Connection#data[:proxy]= instead (#{caller.first})")
+      @data[:proxy] = new_proxy
+    end
 
     # Initializes a new Connection instance
     #   @param [String] url The destination URL
@@ -18,49 +37,153 @@ module Excon
     #     @option params [String] :instrumentor_name Name prefix for #instrument events.  Defaults to 'excon'
     def initialize(url, params = {})
       uri = URI.parse(url)
-      @connection = Excon.defaults.merge({
+      @data = Excon.defaults.merge({
         :host       => uri.host,
         :host_port  => '' << uri.host << ':' << uri.port.to_s,
         :path       => uri.path,
-        :port       => uri.port.to_s,
+        :port       => uri.port,
         :query      => uri.query,
         :scheme     => uri.scheme,
       }).merge!(params)
       # merge does not deep-dup, so make sure headers is not the original
-      @connection[:headers] = @connection[:headers].dup
-
-      @proxy = nil
+      @data[:headers] = @data[:headers].dup
 
-      if @connection[:scheme] == HTTPS && (ENV.has_key?('https_proxy') || ENV.has_key?('HTTPS_PROXY'))
-        @proxy = setup_proxy(ENV['https_proxy'] || ENV['HTTPS_PROXY'])
+      if @data[:scheme] == HTTPS && (ENV.has_key?('https_proxy') || ENV.has_key?('HTTPS_PROXY'))
+        @data[:proxy] = setup_proxy(ENV['https_proxy'] || ENV['HTTPS_PROXY'])
       elsif (ENV.has_key?('http_proxy') || ENV.has_key?('HTTP_PROXY'))
-        @proxy = setup_proxy(ENV['http_proxy'] || ENV['HTTP_PROXY'])
-      elsif @connection.has_key?(:proxy)
-        @proxy = setup_proxy(@connection[:proxy])
+        @data[:proxy] = setup_proxy(ENV['http_proxy'] || ENV['HTTP_PROXY'])
+      elsif @data.has_key?(:proxy)
+        @data[:proxy] = setup_proxy(@data[:proxy])
       end
 
-      if @proxy
-        @connection[:headers]['Proxy-Connection'] ||= 'Keep-Alive'
+      if @data[:proxy]
+        @data[:headers]['Proxy-Connection'] ||= 'Keep-Alive'
         # https credentials happen in handshake
-        if @connection[:scheme] == 'http' && (@proxy[:user] || @proxy[:password])
-          auth = ['' << @proxy[:user].to_s << ':' << @proxy[:password].to_s].pack('m').delete(Excon::CR_NL)
-          @connection[:headers]['Proxy-Authorization'] = 'Basic ' << auth
+        if @data[:scheme] == 'http' && (@data[:proxy][:user] || @data[:proxy][:password])
+          auth = ['' << @data[:proxy][:user].to_s << ':' << @data[:proxy][:password].to_s].pack('m').delete(Excon::CR_NL)
+          @data[:headers]['Proxy-Authorization'] = 'Basic ' << auth
         end
       end
 
       if ENV.has_key?('EXCON_DEBUG') || ENV.has_key?('EXCON_STANDARD_INSTRUMENTOR')
-        @connection[:instrumentor] = Excon::StandardInstrumentor
+        @data[:instrumentor] = Excon::StandardInstrumentor
       end
 
       # Use Basic Auth if url contains a login
       if uri.user || uri.password
-        @connection[:headers]['Authorization'] ||= 'Basic ' << ['' << uri.user.to_s << ':' << uri.password.to_s].pack('m').delete(Excon::CR_NL)
+        @data[:headers]['Authorization'] ||= 'Basic ' << ['' << uri.user.to_s << ':' << uri.password.to_s].pack('m').delete(Excon::CR_NL)
       end
 
-      @socket_key = '' << @connection[:host_port]
+      @socket_key = '' << @data[:host_port]
       reset
     end
 
+    def call(datum)
+      begin
+        response_datum = if datum[:mock]
+          invoke_stub(datum)
+        else
+          socket.data = datum
+          # start with "METHOD /path"
+          request = datum[:method].to_s.upcase << ' '
+          if @data[:proxy]
+            request << datum[:scheme] << '://' << datum[:host_port]
+          end
+          request << datum[:path]
+
+          # add query to path, if there is one
+          case datum[:query]
+          when String
+            request << '?' << datum[:query]
+          when Hash
+            request << '?'
+            datum[:query].each do |key, values|
+              if values.nil?
+                request << key.to_s << '&'
+              else
+                [values].flatten.each do |value|
+                  request << key.to_s << '=' << CGI.escape(value.to_s) << '&'
+                end
+              end
+            end
+            request.chop! # remove trailing '&'
+          end
+
+          # finish first line with "HTTP/1.1\r\n"
+          request << HTTP_1_1
+
+          if datum.has_key?(:request_block)
+            datum[:headers]['Transfer-Encoding'] = 'chunked'
+          elsif ! (datum[:method].to_s.casecmp('GET') == 0 && datum[:body].nil?)
+            # The HTTP spec isn't clear on it, but specifically, GET requests don't usually send bodies;
+            # if they don't, sending Content-Length:0 can cause issues.
+            datum[:headers]['Content-Length'] = detect_content_length(datum[:body])
+          end
+
+          # add headers to request
+          datum[:headers].each do |key, values|
+            [values].flatten.each do |value|
+              request << key.to_s << ': ' << value.to_s << CR_NL
+            end
+          end
+
+          # add additional "\r\n" to indicate end of headers
+          request << CR_NL
+
+          # write out the request, sans body
+          socket.write(request)
+
+          # write out the body
+          if datum.has_key?(:request_block)
+            while true
+              chunk = datum[:request_block].call
+              if FORCE_ENC
+                chunk.force_encoding('BINARY')
+              end
+              if chunk.length > 0
+                socket.write(chunk.length.to_s(16) << CR_NL << chunk << CR_NL)
+              else
+                socket.write('0' << CR_NL << CR_NL)
+                break
+              end
+            end
+          elsif !datum[:body].nil?
+            if datum[:body].is_a?(String)
+              unless datum[:body].empty?
+                socket.write(datum[:body])
+              end
+            else
+              if datum[:body].respond_to?(:binmode)
+                datum[:body].binmode
+              end
+              if datum[:body].respond_to?(:pos=)
+                datum[:body].pos = 0
+              end
+              while chunk = datum[:body].read(datum[:chunk_size])
+                socket.write(chunk)
+              end
+            end
+          end
+
+          # read the response
+          response_datum = Excon::Response.parse(socket, datum)
+
+          if response_datum[:headers]['Connection'] == 'close'
+            reset
+          end
+
+          response_datum
+        end
+      rescue Excon::Errors::StubNotFound, Excon::Errors::Timeout => error
+        raise(error)
+      rescue => socket_error
+        reset
+        raise(Excon::Errors::SocketError.new(socket_error))
+      end
+
+      response_datum
+    end
+
     # Sends the supplied request to the destination host.
     #   @yield [chunk] @see Response#self.parse
     #   @param [Hash<Symbol, >] params One or more optional params, override defaults set in Connection.new
@@ -72,53 +195,50 @@ module Excon
     #     @option params [Hash]   :query appended to the 'scheme://host:port/path/' in the form of '?key=value'
     #     @option params [String] :scheme The protocol; 'https' causes OpenSSL to be used
     def request(params, &block)
-      # connection has defaults, merge in new params to override
-      params = @connection.merge(params)
-      params[:host_port]  = '' << params[:host] << ':' << params[:port].to_s
-      params[:headers] = @connection[:headers].merge(params[:headers] || {})
-      params[:headers]['Host'] ||= '' << params[:host_port]
+      # @data has defaults, merge in new params to override
+      datum = @data.merge(params)
+      datum[:host_port]  = '' << datum[:host] << ':' << datum[:port].to_s
+      datum[:headers] = @data[:headers].merge(datum[:headers] || {})
+      datum[:headers]['Host'] = '' << datum[:host_port]
+      datum[:retries_remaining] ||= datum[:retry_limit]
 
       # if path is empty or doesn't start with '/', insert one
-      unless params[:path][0, 1] == '/'
-        params[:path].insert(0, '/')
+      unless datum[:path][0, 1] == '/'
+        datum[:path].insert(0, '/')
       end
 
       if block_given?
         $stderr.puts("Excon requests with a block are deprecated, pass :response_block instead (#{caller.first})")
-        params[:response_block] = Proc.new
+        datum[:response_block] = Proc.new
       end
 
-      if params.has_key?(:instrumentor)
-        if (retries_remaining ||= params[:retry_limit]) < params[:retry_limit]
-          event_name = "#{params[:instrumentor_name]}.retry"
-        else
-          event_name = "#{params[:instrumentor_name]}.request"
-        end
-        response = params[:instrumentor].instrument(event_name, params) do
-          request_kernel(params)
-        end
-        params[:instrumentor].instrument("#{params[:instrumentor_name]}.response", response.attributes)
-        response
-      else
-        request_kernel(params)
+      datum[:middlewares] = [
+        lambda {|app| Excon::Middleware::Instrumentor.new(app) },
+        lambda {|app| Excon::Middleware::Expects.new(app) }
+      ]
+      stack = datum[:middlewares].reverse.inject(self) do |middlewares, middleware|
+        middleware.call(middlewares)
       end
+      response_datum = stack.call(datum)
+
+      Excon::Response.new(response_datum)
     rescue => request_error
-      if params[:idempotent] && [Excon::Errors::Timeout, Excon::Errors::SocketError,
+      if datum[:idempotent] && [Excon::Errors::Timeout, Excon::Errors::SocketError,
           Excon::Errors::HTTPStatusError].any? {|ex| request_error.kind_of? ex }
-        retries_remaining ||= params[:retry_limit]
-        retries_remaining -= 1
-        if retries_remaining > 0
-          retry
+        datum[:retries_remaining] -= 1
+        if datum[:retries_remaining] > 0
+          request(datum, &block)
         else
-          if params.has_key?(:instrumentor)
-            params[:instrumentor].instrument("#{params[:instrumentor_name]}.error", :error => request_error)
+          if datum.has_key?(:instrumentor)
+            datum[:instrumentor].instrument("#{datum[:instrumentor_name]}.error", :error => request_error)
           end
           raise(request_error)
         end
       else
-        if params.has_key?(:instrumentor)
-          params[:instrumentor].instrument("#{params[:instrumentor_name]}.error", :error => request_error)
+        if datum.has_key?(:instrumentor)
+          datum[:instrumentor].instrument("#{datum[:instrumentor_name]}.error", :error => request_error)
         end
+        reset
         raise(request_error)
       end
     end
@@ -138,22 +258,22 @@ module Excon
 
     def retry_limit=(new_retry_limit)
       $stderr.puts("Excon::Connection#retry_limit= is deprecated, pass :retry_limit to the initializer (#{caller.first})")
-      @connection[:retry_limit] = new_retry_limit
+      @data[:retry_limit] = new_retry_limit
     end
 
     def retry_limit
       $stderr.puts("Excon::Connection#retry_limit is deprecated, pass :retry_limit to the initializer (#{caller.first})")
-      @connection[:retry_limit] ||= DEFAULT_RETRY_LIMIT
+      @data[:retry_limit] ||= DEFAULT_RETRY_LIMIT
     end
 
     def inspect
       vars = instance_variables.inject({}) do |accum, var|
         accum.merge!(var.to_sym => instance_variable_get(var))
       end
-      if vars[:'@connection'][:headers].has_key?('Authorization')
-        vars[:'@connection'] = vars[:'@connection'].dup
-        vars[:'@connection'][:headers] = vars[:'@connection'][:headers].dup
-        vars[:'@connection'][:headers]['Authorization'] = REDACTED
+      if vars[:'@data'][:headers].has_key?('Authorization')
+        vars[:'@data'] = vars[:'@data'].dup
+        vars[:'@data'][:headers] = vars[:'@data'][:headers].dup
+        vars[:'@data'][:headers]['Authorization'] = REDACTED
       end
       inspection = '#<Excon::Connection:'
       inspection << (object_id << 1).to_s(16)
@@ -184,186 +304,74 @@ module Excon
       end
     end
 
-    def request_kernel(params)
-      begin
-        response = if params[:mock]
-          invoke_stub(params)
-        else
-          socket.params = params
-          # start with "METHOD /path"
-          request = params[:method].to_s.upcase << ' '
-          if @proxy
-            request << params[:scheme] << '://' << params[:host_port]
-          end
-          request << params[:path]
-
-          # add query to path, if there is one
-          case params[:query]
-          when String
-            request << '?' << params[:query]
-          when Hash
-            request << '?'
-            params[:query].each do |key, values|
-              if values.nil?
-                request << key.to_s << '&'
-              else
-                [*values].each do |value|
-                  request << key.to_s << '=' << CGI.escape(value.to_s) << '&'
-                end
-              end
-            end
-            request.chop! # remove trailing '&'
-          end
-
-          # finish first line with "HTTP/1.1\r\n"
-          request << HTTP_1_1
-
-          if params.has_key?(:request_block)
-            params[:headers]['Transfer-Encoding'] = 'chunked'
-          elsif ! (params[:method].to_s.casecmp('GET') == 0 && params[:body].nil?)
-            # The HTTP spec isn't clear on it, but specifically, GET requests don't usually send bodies;
-            # if they don't, sending Content-Length:0 can cause issues.
-            params[:headers]['Content-Length'] = detect_content_length(params[:body])
-          end
-
-          # add headers to request
-          params[:headers].each do |key, values|
-            [*values].each do |value|
-              request << key.to_s << ': ' << value.to_s << CR_NL
-            end
-          end
-
-          # add additional "\r\n" to indicate end of headers
-          request << CR_NL
-
-          # write out the request, sans body
-          socket.write(request)
-
-          # write out the body
-          if params.has_key?(:request_block)
-            while true
-              chunk = params[:request_block].call
-              if FORCE_ENC
-                chunk.force_encoding('BINARY')
-              end
-              if chunk.length > 0
-                socket.write(chunk.length.to_s(16) << CR_NL << chunk << CR_NL)
-              else
-                socket.write('0' << CR_NL << CR_NL)
-                break
-              end
-            end
-          elsif !params[:body].nil?
-            if params[:body].is_a?(String)
-              unless params[:body].empty?
-                socket.write(params[:body])
-              end
-            else
-              if params[:body].respond_to?(:binmode)
-                params[:body].binmode
-              end
-              if params[:body].respond_to?(:pos=)
-                params[:body].pos = 0
-              end
-              while chunk = params[:body].read(params[:chunk_size])
-                socket.write(chunk)
-              end
-            end
-          end
-
-          # read the response
-          response = Excon::Response.parse(socket, params)
-
-          if response.headers['Connection'] == 'close'
-            reset
-          end
-
-          response
-        end
-      rescue Excon::Errors::StubNotFound, Excon::Errors::Timeout => error
-        raise(error)
-      rescue => socket_error
-        reset
-        raise(Excon::Errors::SocketError.new(socket_error))
-      end
-
-      if params.has_key?(:expects) && ![*params[:expects]].include?(response.status)
-        reset
-        raise(Excon::Errors.status_error(params, response))
-      else
-        response
-      end
-    end
-
-    def invoke_stub(params)
-
+    def invoke_stub(datum)
       # convert File/Tempfile body to string before matching:
-      unless params[:body].nil? || params[:body].is_a?(String)
-       if params[:body].respond_to?(:binmode)
-         params[:body].binmode
+      unless datum[:body].nil? || datum[:body].is_a?(String)
+       if datum[:body].respond_to?(:binmode)
+         datum[:body].binmode
        end
-       if params[:body].respond_to?(:rewind)
-         params[:body].rewind
+       if datum[:body].respond_to?(:rewind)
+         datum[:body].rewind
        end
-       params[:body] = params[:body].read
+       datum[:body] = datum[:body].read
       end
 
-      params[:captures] = {:headers => {}} # setup data to hold captures
+      datum[:captures] = {:headers => {}} # setup data to hold captures
       Excon.stubs.each do |stub, response|
         headers_match = !stub.has_key?(:headers) || stub[:headers].keys.all? do |key|
           case value = stub[:headers][key]
           when Regexp
-            if match = value.match(params[:headers][key])
-              params[:captures][:headers][key] = match.captures
+            if match = value.match(datum[:headers][key])
+              datum[:captures][:headers][key] = match.captures
             end
             match
           else
-            value == params[:headers][key]
+            value == datum[:headers][key]
           end
         end
         non_headers_match = (stub.keys - [:headers]).all? do |key|
           case value = stub[key]
           when Regexp
-            if match = value.match(params[key])
-              params[:captures][key] = match.captures
+            if match = value.match(datum[key])
+              datum[:captures][key] = match.captures
             end
             match
           else
-            value == params[key]
+            value == datum[key]
           end
         end
         if headers_match && non_headers_match
-          response_attributes = case response
+          response_datum = case response
           when Proc
-            response.call(params)
+            response.call(datum)
           else
             response
           end
 
-          if params[:expects] && ![*params[:expects]].include?(response_attributes[:status])
+          if datum[:expects] && ![*datum[:expects]].include?(response_datum[:status])
             # don't pass stuff into a block if there was an error
-          elsif params.has_key?(:response_block) && response_attributes.has_key?(:body)
-            body = response_attributes.delete(:body)
+          elsif datum.has_key?(:response_block) && response_datum.has_key?(:body)
+            body = response_datum.delete(:body)
             content_length = remaining = body.bytesize
             i = 0
             while i < body.length
-              params[:response_block].call(body[i, params[:chunk_size]], [remaining - params[:chunk_size], 0].max, content_length)
-              remaining -= params[:chunk_size]
-              i += params[:chunk_size]
+              datum[:response_block].call(body[i, datum[:chunk_size]], [remaining - datum[:chunk_size], 0].max, content_length)
+              remaining -= datum[:chunk_size]
+              i += datum[:chunk_size]
             end
           end
-          return Excon::Response.new(response_attributes)
+          return response_datum
         end
       end
       # if we reach here no stubs matched
-      raise(Excon::Errors::StubNotFound.new('no stubs matched ' << params.inspect))
+      raise(Excon::Errors::StubNotFound.new('no stubs matched ' << datum.inspect))
     end
 
     def socket
-      sockets[@socket_key] ||= if @connection[:scheme] == HTTPS
-        Excon::SSLSocket.new(@connection, @proxy)
+      sockets[@socket_key] ||= if @data[:scheme] == HTTPS
+        Excon::SSLSocket.new(@data)
       else
-        Excon::Socket.new(@connection, @proxy)
+        Excon::Socket.new(@data)
       end
     end
 
@@ -372,18 +380,23 @@ module Excon
     end
 
     def setup_proxy(proxy)
-      uri = URI.parse(proxy)
-      unless uri.host and uri.port and uri.scheme
-        raise Excon::Errors::ProxyParseError, "Proxy is invalid"
+      case proxy
+      when String
+        uri = URI.parse(proxy)
+        unless uri.host and uri.port and uri.scheme
+          raise Excon::Errors::ProxyParseError, "Proxy is invalid"
+        end
+        {
+          :host       => uri.host,
+          :host_port  => '' << uri.host << ':' << uri.port.to_s,
+          :password   => uri.password,
+          :port       => uri.port,
+          :scheme     => uri.scheme,
+          :user       => uri.user
+        }
+      else
+        proxy
       end
-      {
-        :host       => uri.host,
-        :host_port  => '' << uri.host << ':' << uri.port.to_s,
-        :password   => uri.password,
-        :port       => uri.port,
-        :scheme     => uri.scheme,
-        :user       => uri.user
-      }
     end
 
   end