excon 0.62.0 → 0.92.3

data/lib/excon/constants.rb

@@ -12,8 +12,16 @@ module Excon
     CHUNK_SIZE = DEFAULT_CHUNK_SIZE
   end
 
+  DEFAULT_REDIRECT_LIMIT = 10
+
   DEFAULT_RETRY_LIMIT = 4
 
+  DEFAULT_RETRY_ERRORS = [
+    Excon::Error::Timeout,
+    Excon::Error::Socket,
+    Excon::Error::HTTPStatus
+  ]
+
   FORCE_ENC = CR_NL.respond_to?(:force_encoding)
 
   HTTP_1_1 = " HTTP/1.1\r\n"
@@ -33,19 +41,17 @@ module Excon
   VERSIONS = "#{USER_AGENT} (#{RUBY_PLATFORM}) ruby/#{RUBY_VERSION}"
 
   VALID_REQUEST_KEYS = [
+    :allow_unstubbed_requests,
     :body,
-    :captures,
     :chunk_size,
     :debug_request,
     :debug_response,
-    :expects,
     :headers,
-    :idempotent,
-    :instrumentor,
-    :instrumentor_name,
+    :instrumentor, # Used for setting logging within Connection
+    :logger,
     :method,
     :middlewares,
-    :mock,
+    :password,
     :path,
     :persistent,
     :pipeline,
@@ -53,9 +59,8 @@ module Excon
     :read_timeout,
     :request_block,
     :response_block,
-    :retries_remaining, # used internally
-    :retry_limit,
-    :retry_interval,
+    :stubs,
+    :user,
     :versions,
     :write_timeout
   ]
@@ -74,12 +79,12 @@ module Excon
     :private_key_path,
     :connect_timeout,
     :family,
+    :keepalive,
     :host,
     :hostname,
     :omit_default_port,
     :nonblock,
     :reuseaddr,
-    :password,
     :port,
     :proxy,
     :scheme,
@@ -90,13 +95,30 @@ module Excon
     :ssl_verify_callback,
     :ssl_verify_peer,
     :ssl_verify_peer_host,
+    :ssl_verify_hostname,
     :ssl_version,
+    :ssl_min_version,
+    :ssl_max_version,
+    :ssl_security_level,
+    :ssl_proxy_headers,
+    :ssl_uri_schemes,
     :tcp_nodelay,
     :thread_safe_sockets,
     :uri_parser,
-    :user
   ]
 
+  DEPRECATED_VALID_REQUEST_KEYS = {
+    :captures => 'Mock',
+    :expects => 'Expects',
+    :idempotent => 'Idempotent',
+    :instrumentor_name => 'Instrumentor',
+    :mock => 'Mock',
+    :retries_remaining => 'Idempotent', # referenced in Instrumentor, but only relevant with Idempotent
+    :retry_errors => 'Idempotent',
+    :retry_interval => 'Idempotent',
+    :retry_limit => 'Idempotent' # referenced in Instrumentor, but only relevant with Idempotent
+  }
+
   unless ::IO.const_defined?(:WaitReadable)
     class ::IO
       module WaitReadable; end
@@ -112,12 +134,14 @@ module Excon
   DEFAULTS = {
     :chunk_size           => CHUNK_SIZE || DEFAULT_CHUNK_SIZE,
     # see https://wiki.mozilla.org/Security/Server_Side_TLS#Intermediate_compatibility_.28default.29
-    :ciphers              => 'ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA256:ECDHE-ECDSA-AES128-SHA:ECDHE-RSA-AES256-SHA384:ECDHE-RSA-AES128-SHA:ECDHE-ECDSA-AES256-SHA384:ECDHE-ECDSA-AES256-SHA:ECDHE-RSA-AES256-SHA:DHE-RSA-AES128-SHA256:DHE-RSA-AES128-SHA:DHE-RSA-AES256-SHA256:DHE-RSA-AES256-SHA:ECDHE-ECDSA-DES-CBC3-SHA:ECDHE-RSA-DES-CBC3-SHA:EDH-RSA-DES-CBC3-SHA:AES128-GCM-SHA256:AES256-GCM-SHA384:AES128-SHA256:AES256-SHA256:AES128-SHA:AES256-SHA:DES-CBC3-SHA:!DSS',
+    # list provided then had DES related things sorted to the end
+    :ciphers              => 'ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA256:ECDHE-ECDSA-AES128-SHA:ECDHE-RSA-AES256-SHA384:ECDHE-RSA-AES128-SHA:ECDHE-ECDSA-AES256-SHA384:ECDHE-ECDSA-AES256-SHA:ECDHE-RSA-AES256-SHA:DHE-RSA-AES128-SHA256:DHE-RSA-AES128-SHA:DHE-RSA-AES256-SHA256:DHE-RSA-AES256-SHA:AES128-GCM-SHA256:AES256-GCM-SHA384:AES128-SHA256:AES256-SHA256:AES128-SHA:AES256-SHA:ECDHE-ECDSA-DES-CBC3-SHA:ECDHE-RSA-DES-CBC3-SHA:EDH-RSA-DES-CBC3-SHA:DES-CBC3-SHA:!DSS',
     :connect_timeout      => 60,
     :debug_request        => false,
     :debug_response       => false,
     :headers              => {
-      'User-Agent' => USER_AGENT
+      'User-Agent' => USER_AGENT,
+      'Accept'     =>  '*/*'
     },
     :idempotent           => false,
     :instrumentor_name    => 'excon',
@@ -133,6 +157,7 @@ module Excon
     :omit_default_port    => false,
     :persistent           => false,
     :read_timeout         => 60,
+    :retry_errors         => DEFAULT_RETRY_ERRORS,
     :retry_limit          => DEFAULT_RETRY_LIMIT,
     :ssl_verify_peer      => true,
     :ssl_uri_schemes      => [HTTPS],

data/lib/excon/error.rb

@@ -6,6 +6,7 @@ module Excon
 
     class StubNotFound < Error; end
     class InvalidStub < Error; end
+    class Warning < Error; end
 
     # Socket related errors
     class Socket < Error
@@ -45,9 +46,23 @@ or:
       end
     end
 
+    class InvalidHeaderKey < Error; end
+    class InvalidHeaderValue < Error; end
     class Timeout < Error; end
     class ResponseParse < Error; end
+
+    class ProxyConnectionError < Error
+      attr_reader :request, :response
+
+      def initialize(msg, request = nil, response = nil)
+        super(msg)
+        @request = request
+        @response = response
+      end
+    end
+
     class ProxyParse < Error; end
+    class TooManyRedirects < Error; end
 
     # Base class for HTTP Error classes
     class HTTPStatus < Error

data/lib/excon/headers.rb

@@ -22,6 +22,7 @@ module Excon
     alias_method :raw_values_at, :values_at
 
     def initialize
+      super
       @downcased = {}
     end
 
@@ -29,11 +30,11 @@ module Excon
       @downcased[key.to_s.downcase]
     end
 
-    alias_method :[]=, :store
     def []=(key, value)
       raw_writer(key, value)
       @downcased[key.to_s.downcase] = value
     end
+    alias_method :store, :[]=
 
     if SENTINEL.respond_to? :assoc
       def assoc(obj)
@@ -54,11 +55,11 @@ module Excon
       end
     end
 
-    alias_method :has_key?, :key?
-    alias_method :has_key?, :member?
     def has_key?(key)
       raw_key?(key) || @downcased.has_key?(key.to_s.downcase)
     end
+    alias_method :key?, :has_key?
+    alias_method :member?, :has_key?
 
     def merge(other_hash)
       self.dup.merge!(other_hash)

data/lib/excon/instrumentors/logging_instrumentor.rb

@@ -2,23 +2,12 @@ require 'logger'
 
 module Excon
   class LoggingInstrumentor
-    # Returns the Logger object for the LoggingInstrumentor. If one doesn't
-    # already exist, then one will be created using $stderr as the output
-    # stream.
-    #
-    def self.logger
-      @logger ||= Logger.new($stderr)
-    end
-
-    # Sets the logger object for the LoggingInstrumentor.
-    #
-    def self.logger=(logger)
-      @logger = logger
-    end
 
-    def self.instrument(name, params = {}, &block)
+    def self.instrument(name, params = {})
       params = params.dup
 
+      logger = params[:logger] || Logger.new($stderr)
+
       # reduce duplication/noise of output
       params.delete(:connection)
       params.delete(:stack)
@@ -39,7 +28,7 @@ module Excon
           info << "?"
 
           if params[:query].is_a?(Hash)
-            info << params.to_a.map{ |key,value| "#{key}=#{value}" }.join('&')
+            info << params[:query].to_a.map { |key,value| "#{key}=#{value}" }.join('&')
           else
             info << params[:query]
           end
@@ -51,7 +40,7 @@ module Excon
         end
       end
 
-      self.logger.log(logger.level, info) if info
+      logger.info(info) if info
 
       yield if block_given?
     end

data/lib/excon/instrumentors/standard_instrumentor.rb

@@ -1,21 +1,14 @@
 # frozen_string_literal: true
 module Excon
   class StandardInstrumentor
-    def self.instrument(name, params = {}, &block)
+    def self.instrument(name, params = {})
       params = params.dup
 
       # reduce duplication/noise of output
       params.delete(:connection)
       params.delete(:stack)
 
-      if params.has_key?(:headers) && params[:headers].has_key?('Authorization')
-        params[:headers] = params[:headers].dup
-        params[:headers]['Authorization'] = REDACTED
-      end
-
-      if params.has_key?(:password)
-        params[:password] = REDACTED
-      end
+      params = Utils.redact(params)
 
       $stderr.puts(name)
       Excon::PrettyPrinter.pp($stderr, params)

data/lib/excon/middlewares/base.rb

@@ -2,6 +2,12 @@
 module Excon
   module Middleware
     class Base
+      # Returns the list of parameters that this middleware uses that are valid
+      # as arguments to `Connection#request` or `Connection#new`.
+      def self.valid_parameter_keys
+        []
+      end
+
       def initialize(stack)
         @stack = stack
       end

data/lib/excon/middlewares/capture_cookies.rb

@@ -8,7 +8,7 @@ module Excon
       end
 
       def get_header(datum, header)
-        _, header_value = datum[:response][:headers].detect do |key, value|
+        _, header_value = datum[:response][:headers].detect do |key, _|
           key.casecmp(header) == 0
         end
         header_value

data/lib/excon/middlewares/decompress.rb

@@ -2,6 +2,10 @@
 module Excon
   module Middleware
     class Decompress < Excon::Middleware::Base
+
+      INFLATE_ZLIB_OR_GZIP  = 47 # Zlib::MAX_WBITS + 32
+      INFLATE_RAW           = -15 # Zlib::MAX_WBITS * -1
+
       def request_call(datum)
         unless datum.has_key?(:response_block)
           key = datum[:headers].keys.detect {|k| k.to_s.casecmp('Accept-Encoding') == 0 } || 'Accept-Encoding'
@@ -15,12 +19,15 @@ module Excon
       def response_call(datum)
         body = datum[:response][:body]
         unless datum.has_key?(:response_block) || body.nil? || body.empty?
-          if key = datum[:response][:headers].keys.detect {|k| k.casecmp('Content-Encoding') == 0 }
+          if (key = datum[:response][:headers].keys.detect {|k| k.casecmp('Content-Encoding') == 0 })
             encodings = Utils.split_header_value(datum[:response][:headers][key])
-            if encoding = encodings.last
+            if (encoding = encodings.last)
               if encoding.casecmp('deflate') == 0
-                # assume inflate omits header
-                datum[:response][:body] = Zlib::Inflate.new(-Zlib::MAX_WBITS).inflate(body)
+                datum[:response][:body] = begin
+                  Zlib::Inflate.new(INFLATE_ZLIB_OR_GZIP).inflate(body)
+                rescue Zlib::DataError # fallback to raw on error
+                  Zlib::Inflate.new(INFLATE_RAW).inflate(body)
+                end
                 encodings.pop
               elsif encoding.casecmp('gzip') == 0 || encoding.casecmp('x-gzip') == 0
                 datum[:response][:body] = Zlib::GzipReader.new(StringIO.new(body)).read

data/lib/excon/middlewares/expects.rb

@@ -2,11 +2,17 @@
 module Excon
   module Middleware
     class Expects < Excon::Middleware::Base
+      def self.valid_parameter_keys
+        [
+          :expects
+        ]
+      end
+
       def response_call(datum)
         if datum.has_key?(:expects) && ![*datum[:expects]].include?(datum[:response][:status])
           raise(
             Excon::Errors.status_error(
-              datum.reject {|key,value| key == :response},
+              datum.reject {|key,_| key == :response},
               Excon::Response.new(datum[:response])
             )
           )

data/lib/excon/middlewares/idempotent.rb

@@ -1,7 +1,24 @@
 # frozen_string_literal: true
+require 'set'
+
 module Excon
   module Middleware
     class Idempotent < Excon::Middleware::Base
+      def self.valid_parameter_keys
+        [
+          :idempotent,
+          :retries_remaining,
+          :retry_errors,
+          :retry_interval,
+          :retry_limit
+        ]
+      end
+
+      def request_call(datum)
+        datum[:retries_remaining] ||= datum[:retry_limit]
+        @stack.request_call(datum)
+      end
+
       def error_call(datum)
         if datum[:idempotent]
           if datum.has_key?(:request_block)
@@ -21,15 +38,15 @@ module Excon
           end
         end
 
-        if datum[:idempotent] && [Excon::Errors::Timeout, Excon::Errors::SocketError,
-            Excon::Errors::HTTPStatusError].any? {|ex| datum[:error].kind_of?(ex) } && datum[:retries_remaining] > 1
+        if datum[:idempotent] && datum[:retry_errors].any? {|ex| datum[:error].kind_of?(ex) } && datum[:retries_remaining] > 1
 
           sleep(datum[:retry_interval]) if datum[:retry_interval]
 
           # reduces remaining retries, reset connection, and restart request_call
           datum[:retries_remaining] -= 1
           connection = datum.delete(:connection)
-          datum.reject! {|key, _| !Excon::VALID_REQUEST_KEYS.include?(key) }
+          valid_keys = Set.new(connection.valid_request_keys(datum[:middlewares]))
+          datum.select! {|key, _| valid_keys.include?(key) }
           connection.request(datum)
         else
           @stack.error_call(datum)

data/lib/excon/middlewares/instrumentor.rb

@@ -2,6 +2,14 @@
 module Excon
   module Middleware
     class Instrumentor < Excon::Middleware::Base
+      def self.valid_parameter_keys
+        [
+          :logger,
+          :instrumentor,
+          :instrumentor_name
+        ]
+      end
+
       def error_call(datum)
         if datum.has_key?(:instrumentor)
           datum[:instrumentor].instrument("#{datum[:instrumentor_name]}.error", :error => datum[:error]) do

data/lib/excon/middlewares/mock.rb

@@ -2,6 +2,14 @@
 module Excon
   module Middleware
     class Mock < Excon::Middleware::Base
+      def self.valid_parameter_keys
+        [
+          :allow_unstubbed_requests,
+          :captures,
+          :mock
+        ]
+      end
+
       def request_call(datum)
         if datum[:mock]
           # convert File/Tempfile body to string before matching:
@@ -17,12 +25,13 @@ module Excon
             raise Excon::Errors::InvalidStub.new("Request body should be a string or an IO object. #{datum[:body].class} provided")
           end
 
-          if stub = Excon.stub_for(datum)
+          if (stub = Excon.stub_for(datum))
+            datum[:remote_ip] ||= '127.0.0.1'
             datum[:response] = {
               :body       => '',
               :headers    => {},
               :status     => 200,
-              :remote_ip  => '127.0.0.1'
+              :remote_ip  => datum[:remote_ip]
             }
 
             stub_datum = case stub.last
@@ -32,7 +41,7 @@ module Excon
               stub.last
             end
 
-            datum[:response].merge!(stub_datum.reject {|key,value| key == :headers})
+            datum[:response].merge!(stub_datum.reject {|key,_| key == :headers})
             if stub_datum.has_key?(:headers)
               datum[:response][:headers].merge!(stub_datum[:headers])
             end

data/lib/excon/middlewares/redirect_follower.rb

@@ -2,9 +2,21 @@
 module Excon
   module Middleware
     class RedirectFollower < Excon::Middleware::Base
+      def self.valid_parameter_keys
+        [
+          :redirects_remaining,
+          :redirect_limit
+        ]
+      end
+
+      def request_call(datum)
+        datum[:redirects_remaining] ||= datum[:redirect_limit] ||
+                                        Excon::DEFAULT_REDIRECT_LIMIT
+        @stack.request_call(datum)
+      end
 
       def get_header(datum, header)
-        _, header_value = datum[:response][:headers].detect do |key, value|
+        _, header_value = datum[:response][:headers].detect do |key, _|
           key.casecmp(header) == 0
         end
         header_value
@@ -14,14 +26,21 @@ module Excon
         if datum.has_key?(:response)
           case datum[:response][:status]
           when 301, 302, 303, 307, 308
+            if datum[:redirects_remaining] <= 0
+              raise Excon::Errors::TooManyRedirects
+            end
+
+            datum[:redirects_remaining] -= 1
+
             uri_parser = datum[:uri_parser] || Excon.defaults[:uri_parser]
 
             location = get_header(datum, 'Location')
             base_uri = Excon::Utils.request_uri(datum)
             uri = uri_parser.join(base_uri, location)
 
-            # delete old/redirect response
+            # delete old/redirect response and remote_ip
             response = datum.delete(:response)
+            datum.delete(:remote_ip)
 
             params = datum.dup
             params.delete(:connection)
@@ -52,7 +71,10 @@ module Excon
             params.merge!(:password => Utils.unescape_uri(uri.password)) if uri.password
 
             response = Excon::Connection.new(params).request
-            datum.merge!({:response => response.data})
+            datum.merge!({
+              :remote_ip  => response.remote_ip,
+              :response   => response.data
+            })
           else
             @stack.response_call(datum)
           end

data/lib/excon/middlewares/response_parser.rb

@@ -6,6 +6,9 @@ module Excon
         unless datum.has_key?(:response)
           datum = Excon::Response.parse(datum[:connection].send(:socket), datum)
         end
+        if datum.has_key?(:logger)
+          datum[:response][:logger] = datum[:logger]
+        end
         @stack.response_call(datum)
       end
     end

data/lib/excon/pretty_printer.rb

@@ -9,14 +9,7 @@ module Excon
         datum.delete(:connection)
         datum.delete(:stack)
 
-        if datum.has_key?(:headers) && datum[:headers].has_key?('Authorization')
-          datum[:headers] = datum[:headers].dup
-          datum[:headers]['Authorization'] = REDACTED
-        end
-
-        if datum.has_key?(:password)
-          datum[:password] = REDACTED
-        end
+        datum = Utils.redact(datum)
       end
 
       indent += 2

data/lib/excon/response.rb

@@ -59,10 +59,13 @@ module Excon
 
     def self.parse(socket, datum)
       # this will discard any trailing lines from the previous response if any.
-      begin
+      line = nil
+      loop do
         line = socket.readline
-      end until status = line[9, 3].to_i
+        break if line[9,3].to_i != 0
+      end
 
+      status = line[9, 3].to_i
       reason_phrase = line[13..-3] # -3 strips the trailing "\r\n"
 
       datum[:response] = {
@@ -90,7 +93,7 @@ module Excon
 
       unless (['HEAD', 'CONNECT'].include?(datum[:method].to_s.upcase)) || NO_ENTITY.include?(datum[:response][:status])
 
-        if key = datum[:response][:headers].keys.detect {|k| k.casecmp('Transfer-Encoding') == 0 }
+        if (key = datum[:response][:headers].keys.detect {|k| k.casecmp('Transfer-Encoding') == 0 })
           encodings = Utils.split_header_value(datum[:response][:headers][key])
           if (encoding = encodings.last) && encoding.casecmp('chunked') == 0
             transfer_encoding_chunked = true
@@ -103,7 +106,7 @@ module Excon
         end
 
         # use :response_block unless :expects would fail
-        if response_block = datum[:response_block]
+        if (response_block = datum[:response_block])
           if datum[:middlewares].include?(Excon::Middleware::Expects) && datum[:expects] &&
                                 !Array(datum[:expects]).include?(datum[:response][:status])
             response_block = nil
@@ -140,11 +143,11 @@ module Excon
           end
           parse_headers(socket, datum) # merge trailers into headers
         else
-          if key = datum[:response][:headers].keys.detect {|k| k.casecmp('Content-Length') == 0 }
+          if (key = datum[:response][:headers].keys.detect {|k| k.casecmp('Content-Length') == 0 })
             content_length = datum[:response][:headers][key].to_i
           end
 
-          if remaining = content_length
+          if (remaining = content_length)
             if response_block
               while remaining > 0
                 chunk = socket.read([datum[:chunk_size], remaining].min) || raise(EOFError)
@@ -160,11 +163,11 @@ module Excon
             end
           else
             if response_block
-              while chunk = socket.read(datum[:chunk_size])
+              while (chunk = socket.read(datum[:chunk_size]))
                 response_block.call(chunk, nil, nil)
               end
             else
-              while chunk = socket.read(datum[:chunk_size])
+              while (chunk = socket.read(datum[:chunk_size]))
                 datum[:response][:body] << chunk
               end
             end
@@ -222,7 +225,7 @@ module Excon
     end
 
     # Retrieve a specific header value. Header names are treated case-insensitively.
-    #   @param [String] name Header name
+    # @param [String] name Header name
     def get_header(name)
       headers[name]
     end