excon 0.62.0 → 0.63.0

data/excon.gemspec

@@ -26,4 +26,13 @@ Gem::Specification.new do |s|
   s.add_development_dependency('sinatra-contrib')
   s.add_development_dependency('json', '>= 1.8.5')
   s.add_development_dependency('puma')
+
+  s.metadata = {
+    'homepage_uri'      => 'https://github.com/excon/excon',
+    'bug_tracker_uri'   => 'https://github.com/excon/excon/issues',
+    'changelog_uri'     => 'https://github.com/excon/excon/blob/master/changelog.txt',
+    'documentation_uri' => 'https://github.com/excon/excon/blob/master/README.md',
+    'source_code_uri'   => 'https://github.com/excon/excon',
+    'wiki_uri'          => 'https://github.com/excon/excon/wiki'
+  }
 end

data/lib/excon.rb

@@ -23,11 +23,11 @@ require 'excon/middlewares/instrumentor'
 require 'excon/middlewares/mock'
 require 'excon/middlewares/response_parser'
 
+require 'excon/error'
 require 'excon/constants'
 require 'excon/utils'
 
 require 'excon/connection'
-require 'excon/error'
 require 'excon/headers'
 require 'excon/response'
 require 'excon/middlewares/decompress'
@@ -61,6 +61,14 @@ module Excon
       if $VERBOSE || ENV['EXCON_DEBUG']
         $stderr.puts "[excon][WARNING] #{warning}\n#{ caller.join("\n") }"
       end
+
+      if @raise_on_warnings
+        raise Error::Warning.new(warning)
+      end
+    end
+
+    def set_raise_on_warnings!(should_raise)
+      @raise_on_warnings = should_raise
     end
 
     # Status of mocking

data/lib/excon/connection.rb

@@ -38,8 +38,8 @@ module Excon
       end
     end
     def logger=(logger)
-      Excon::LoggingInstrumentor.logger = logger
       @data[:instrumentor] = Excon::LoggingInstrumentor
+      @data[:logger] = logger
     end
 
     # Initializes a new Connection instance
@@ -67,8 +67,13 @@ module Excon
       # the same goes for :middlewares
       @data[:middlewares] = @data[:middlewares].dup
 
-      params = validate_params(:connection, params)
       @data.merge!(params)
+      validate_params(:connection, @data, @data[:middlewares])
+
+      if @data.key?(:host) && !@data.key?(:hostname)
+        Excon.display_warning('hostname is missing! For IPv6 support, provide both host and hostname: Excon::Connection#new(:host => uri.host, :hostname => uri.hostname, ...).')
+        @data[:hostname] = @data[:host]
+      end
 
       setup_proxy
 
@@ -138,7 +143,13 @@ module Excon
 
           # add headers to request
           datum[:headers].each do |key, values|
+            if key.to_s.match(/[\r\n]/)
+              raise Excon::Errors::InvalidHeaderKey.new(key.to_s.inspect + ' contains forbidden "\r" or "\n"')
+            end
             [values].flatten.each do |value|
+              if value.to_s.match(/[\r\n]/)
+                raise Excon::Errors::InvalidHeaderValue.new(value.to_s.inspect + ' contains forbidden "\r" or "\n"')
+              end
               request << key.to_s << ': ' << value.to_s << CR_NL
             end
           end
@@ -150,9 +161,7 @@ module Excon
             socket.write(request) # write out request + headers
             while true # write out body with chunked encoding
               chunk = datum[:request_block].call
-              if FORCE_ENC
-                chunk.force_encoding('BINARY')
-              end
+              binary_encode(chunk)
               if chunk.length > 0
                 socket.write(chunk.length.to_s(16) << CR_NL << chunk << CR_NL)
               else
@@ -171,14 +180,10 @@ module Excon
             end
 
             # if request + headers is less than chunk size, fill with body
-            if FORCE_ENC
-              request.force_encoding('BINARY')
-            end
+            binary_encode(request)
             chunk = body.read([datum[:chunk_size] - request.length, 0].max)
             if chunk
-              if FORCE_ENC
-                chunk.force_encoding('BINARY')
-              end
+              binary_encode(chunk)
               socket.write(request << chunk)
             else
               socket.write(request) # write out request + headers
@@ -191,7 +196,7 @@ module Excon
         end
       rescue => error
         case error
-        when Excon::Errors::StubNotFound, Excon::Errors::Timeout
+        when Excon::Errors::InvalidHeaderKey, Excon::Errors::InvalidHeaderValue, Excon::Errors::StubNotFound, Excon::Errors::Timeout
           raise(error)
         else
           raise_socket_error(error)
@@ -223,11 +228,17 @@ module Excon
     #     @option params [String] :path appears after 'scheme://host:port/'
     #     @option params [Hash]   :query appended to the 'scheme://host:port/path/' in the form of '?key=value'
     def request(params={}, &block)
-      params = validate_params(:request, params)
       # @data has defaults, merge in new params to override
       datum = @data.merge(params)
       datum[:headers] = @data[:headers].merge(datum[:headers] || {})
 
+      validate_params(:request, params, datum[:middlewares])
+      # If the user passed in new middleware, we want to validate that the original connection parameters
+      # are still valid with the provided middleware.
+      if params[:middlewares]
+        validate_params(:connection, @data, datum[:middlewares])
+      end
+
       if datum[:user] || datum[:password]
         user, pass = Utils.unescape_uri(datum[:user].to_s), Utils.unescape_uri(datum[:password].to_s)
         datum[:headers]['Authorization'] ||= 'Basic ' + ["#{user}:#{pass}"].pack('m').delete(Excon::CR_NL)
@@ -238,7 +249,6 @@ module Excon
       else
         datum[:headers]['Host']   ||= datum[:host] + port_string(datum)
       end
-      datum[:retries_remaining] ||= datum[:retry_limit]
 
       # if path is empty or doesn't start with '/', insert one
       unless datum[:path][0, 1] == '/'
@@ -278,6 +288,10 @@ module Excon
       end
     rescue => error
       reset
+
+      # If we didn't get far enough to initialize datum and the middleware stack, just raise
+      raise error if !datum
+
       datum[:error] = error
       if datum[:stack]
         datum[:stack].error_call(datum)
@@ -355,15 +369,7 @@ module Excon
       vars = instance_variables.inject({}) do |accum, var|
         accum.merge!(var.to_sym => instance_variable_get(var))
       end
-      if vars[:'@data'][:headers].has_key?('Authorization')
-        vars[:'@data'] = vars[:'@data'].dup
-        vars[:'@data'][:headers] = vars[:'@data'][:headers].dup
-        vars[:'@data'][:headers]['Authorization'] = REDACTED
-      end
-      if vars[:'@data'][:password]
-        vars[:'@data'] = vars[:'@data'].dup
-        vars[:'@data'][:password] = REDACTED
-      end
+      vars[:'@data'] = Utils.redact(vars[:'@data'])
       inspection = '#<Excon::Connection:'
       inspection += (object_id << 1).to_s(16)
       vars.each do |key, value|
@@ -373,6 +379,10 @@ module Excon
       inspection
     end
 
+    def valid_request_keys(middlewares)
+      valid_middleware_keys(middlewares) + Excon::VALID_REQUEST_KEYS
+    end
+
     private
 
     def detect_content_length(body)
@@ -387,27 +397,34 @@ module Excon
       end
     end
 
-    def validate_params(validation, params)
+    def valid_middleware_keys(middlewares)
+      middlewares.flat_map(&:valid_parameter_keys)
+    end
+
+    def validate_params(validation, params, middlewares)
       valid_keys = case validation
       when :connection
-        Excon::VALID_CONNECTION_KEYS
+        valid_middleware_keys(middlewares) + Excon::VALID_CONNECTION_KEYS
       when :request
-        Excon::VALID_REQUEST_KEYS
+        valid_request_keys(middlewares)
+      else
+        raise ArgumentError.new("Invalid validation type '#{validation}'")
       end
+
       invalid_keys = params.keys - valid_keys
       unless invalid_keys.empty?
         Excon.display_warning("Invalid Excon #{validation} keys: #{invalid_keys.map(&:inspect).join(', ')}")
-        # FIXME: for now, just warn, don't mutate, give things (ie fog) a chance to catch up
-        #params = params.dup
-        #invalid_keys.each {|key| params.delete(key) }
-      end
 
-      if validation == :connection && params.key?(:host) && !params.key?(:hostname)
-        Excon.display_warning('hostname is missing! For IPv6 support, provide both host and hostname: Excon::Connection#new(:host => uri.host, :hostname => uri.hostname, ...).')
-        params[:hostname] = params[:host]
+        if validation == :request
+          deprecated_keys = invalid_keys & Excon::DEPRECATED_VALID_REQUEST_KEYS.keys
+          mw_msg = deprecated_keys.map do |k|
+            "#{k}: #{Excon::DEPRECATED_VALID_REQUEST_KEYS[k]}"
+          end.join(', ')
+          Excon.display_warning(
+            "The following request keys are only valid with the associated middleware: #{mw_msg}"
+          )
+        end
       end
-
-      params
     end
 
     def response(datum={})

data/lib/excon/constants.rb

@@ -14,6 +14,12 @@ module Excon
 
   DEFAULT_RETRY_LIMIT = 4
 
+  DEFAULT_RETRY_ERRORS = [
+    Excon::Error::Timeout,
+    Excon::Error::Socket,
+    Excon::Error::HTTPStatus
+  ]
+
   FORCE_ENC = CR_NL.respond_to?(:force_encoding)
 
   HTTP_1_1 = " HTTP/1.1\r\n"
@@ -33,19 +39,17 @@ module Excon
   VERSIONS = "#{USER_AGENT} (#{RUBY_PLATFORM}) ruby/#{RUBY_VERSION}"
 
   VALID_REQUEST_KEYS = [
+    :allow_unstubbed_requests,
     :body,
-    :captures,
     :chunk_size,
     :debug_request,
     :debug_response,
-    :expects,
     :headers,
-    :idempotent,
-    :instrumentor,
-    :instrumentor_name,
+    :instrumentor, # Used for setting logging within Connection
+    :logger,
     :method,
     :middlewares,
-    :mock,
+    :password,
     :path,
     :persistent,
     :pipeline,
@@ -53,9 +57,8 @@ module Excon
     :read_timeout,
     :request_block,
     :response_block,
-    :retries_remaining, # used internally
-    :retry_limit,
-    :retry_interval,
+    :stubs,
+    :user,
     :versions,
     :write_timeout
   ]
@@ -74,12 +77,12 @@ module Excon
     :private_key_path,
     :connect_timeout,
     :family,
+    :keepalive,
     :host,
     :hostname,
     :omit_default_port,
     :nonblock,
     :reuseaddr,
-    :password,
     :port,
     :proxy,
     :scheme,
@@ -91,12 +94,26 @@ module Excon
     :ssl_verify_peer,
     :ssl_verify_peer_host,
     :ssl_version,
+    :ssl_min_version,
+    :ssl_max_version,
+    :ssl_uri_schemes,
     :tcp_nodelay,
     :thread_safe_sockets,
     :uri_parser,
-    :user
   ]
 
+  DEPRECATED_VALID_REQUEST_KEYS = {
+    :captures => 'Mock',
+    :expects => 'Expects',
+    :idempotent => 'Idempotent',
+    :instrumentor_name => 'Instrumentor',
+    :mock => 'Mock',
+    :retries_remaining => 'Idempotent', # referenced in Instrumentor, but only relevant with Idempotent
+    :retry_errors => 'Idempotent',
+    :retry_interval => 'Idempotent',
+    :retry_limit => 'Idempotent' # referenced in Instrumentor, but only relevant with Idempotent
+  }
+
   unless ::IO.const_defined?(:WaitReadable)
     class ::IO
       module WaitReadable; end
@@ -112,12 +129,14 @@ module Excon
   DEFAULTS = {
     :chunk_size           => CHUNK_SIZE || DEFAULT_CHUNK_SIZE,
     # see https://wiki.mozilla.org/Security/Server_Side_TLS#Intermediate_compatibility_.28default.29
-    :ciphers              => 'ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA256:ECDHE-ECDSA-AES128-SHA:ECDHE-RSA-AES256-SHA384:ECDHE-RSA-AES128-SHA:ECDHE-ECDSA-AES256-SHA384:ECDHE-ECDSA-AES256-SHA:ECDHE-RSA-AES256-SHA:DHE-RSA-AES128-SHA256:DHE-RSA-AES128-SHA:DHE-RSA-AES256-SHA256:DHE-RSA-AES256-SHA:ECDHE-ECDSA-DES-CBC3-SHA:ECDHE-RSA-DES-CBC3-SHA:EDH-RSA-DES-CBC3-SHA:AES128-GCM-SHA256:AES256-GCM-SHA384:AES128-SHA256:AES256-SHA256:AES128-SHA:AES256-SHA:DES-CBC3-SHA:!DSS',
+    # list provided then had DES related things sorted to the end
+    :ciphers              => 'ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:ECDHE-RSA-AES256-GCM-SHA384:DHE-RSA-AES128-GCM-SHA256:DHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA256:ECDHE-ECDSA-AES128-SHA:ECDHE-RSA-AES256-SHA384:ECDHE-RSA-AES128-SHA:ECDHE-ECDSA-AES256-SHA384:ECDHE-ECDSA-AES256-SHA:ECDHE-RSA-AES256-SHA:DHE-RSA-AES128-SHA256:DHE-RSA-AES128-SHA:DHE-RSA-AES256-SHA256:DHE-RSA-AES256-SHA:AES128-GCM-SHA256:AES256-GCM-SHA384:AES128-SHA256:AES256-SHA256:AES128-SHA:AES256-SHA:ECDHE-ECDSA-DES-CBC3-SHA:ECDHE-RSA-DES-CBC3-SHA:EDH-RSA-DES-CBC3-SHA:DES-CBC3-SHA:!DSS',
     :connect_timeout      => 60,
     :debug_request        => false,
     :debug_response       => false,
     :headers              => {
-      'User-Agent' => USER_AGENT
+      'User-Agent' => USER_AGENT,
+      'Accept'     =>  '*/*'
     },
     :idempotent           => false,
     :instrumentor_name    => 'excon',
@@ -133,6 +152,7 @@ module Excon
     :omit_default_port    => false,
     :persistent           => false,
     :read_timeout         => 60,
+    :retry_errors         => DEFAULT_RETRY_ERRORS,
     :retry_limit          => DEFAULT_RETRY_LIMIT,
     :ssl_verify_peer      => true,
     :ssl_uri_schemes      => [HTTPS],

data/lib/excon/error.rb

@@ -6,6 +6,7 @@ module Excon
 
     class StubNotFound < Error; end
     class InvalidStub < Error; end
+    class Warning < Error; end
 
     # Socket related errors
     class Socket < Error
@@ -45,6 +46,8 @@ or:
       end
     end
 
+    class InvalidHeaderKey < Error; end
+    class InvalidHeaderValue < Error; end
     class Timeout < Error; end
     class ResponseParse < Error; end
     class ProxyParse < Error; end

data/lib/excon/instrumentors/logging_instrumentor.rb

@@ -2,23 +2,12 @@ require 'logger'
 
 module Excon
   class LoggingInstrumentor
-    # Returns the Logger object for the LoggingInstrumentor. If one doesn't
-    # already exist, then one will be created using $stderr as the output
-    # stream.
-    #
-    def self.logger
-      @logger ||= Logger.new($stderr)
-    end
-
-    # Sets the logger object for the LoggingInstrumentor.
-    #
-    def self.logger=(logger)
-      @logger = logger
-    end
 
     def self.instrument(name, params = {}, &block)
       params = params.dup
 
+      logger = params[:logger] || Logger.new($stderr)
+
       # reduce duplication/noise of output
       params.delete(:connection)
       params.delete(:stack)
@@ -51,7 +40,7 @@ module Excon
         end
       end
 
-      self.logger.log(logger.level, info) if info
+      logger.log(Logger::INFO, info) if info
 
       yield if block_given?
     end

data/lib/excon/instrumentors/standard_instrumentor.rb

@@ -8,14 +8,7 @@ module Excon
       params.delete(:connection)
       params.delete(:stack)
 
-      if params.has_key?(:headers) && params[:headers].has_key?('Authorization')
-        params[:headers] = params[:headers].dup
-        params[:headers]['Authorization'] = REDACTED
-      end
-
-      if params.has_key?(:password)
-        params[:password] = REDACTED
-      end
+      params = Utils.redact(params)
 
       $stderr.puts(name)
       Excon::PrettyPrinter.pp($stderr, params)

data/lib/excon/middlewares/base.rb

@@ -2,6 +2,12 @@
 module Excon
   module Middleware
     class Base
+      # Returns the list of parameters that this middleware uses that are valid
+      # as arguments to `Connection#request` or `Connection#new`.
+      def self.valid_parameter_keys
+        []
+      end
+
       def initialize(stack)
         @stack = stack
       end

data/lib/excon/middlewares/expects.rb

@@ -2,6 +2,12 @@
 module Excon
   module Middleware
     class Expects < Excon::Middleware::Base
+      def self.valid_parameter_keys
+        [
+          :expects
+        ]
+      end
+
       def response_call(datum)
         if datum.has_key?(:expects) && ![*datum[:expects]].include?(datum[:response][:status])
           raise(

data/lib/excon/middlewares/idempotent.rb

@@ -1,7 +1,24 @@
 # frozen_string_literal: true
+require 'set'
+
 module Excon
   module Middleware
     class Idempotent < Excon::Middleware::Base
+      def self.valid_parameter_keys
+        [
+          :idempotent,
+          :retries_remaining,
+          :retry_errors,
+          :retry_interval,
+          :retry_limit
+        ]
+      end
+
+      def request_call(datum)
+        datum[:retries_remaining] ||= datum[:retry_limit]
+        @stack.request_call(datum)
+      end
+
       def error_call(datum)
         if datum[:idempotent]
           if datum.has_key?(:request_block)
@@ -21,15 +38,15 @@ module Excon
           end
         end
 
-        if datum[:idempotent] && [Excon::Errors::Timeout, Excon::Errors::SocketError,
-            Excon::Errors::HTTPStatusError].any? {|ex| datum[:error].kind_of?(ex) } && datum[:retries_remaining] > 1
+        if datum[:idempotent] && datum[:retry_errors].any? {|ex| datum[:error].kind_of?(ex) } && datum[:retries_remaining] > 1
 
           sleep(datum[:retry_interval]) if datum[:retry_interval]
 
           # reduces remaining retries, reset connection, and restart request_call
           datum[:retries_remaining] -= 1
           connection = datum.delete(:connection)
-          datum.reject! {|key, _| !Excon::VALID_REQUEST_KEYS.include?(key) }
+          valid_keys = Set.new(connection.valid_request_keys(datum[:middlewares]))
+          datum.select! {|key, _| valid_keys.include?(key) }
           connection.request(datum)
         else
           @stack.error_call(datum)