httparty 0.13.7 → 0.20.0

data/lib/httparty/connection_adapter.rb

@@ -1,10 +1,12 @@
+# frozen_string_literal: true
+
 module HTTParty
   # Default connection adapter that returns a new Net::HTTP each time
   #
   # == Custom Connection Factories
   #
   # If you like to implement your own connection adapter, subclassing
-  # HTTPParty::ConnectionAdapter will make it easier. Just override
+  # HTTParty::ConnectionAdapter will make it easier. Just override
   # the #connection method. The uri and options attributes will have
   # all the info you need to construct your http connection. Whatever
   # you return from your connection method needs to adhere to the
@@ -38,26 +40,52 @@ module HTTParty
   # in the #options attribute. It is up to you to interpret them within your
   # connection adapter. Take a look at the implementation of
   # HTTParty::ConnectionAdapter#connection for examples of how they are used.
-  # Some things that are probably interesting are as follows:
+  # The keys used in options are
   # * :+timeout+: timeout in seconds
   # * :+open_timeout+: http connection open_timeout in seconds, overrides timeout if set
   # * :+read_timeout+: http connection read_timeout in seconds, overrides timeout if set
+  # * :+write_timeout+: http connection write_timeout in seconds, overrides timeout if set (Ruby >= 2.6.0 required)
   # * :+debug_output+: see HTTParty::ClassMethods.debug_output.
-  # * :+pem+: contains pem data. see HTTParty::ClassMethods.pem.
+  # * :+cert_store+: contains certificate data. see method 'attach_ssl_certificates'
+  # * :+pem+: contains pem client certificate data. see method 'attach_ssl_certificates'
+  # * :+p12+: contains PKCS12 client client certificate data.  see method 'attach_ssl_certificates'
   # * :+verify+: verify the server’s certificate against the ca certificate.
   # * :+verify_peer+: set to false to turn off server verification but still send client certificate
   # * :+ssl_ca_file+: see HTTParty::ClassMethods.ssl_ca_file.
   # * :+ssl_ca_path+: see HTTParty::ClassMethods.ssl_ca_path.
+  # * :+ssl_version+: SSL versions to allow. see method 'attach_ssl_certificates'
+  # * :+ciphers+: The list of SSL ciphers to support
   # * :+connection_adapter_options+: contains the hash you passed to HTTParty.connection_adapter when you configured your connection adapter
+  # * :+local_host+: The local address to bind to
+  # * :+local_port+: The local port to bind to
+  # * :+http_proxyaddr+: HTTP Proxy address
+  # * :+http_proxyport+: HTTP Proxy port
+  # * :+http_proxyuser+: HTTP Proxy user
+  # * :+http_proxypass+: HTTP Proxy password
+  #
+  # === Inherited methods
+  # * :+clean_host+: Method used to sanitize host names
+
   class ConnectionAdapter
     # Private: Regex used to strip brackets from IPv6 URIs.
     StripIpv6BracketsRegex = /\A\[(.*)\]\z/
 
+    OPTION_DEFAULTS = {
+      verify: true,
+      verify_peer: true
+    }
+
     # Public
     def self.call(uri, options)
       new(uri, options).connection
     end
 
+    def self.default_cert_store
+      @default_cert_store ||= OpenSSL::X509::Store.new.tap do |cert_store|
+        cert_store.set_default_paths
+      end
+    end
+
     attr_reader :uri, :options
 
     def initialize(uri, options = {})
@@ -65,14 +93,21 @@ module HTTParty
       raise ArgumentError, "uri must be a #{uri_adapter}, not a #{uri.class}" unless uri.is_a? uri_adapter
 
       @uri = uri
-      @options = options
+      @options = OPTION_DEFAULTS.merge(options)
     end
 
     def connection
       host = clean_host(uri.host)
       port = uri.port || (uri.scheme == 'https' ? 443 : 80)
-      if options[:http_proxyaddr]
-        http = Net::HTTP.new(host, port, options[:http_proxyaddr], options[:http_proxyport], options[:http_proxyuser], options[:http_proxypass])
+      if options.key?(:http_proxyaddr)
+        http = Net::HTTP.new(
+          host,
+          port,
+          options[:http_proxyaddr],
+          options[:http_proxyport],
+          options[:http_proxyuser],
+          options[:http_proxypass]
+        )
       else
         http = Net::HTTP.new(host, port)
       end
@@ -81,19 +116,35 @@ module HTTParty
 
       attach_ssl_certificates(http, options)
 
-      if options[:timeout] && (options[:timeout].is_a?(Integer) || options[:timeout].is_a?(Float))
+      if add_timeout?(options[:timeout])
         http.open_timeout = options[:timeout]
         http.read_timeout = options[:timeout]
+
+        from_ruby_version('2.6.0', option: :write_timeout, warn: false) do
+          http.write_timeout = options[:timeout]
+        end
       end
 
-      if options[:read_timeout] && (options[:read_timeout].is_a?(Integer) || options[:read_timeout].is_a?(Float))
+      if add_timeout?(options[:read_timeout])
         http.read_timeout = options[:read_timeout]
       end
 
-      if options[:open_timeout] && (options[:open_timeout].is_a?(Integer) || options[:open_timeout].is_a?(Float))
+      if add_timeout?(options[:open_timeout])
         http.open_timeout = options[:open_timeout]
       end
 
+      if add_timeout?(options[:write_timeout])
+        from_ruby_version('2.6.0', option: :write_timeout) do
+          http.write_timeout = options[:write_timeout]
+        end
+      end
+
+      if add_max_retries?(options[:max_retries])
+        from_ruby_version('2.5.0', option: :max_retries) do
+          http.max_retries = options[:max_retries]
+        end
+      end
+
       if options[:debug_output]
         http.set_debug_output(options[:debug_output])
       end
@@ -106,18 +157,14 @@ module HTTParty
       #
       # @see https://bugs.ruby-lang.org/issues/6617
       if options[:local_host]
-        if RUBY_VERSION >= "2.0.0"
+        from_ruby_version('2.0.0', option: :local_host) do
           http.local_host = options[:local_host]
-        else
-          Kernel.warn("Warning: option :local_host requires Ruby version 2.0 or later")
         end
       end
 
       if options[:local_port]
-        if RUBY_VERSION >= "2.0.0"
+        from_ruby_version('2.0.0', option: :local_port) do
           http.local_port = options[:local_port]
-        else
-          Kernel.warn("Warning: option :local_port requires Ruby version 2.0 or later")
         end
       end
 
@@ -126,6 +173,22 @@ module HTTParty
 
     private
 
+    def from_ruby_version(ruby_version, option: nil, warn: true)
+      if RUBY_VERSION >= ruby_version
+        yield
+      elsif warn
+        Kernel.warn("Warning: option #{ option } requires Ruby version #{ ruby_version } or later")
+      end
+    end
+
+    def add_timeout?(timeout)
+      timeout && (timeout.is_a?(Integer) || timeout.is_a?(Float))
+    end
+
+    def add_max_retries?(max_retries)
+      max_retries && max_retries.is_a?(Integer) && max_retries >= 0
+    end
+
     def clean_host(host)
       strip_ipv6_brackets(host)
     end
@@ -138,6 +201,10 @@ module HTTParty
       uri.port == 443 || uri.scheme == 'https'
     end
 
+    def verify_ssl_certificate?
+      !(options[:verify] == false || options[:verify_peer] == false)
+    end
+
     def attach_ssl_certificates(http, options)
       if http.use_ssl?
         if options.fetch(:verify, true)
@@ -146,8 +213,7 @@ module HTTParty
             http.cert_store = options[:cert_store]
           else
             # Use the default cert store by default, i.e. system ca certs
-            http.cert_store = OpenSSL::X509::Store.new
-            http.cert_store.set_default_paths
+            http.cert_store = self.class.default_cert_store
           end
         else
           http.verify_mode = OpenSSL::SSL::VERIFY_NONE
@@ -157,8 +223,8 @@ module HTTParty
         # Note: options[:pem] must contain the content of a PEM file having the private key appended
         if options[:pem]
           http.cert = OpenSSL::X509::Certificate.new(options[:pem])
-          http.key = OpenSSL::PKey::RSA.new(options[:pem], options[:pem_password])
-          http.verify_mode = options[:verify_peer] == false ? OpenSSL::SSL::VERIFY_NONE : OpenSSL::SSL::VERIFY_PEER
+          http.key = OpenSSL::PKey.read(options[:pem], options[:pem_password])
+          http.verify_mode = verify_ssl_certificate? ? OpenSSL::SSL::VERIFY_PEER : OpenSSL::SSL::VERIFY_NONE
         end
 
         # PKCS12 client certificate authentication
@@ -166,7 +232,7 @@ module HTTParty
           p12 = OpenSSL::PKCS12.new(options[:p12], options[:p12_password])
           http.cert = p12.certificate
           http.key = p12.key
-          http.verify_mode = options[:verify_peer] == false ? OpenSSL::SSL::VERIFY_NONE : OpenSSL::SSL::VERIFY_PEER
+          http.verify_mode = verify_ssl_certificate? ? OpenSSL::SSL::VERIFY_PEER : OpenSSL::SSL::VERIFY_NONE
         end
 
         # SSL certificate authority file and/or directory

data/lib/httparty/cookie_hash.rb

@@ -1,14 +1,16 @@
+# frozen_string_literal: true
+
 class HTTParty::CookieHash < Hash #:nodoc:
-  CLIENT_COOKIES = %w(path expires domain path secure httponly)
+  CLIENT_COOKIES = %w(path expires domain path secure httponly samesite)
 
-  def add_cookies(value)
-    case value
+  def add_cookies(data)
+    case data
     when Hash
-      merge!(value)
+      merge!(data)
     when String
-      value.split('; ').each do |cookie|
-        array = cookie.split('=', 2)
-        self[array[0].to_sym] = array[1]
+      data.split('; ').each do |cookie|
+        key, value = cookie.split('=', 2)
+        self[key.to_sym] = value if key
       end
     else
       raise "add_cookies only takes a Hash or a String"
@@ -16,6 +18,6 @@ class HTTParty::CookieHash < Hash #:nodoc:
   end
 
   def to_cookie_string
-    delete_if { |k, v| CLIENT_COOKIES.include?(k.to_s.downcase) }.collect { |k, v| "#{k}=#{v}" }.join("; ")
+    select { |k, v| !CLIENT_COOKIES.include?(k.to_s.downcase) }.collect { |k, v| "#{k}=#{v}" }.join('; ')
   end
 end

data/lib/httparty/decompressor.rb

@@ -0,0 +1,92 @@
+# frozen_string_literal: true
+
+module HTTParty
+  # Decompresses the response body based on the Content-Encoding header.
+  #
+  # Net::HTTP automatically decompresses Content-Encoding values "gzip" and "deflate".
+  # This class will handle "br" (Brotli) and "compress" (LZW) if the requisite
+  # gems are installed. Otherwise, it returns nil if the body data cannot be
+  # decompressed.
+  #
+  # @abstract Read the HTTP Compression section for more information.
+  class Decompressor
+
+    # "gzip" and "deflate" are handled by Net::HTTP
+    # hence they do not need to be handled by HTTParty
+    SupportedEncodings = {
+      'none'     => :none,
+      'identity' => :none,
+      'br'       => :brotli,
+      'compress' => :lzw
+    }.freeze
+
+    # The response body of the request
+    # @return [String]
+    attr_reader :body
+
+    # The Content-Encoding algorithm used to encode the body
+    # @return [Symbol] e.g. :gzip
+    attr_reader :encoding
+
+    # @param [String] body - the response body of the request
+    # @param [Symbol] encoding - the Content-Encoding algorithm used to encode the body
+    def initialize(body, encoding)
+      @body = body
+      @encoding = encoding
+    end
+
+    # Perform decompression on the response body
+    # @return [String] the decompressed body
+    # @return [nil] when the response body is nil or cannot decompressed
+    def decompress
+      return nil if body.nil?
+      return body if encoding.nil? || encoding.strip.empty?
+
+      if supports_encoding?
+        decompress_supported_encoding
+      else
+        nil
+      end
+    end
+
+    protected
+
+    def supports_encoding?
+      SupportedEncodings.keys.include?(encoding)
+    end
+
+    def decompress_supported_encoding
+      method = SupportedEncodings[encoding]
+      if respond_to?(method, true)
+        send(method)
+      else
+        raise NotImplementedError, "#{self.class.name} has not implemented a decompression method for #{encoding.inspect} encoding."
+      end
+    end
+
+    def none
+      body
+    end
+
+    def brotli
+      return nil unless defined?(::Brotli)
+      begin
+        ::Brotli.inflate(body)
+      rescue StandardError
+        nil
+      end
+    end
+
+    def lzw
+      begin
+        if defined?(::LZWS::String)
+          ::LZWS::String.decompress(body)
+        elsif defined?(::LZW::Simple)
+          ::LZW::Simple.new.decompress(body)
+        end
+      rescue StandardError
+        nil
+      end
+    end
+  end
+end

data/lib/httparty/exceptions.rb

@@ -1,8 +1,10 @@
+# frozen_string_literal: true
+
 module HTTParty
-  # @abstact Exceptions raised by HTTParty inherit from Error
+  # @abstract Exceptions raised by HTTParty inherit from Error
   class Error < StandardError; end
 
-  # Exception raised when you attempt to set a non-existant format
+  # Exception raised when you attempt to set a non-existent format
   class UnsupportedFormat < Error; end
 
   # Exception raised when using a URI scheme other than HTTP or HTTPS
@@ -20,10 +22,14 @@ module HTTParty
     # @param [Net::HTTPResponse]
     def initialize(response)
       @response = response
+      super(response)
     end
   end
 
   # Exception that is raised when request has redirected too many times.
   # Calling {#response} returns the Net:HTTP response object.
   class RedirectionTooDeep < ResponseError; end
+
+  # Exception that is raised when request redirects and location header is present more than once
+  class DuplicateLocationHeader < ResponseError; end
 end

data/lib/httparty/hash_conversions.rb

@@ -1,3 +1,7 @@
+# frozen_string_literal: true
+
+require 'erb'
+
 module HTTParty
   module HashConversions
     # @return <String> This hash as a query string
@@ -22,28 +26,46 @@ module HTTParty
     #
     # @example normalize_param(:name, "Bob Jones") #=> "name=Bob%20Jones&"
     def self.normalize_param(key, value)
-      param = ''
+      normalized_keys = normalize_keys(key, value)
+
+      normalized_keys.flatten.each_slice(2).inject(''.dup) do |string, (k, v)|
+        string << "#{ERB::Util.url_encode(k)}=#{ERB::Util.url_encode(v.to_s)}&"
+      end
+    end
+
+    def self.normalize_keys(key, value)
       stack = []
+      normalized_keys = []
 
       if value.respond_to?(:to_ary)
-        param << value.to_ary.map { |element| normalize_param("#{key}[]", element) }.join
+        if value.empty?
+          normalized_keys << ["#{key}[]", '']
+        else
+          normalized_keys = value.to_ary.flat_map do |element|
+            normalize_keys("#{key}[]", element)
+          end
+        end
       elsif value.respond_to?(:to_hash)
         stack << [key, value.to_hash]
       else
-        param << "#{key}=#{ERB::Util.url_encode(value.to_s)}&"
+        normalized_keys << [key.to_s, value]
       end
 
       stack.each do |parent, hash|
-        hash.each do |k, v|
-          if v.respond_to?(:to_hash)
-            stack << ["#{parent}[#{k}]", v.to_hash]
+        hash.each do |child_key, child_value|
+          if child_value.respond_to?(:to_hash)
+            stack << ["#{parent}[#{child_key}]", child_value.to_hash]
+          elsif child_value.respond_to?(:to_ary)
+            child_value.to_ary.each do |v|
+              normalized_keys << normalize_keys("#{parent}[#{child_key}][]", v).flatten
+            end
           else
-            param << normalize_param("#{parent}[#{k}]", v)
+            normalized_keys << normalize_keys("#{parent}[#{child_key}]", child_value).flatten
           end
         end
       end
 
-      param
+      normalized_keys
     end
   end
 end

data/lib/httparty/headers_processor.rb

@@ -0,0 +1,32 @@
+# frozen_string_literal: true
+
+module HTTParty
+  class HeadersProcessor
+    attr_reader :headers, :options
+
+    def initialize(headers, options)
+      @headers = headers
+      @options = options
+    end
+
+    def call
+      return unless options[:headers]
+
+      options[:headers] = headers.merge(options[:headers]) if headers.any?
+      options[:headers] = Utils.stringify_keys(process_dynamic_headers)
+    end
+
+    private
+
+    def process_dynamic_headers
+      options[:headers].each_with_object({}) do |header, processed_headers|
+        key, value = header
+        processed_headers[key] = if value.respond_to?(:call)
+                                   value.arity == 0 ? value.call : value.call(options)
+                                 else
+                                   value
+                                 end
+      end
+    end
+  end
+end

data/lib/httparty/logger/apache_formatter.rb

@@ -1,9 +1,11 @@
+# frozen_string_literal: true
+
 module HTTParty
   module Logger
     class ApacheFormatter #:nodoc:
       TAG_NAME = HTTParty.name
 
-      attr_accessor :level, :logger, :current_time
+      attr_accessor :level, :logger
 
       def initialize(logger, level)
         @logger = logger
@@ -11,11 +13,34 @@ module HTTParty
       end
 
       def format(request, response)
-        current_time   = Time.now.strftime("%Y-%m-%d %H:%M:%S %z")
-        http_method    = request.http_method.name.split("::").last.upcase
-        path           = request.path.to_s
-        content_length = response.respond_to?(:headers) ? response.headers['Content-Length'] : response['Content-Length']
-        @logger.send @level, "[#{TAG_NAME}] [#{current_time}] #{response.code} \"#{http_method} #{path}\" #{content_length || '-'} "
+        @request = request
+        @response = response
+
+        logger.public_send level, message
+      end
+
+      private
+
+      attr_reader :request, :response
+
+      def message
+        "[#{TAG_NAME}] [#{current_time}] #{response.code} \"#{http_method} #{path}\" #{content_length || '-'} "
+      end
+
+      def current_time
+        Time.now.strftime('%Y-%m-%d %H:%M:%S %z')
+      end
+
+      def http_method
+        request.http_method.name.split('::').last.upcase
+      end
+
+      def path
+        request.path.to_s
+      end
+
+      def content_length
+        response.respond_to?(:headers) ? response.headers['Content-Length'] : response['Content-Length']
       end
     end
   end

data/lib/httparty/logger/curl_formatter.rb

@@ -1,47 +1,92 @@
+# frozen_string_literal: true
+
 module HTTParty
   module Logger
     class CurlFormatter #:nodoc:
       TAG_NAME = HTTParty.name
-      OUT = ">"
-      IN = "<"
+      OUT      = '>'
+      IN       = '<'
 
-      attr_accessor :level, :logger, :current_time
+      attr_accessor :level, :logger
 
       def initialize(logger, level)
-        @logger = logger
-        @level  = level.to_sym
+        @logger   = logger
+        @level    = level.to_sym
+        @messages = []
       end
 
       def format(request, response)
-        messages        = []
-        time            = Time.now.strftime("%Y-%m-%d %H:%M:%S %z")
-        http_method     = request.http_method.name.split("::").last.upcase
-        path            = request.path.to_s
+        @request  = request
+        @response = response
 
-        messages << print(time, OUT, "#{http_method} #{path}")
+        log_request
+        log_response
 
-        if request.options[:headers] && request.options[:headers].size > 0
-          request.options[:headers].each do |k, v|
-            messages << print(time, OUT, "#{k}: #{v}")
-          end
-        end
+        logger.public_send level, messages.join('\n')
+      end
+
+      private
+
+      attr_reader :request, :response
+      attr_accessor :messages
 
-        messages << print(time, OUT, request.raw_body)
-        messages << print(time, OUT, "")
-        messages << print(time, IN, "HTTP/#{response.http_version} #{response.code}")
+      def log_request
+        log_url
+        log_headers
+        log_query
+        log OUT, request.raw_body if request.raw_body
+        log OUT
+      end
+
+      def log_response
+        log IN, "HTTP/#{response.http_version} #{response.code}"
+        log_response_headers
+        log IN, "\n#{response.body}"
+        log IN
+      end
 
+      def log_url
+        http_method = request.http_method.name.split('::').last.upcase
+        uri = if request.options[:base_uri]
+                request.options[:base_uri] + request.path.path
+              else
+                request.path.to_s
+              end
+
+        log OUT, "#{http_method} #{uri}"
+      end
+
+      def log_headers
+        return unless request.options[:headers] && request.options[:headers].size > 0
+
+        log OUT, 'Headers: '
+        log_hash request.options[:headers]
+      end
+
+      def log_query
+        return unless request.options[:query]
+
+        log OUT, 'Query: '
+        log_hash request.options[:query]
+      end
+
+      def log_response_headers
         headers = response.respond_to?(:headers) ? response.headers : response
         response.each_header do |response_header|
-          messages << print(time, IN, "#{response_header.capitalize}: #{headers[response_header]}")
+          log IN, "#{response_header.capitalize}: #{headers[response_header]}"
         end
+      end
 
-        messages << print(time, IN, "\n#{response.body}")
+      def log_hash(hash)
+        hash.each { |k, v| log(OUT, "#{k}: #{v}") }
+      end
 
-        @logger.send @level, messages.join("\n")
+      def log(direction, line = '')
+        messages << "[#{TAG_NAME}] [#{current_time}] #{direction} #{line}"
       end
 
-      def print(time, direction, line)
-        "[#{TAG_NAME}] [#{time}] #{direction} #{line}"
+      def current_time
+        Time.now.strftime("%Y-%m-%d %H:%M:%S %z")
       end
     end
   end

data/lib/httparty/logger/logger.rb

@@ -1,12 +1,16 @@
+# frozen_string_literal: true
+
 require 'httparty/logger/apache_formatter'
 require 'httparty/logger/curl_formatter'
+require 'httparty/logger/logstash_formatter'
 
 module HTTParty
   module Logger
     def self.formatters
       @formatters ||= {
         :curl => Logger::CurlFormatter,
-        :apache => Logger::ApacheFormatter
+        :apache => Logger::ApacheFormatter,
+        :logstash => Logger::LogstashFormatter,
       }
     end