httparty 0.13.7 → 0.24.2

data/examples/party_foul_mode.rb

@@ -0,0 +1,90 @@
+require 'httparty'
+
+class APIClient
+  include HTTParty
+  base_uri 'api.example.com'
+
+  def self.fetch_user(id)
+    begin
+      get("/users/#{id}", foul: true)
+    rescue HTTParty::NetworkError => e
+      handle_network_error(e)
+    rescue HTTParty::ResponseError => e
+      handle_api_error(e)
+    end
+  end
+
+  private
+
+  def self.handle_network_error(error)
+    case error.cause
+    when Errno::ECONNREFUSED
+      {
+        error: :server_down,
+        message: "The API server appears to be down",
+        details: error.message
+      }
+    when Net::OpenTimeout, Timeout::Error
+      {
+        error: :timeout,
+        message: "The request timed out",
+        details: error.message
+      }
+    when SocketError
+      {
+        error: :network_error,
+        message: "Could not connect to the API server",
+        details: error.message
+      }
+    when OpenSSL::SSL::SSLError
+      {
+        error: :ssl_error,
+        message: "SSL certificate verification failed",
+        details: error.message
+      }
+    else
+      {
+        error: :unknown_network_error,
+        message: "An unexpected network error occurred",
+        details: error.message
+      }
+    end
+  end
+
+  def self.handle_api_error(error)
+    {
+      error: :api_error,
+      message: "API returned error #{error.response.code}",
+      details: error.response.body
+    }
+  end
+end
+
+# Example usage:
+
+# 1. When server is down
+result = APIClient.fetch_user(123)
+puts "Server down example:"
+puts result.inspect
+puts
+
+# 2. When request times out
+result = APIClient.fetch_user(456)
+puts "Timeout example:"
+puts result.inspect
+puts
+
+# 3. When SSL error occurs
+result = APIClient.fetch_user(789)
+puts "SSL error example:"
+puts result.inspect
+puts
+
+# 4. Simple example without a wrapper class
+begin
+  HTTParty.get('https://api.example.com/users', foul: true)
+rescue HTTParty::Foul => e
+  puts "Direct usage example:"
+  puts "Error type: #{e.cause.class}"
+  puts "Error message: #{e.message}"
+end

data/examples/peer_cert.rb

@@ -0,0 +1,9 @@
+dir = File.expand_path(File.join(File.dirname(__FILE__), '..', 'lib'))
+require File.join(dir, 'httparty')
+
+peer_cert = nil
+HTTParty.get("https://www.example.com") do |fragment|
+  peer_cert ||= fragment.connection.peer_cert
+end
+
+puts "The server's certificate expires #{peer_cert.not_after}"

data/examples/stackexchange.rb

@@ -7,7 +7,7 @@ class StackExchange
   base_uri 'api.stackexchange.com'
 
   def initialize(service, page)
-    @options = { query: {site: service, page: page} }
+    @options = { query: { site: service, page: page } }
   end
 
   def questions

data/examples/stream_download.rb

@@ -0,0 +1,26 @@
+dir = File.expand_path(File.join(File.dirname(__FILE__), '..', 'lib'))
+require File.join(dir, 'httparty')
+require 'pp'
+
+# download file linux-4.6.4.tar.xz without using the memory
+response = nil
+filename = "linux-4.6.4.tar.xz"
+url = "https://cdn.kernel.org/pub/linux/kernel/v4.x/#{filename}"
+
+File.open(filename, "w") do |file|
+  response = HTTParty.get(url, stream_body: true) do |fragment|
+    if [301, 302].include?(fragment.code)
+      print "skip writing for redirect"
+    elsif fragment.code == 200
+      print "."
+      file.write(fragment)
+    else
+      raise StandardError, "Non-success status code while streaming #{fragment.code}"
+    end
+  end
+end
+puts
+
+pp "Success: #{response.success?}"
+pp File.stat(filename).inspect
+File.unlink(filename)

data/examples/tripit_sign_in.rb

@@ -3,30 +3,41 @@ require File.join(dir, 'httparty')
 
 class TripIt
   include HTTParty
-  base_uri 'http://www.tripit.com'
+  base_uri 'https://www.tripit.com'
   debug_output
 
   def initialize(email, password)
     @email = email
-    response = self.class.get('/account/login')
-    response = self.class.post(
+    get_response = self.class.get('/account/login')
+    get_response_cookie = parse_cookie(get_response.headers['Set-Cookie'])
+
+    post_response = self.class.post(
       '/account/login',
       body: {
         login_email_address: email,
         login_password: password
       },
-      headers: {'Cookie' => response.headers['Set-Cookie']}
+      headers: {'Cookie' => get_response_cookie.to_cookie_string }
     )
-    @cookie = response.request.options[:headers]['Cookie']
+
+    @cookie = parse_cookie(post_response.headers['Set-Cookie'])
   end
 
   def account_settings
-    self.class.get('/account/edit', headers: {'Cookie' => @cookie})
+    self.class.get('/account/edit', headers: { 'Cookie' => @cookie.to_cookie_string })
   end
 
   def logged_in?
     account_settings.include? "You're logged in as #{@email}"
   end
+
+  private
+
+  def parse_cookie(resp)
+    cookie_hash = CookieHash.new
+    resp.get_fields('Set-Cookie').each { |c| cookie_hash.add_cookies(c) }
+    cookie_hash
+  end
 end
 
 tripit = TripIt.new('email', 'password')

data/examples/twitter.rb

@@ -14,12 +14,12 @@ class Twitter
   # which can be :friends, :user or :public
   # options[:query] can be things like since, since_id, count, etc.
   def timeline(which = :friends, options = {})
-    options.merge!({basic_auth: @auth})
+    options.merge!({ basic_auth: @auth })
     self.class.get("/statuses/#{which}_timeline.json", options)
   end
 
   def post(text)
-    options = { query: {status: text}, basic_auth: @auth }
+    options = { query: { status: text }, basic_auth: @auth }
     self.class.post('/statuses/update.json', options)
   end
 end

data/examples/whoismyrep.rb

@@ -7,4 +7,4 @@ class Rep
 end
 
 pp Rep.get('http://whoismyrepresentative.com/getall_mems.php?zip=46544')
-pp Rep.get('http://whoismyrepresentative.com/getall_mems.php', query: {zip: 46544})
+pp Rep.get('http://whoismyrepresentative.com/getall_mems.php', query: { zip: 46544 })

data/httparty.gemspec

@@ -9,20 +9,24 @@ Gem::Specification.new do |s|
   s.licenses    = ['MIT']
   s.authors     = ["John Nunemaker", "Sandro Turriate"]
   s.email       = ["nunemaker@gmail.com"]
-  s.homepage    = "http://jnunemaker.github.com/httparty"
+  s.homepage    = "https://github.com/jnunemaker/httparty"
   s.summary     = 'Makes http fun! Also, makes consuming restful web services dead easy.'
   s.description = 'Makes http fun! Also, makes consuming restful web services dead easy.'
+  s.metadata["changelog_uri"] = 'https://github.com/jnunemaker/httparty/releases'
 
-  s.required_ruby_version     = '>= 1.9.3'
+  s.required_ruby_version     = '>= 2.7.0'
 
-  s.add_dependency 'json',      "~> 1.8"
+  s.add_dependency 'csv'
   s.add_dependency 'multi_xml', ">= 0.5.2"
+  s.add_dependency 'mini_mime', ">= 1.0.0"
 
   # If this line is removed, all hard partying will cease.
   s.post_install_message = "When you HTTParty, you must party hard!"
 
-  s.files         = `git ls-files`.split("\n")
-  s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
+  all_files = `git ls-files`.split("\n")
+  test_files = `git ls-files -- {test,spec,features}/*`.split("\n")
+
+  s.files         = all_files - test_files
   s.executables   = `git ls-files -- bin/*`.split("\n").map { |f| File.basename(f) }
   s.require_paths = ["lib"]
 end

data/lib/httparty/connection_adapter.rb

@@ -1,10 +1,12 @@
+# frozen_string_literal: true
+
 module HTTParty
   # Default connection adapter that returns a new Net::HTTP each time
   #
   # == Custom Connection Factories
   #
   # If you like to implement your own connection adapter, subclassing
-  # HTTPParty::ConnectionAdapter will make it easier. Just override
+  # HTTParty::ConnectionAdapter will make it easier. Just override
   # the #connection method. The uri and options attributes will have
   # all the info you need to construct your http connection. Whatever
   # you return from your connection method needs to adhere to the
@@ -38,26 +40,52 @@ module HTTParty
   # in the #options attribute. It is up to you to interpret them within your
   # connection adapter. Take a look at the implementation of
   # HTTParty::ConnectionAdapter#connection for examples of how they are used.
-  # Some things that are probably interesting are as follows:
+  # The keys used in options are
   # * :+timeout+: timeout in seconds
   # * :+open_timeout+: http connection open_timeout in seconds, overrides timeout if set
   # * :+read_timeout+: http connection read_timeout in seconds, overrides timeout if set
+  # * :+write_timeout+: http connection write_timeout in seconds, overrides timeout if set (Ruby >= 2.6.0 required)
   # * :+debug_output+: see HTTParty::ClassMethods.debug_output.
-  # * :+pem+: contains pem data. see HTTParty::ClassMethods.pem.
+  # * :+cert_store+: contains certificate data. see method 'attach_ssl_certificates'
+  # * :+pem+: contains pem client certificate data. see method 'attach_ssl_certificates'
+  # * :+p12+: contains PKCS12 client client certificate data.  see method 'attach_ssl_certificates'
   # * :+verify+: verify the server’s certificate against the ca certificate.
   # * :+verify_peer+: set to false to turn off server verification but still send client certificate
   # * :+ssl_ca_file+: see HTTParty::ClassMethods.ssl_ca_file.
   # * :+ssl_ca_path+: see HTTParty::ClassMethods.ssl_ca_path.
+  # * :+ssl_version+: SSL versions to allow. see method 'attach_ssl_certificates'
+  # * :+ciphers+: The list of SSL ciphers to support
   # * :+connection_adapter_options+: contains the hash you passed to HTTParty.connection_adapter when you configured your connection adapter
+  # * :+local_host+: The local address to bind to
+  # * :+local_port+: The local port to bind to
+  # * :+http_proxyaddr+: HTTP Proxy address
+  # * :+http_proxyport+: HTTP Proxy port
+  # * :+http_proxyuser+: HTTP Proxy user
+  # * :+http_proxypass+: HTTP Proxy password
+  #
+  # === Inherited methods
+  # * :+clean_host+: Method used to sanitize host names
+
   class ConnectionAdapter
     # Private: Regex used to strip brackets from IPv6 URIs.
     StripIpv6BracketsRegex = /\A\[(.*)\]\z/
 
+    OPTION_DEFAULTS = {
+      verify: true,
+      verify_peer: true
+    }
+
     # Public
     def self.call(uri, options)
       new(uri, options).connection
     end
 
+    def self.default_cert_store
+      @default_cert_store ||= OpenSSL::X509::Store.new.tap do |cert_store|
+        cert_store.set_default_paths
+      end
+    end
+
     attr_reader :uri, :options
 
     def initialize(uri, options = {})
@@ -65,14 +93,21 @@ module HTTParty
       raise ArgumentError, "uri must be a #{uri_adapter}, not a #{uri.class}" unless uri.is_a? uri_adapter
 
       @uri = uri
-      @options = options
+      @options = OPTION_DEFAULTS.merge(options)
     end
 
     def connection
       host = clean_host(uri.host)
       port = uri.port || (uri.scheme == 'https' ? 443 : 80)
-      if options[:http_proxyaddr]
-        http = Net::HTTP.new(host, port, options[:http_proxyaddr], options[:http_proxyport], options[:http_proxyuser], options[:http_proxypass])
+      if options.key?(:http_proxyaddr)
+        http = Net::HTTP.new(
+          host,
+          port,
+          options[:http_proxyaddr],
+          options[:http_proxyport],
+          options[:http_proxyuser],
+          options[:http_proxypass]
+        )
       else
         http = Net::HTTP.new(host, port)
       end
@@ -81,19 +116,28 @@ module HTTParty
 
       attach_ssl_certificates(http, options)
 
-      if options[:timeout] && (options[:timeout].is_a?(Integer) || options[:timeout].is_a?(Float))
+      if add_timeout?(options[:timeout])
         http.open_timeout = options[:timeout]
         http.read_timeout = options[:timeout]
+        http.write_timeout = options[:timeout]
       end
 
-      if options[:read_timeout] && (options[:read_timeout].is_a?(Integer) || options[:read_timeout].is_a?(Float))
+      if add_timeout?(options[:read_timeout])
         http.read_timeout = options[:read_timeout]
       end
 
-      if options[:open_timeout] && (options[:open_timeout].is_a?(Integer) || options[:open_timeout].is_a?(Float))
+      if add_timeout?(options[:open_timeout])
         http.open_timeout = options[:open_timeout]
       end
 
+      if add_timeout?(options[:write_timeout])
+        http.write_timeout = options[:write_timeout]
+      end
+
+      if add_max_retries?(options[:max_retries])
+        http.max_retries = options[:max_retries]
+      end
+
       if options[:debug_output]
         http.set_debug_output(options[:debug_output])
       end
@@ -106,19 +150,11 @@ module HTTParty
       #
       # @see https://bugs.ruby-lang.org/issues/6617
       if options[:local_host]
-        if RUBY_VERSION >= "2.0.0"
-          http.local_host = options[:local_host]
-        else
-          Kernel.warn("Warning: option :local_host requires Ruby version 2.0 or later")
-        end
+        http.local_host = options[:local_host]
       end
 
       if options[:local_port]
-        if RUBY_VERSION >= "2.0.0"
-          http.local_port = options[:local_port]
-        else
-          Kernel.warn("Warning: option :local_port requires Ruby version 2.0 or later")
-        end
+        http.local_port = options[:local_port]
       end
 
       http
@@ -126,6 +162,14 @@ module HTTParty
 
     private
 
+    def add_timeout?(timeout)
+      timeout && (timeout.is_a?(Integer) || timeout.is_a?(Float))
+    end
+
+    def add_max_retries?(max_retries)
+      max_retries && max_retries.is_a?(Integer) && max_retries >= 0
+    end
+
     def clean_host(host)
       strip_ipv6_brackets(host)
     end
@@ -138,6 +182,10 @@ module HTTParty
       uri.port == 443 || uri.scheme == 'https'
     end
 
+    def verify_ssl_certificate?
+      !(options[:verify] == false || options[:verify_peer] == false)
+    end
+
     def attach_ssl_certificates(http, options)
       if http.use_ssl?
         if options.fetch(:verify, true)
@@ -146,8 +194,7 @@ module HTTParty
             http.cert_store = options[:cert_store]
           else
             # Use the default cert store by default, i.e. system ca certs
-            http.cert_store = OpenSSL::X509::Store.new
-            http.cert_store.set_default_paths
+            http.cert_store = self.class.default_cert_store
           end
         else
           http.verify_mode = OpenSSL::SSL::VERIFY_NONE
@@ -157,8 +204,8 @@ module HTTParty
         # Note: options[:pem] must contain the content of a PEM file having the private key appended
         if options[:pem]
           http.cert = OpenSSL::X509::Certificate.new(options[:pem])
-          http.key = OpenSSL::PKey::RSA.new(options[:pem], options[:pem_password])
-          http.verify_mode = options[:verify_peer] == false ? OpenSSL::SSL::VERIFY_NONE : OpenSSL::SSL::VERIFY_PEER
+          http.key = OpenSSL::PKey.read(options[:pem], options[:pem_password])
+          http.verify_mode = verify_ssl_certificate? ? OpenSSL::SSL::VERIFY_PEER : OpenSSL::SSL::VERIFY_NONE
         end
 
         # PKCS12 client certificate authentication
@@ -166,7 +213,7 @@ module HTTParty
           p12 = OpenSSL::PKCS12.new(options[:p12], options[:p12_password])
           http.cert = p12.certificate
           http.key = p12.key
-          http.verify_mode = options[:verify_peer] == false ? OpenSSL::SSL::VERIFY_NONE : OpenSSL::SSL::VERIFY_PEER
+          http.verify_mode = verify_ssl_certificate? ? OpenSSL::SSL::VERIFY_PEER : OpenSSL::SSL::VERIFY_NONE
         end
 
         # SSL certificate authority file and/or directory

data/lib/httparty/cookie_hash.rb

@@ -1,14 +1,16 @@
+# frozen_string_literal: true
+
 class HTTParty::CookieHash < Hash #:nodoc:
-  CLIENT_COOKIES = %w(path expires domain path secure httponly)
+  CLIENT_COOKIES = %w(path expires domain path secure httponly samesite)
 
-  def add_cookies(value)
-    case value
+  def add_cookies(data)
+    case data
     when Hash
-      merge!(value)
+      merge!(data)
     when String
-      value.split('; ').each do |cookie|
-        array = cookie.split('=', 2)
-        self[array[0].to_sym] = array[1]
+      data.split('; ').each do |cookie|
+        key, value = cookie.split('=', 2)
+        self[key.to_sym] = value if key
       end
     else
       raise "add_cookies only takes a Hash or a String"
@@ -16,6 +18,6 @@ class HTTParty::CookieHash < Hash #:nodoc:
   end
 
   def to_cookie_string
-    delete_if { |k, v| CLIENT_COOKIES.include?(k.to_s.downcase) }.collect { |k, v| "#{k}=#{v}" }.join("; ")
+    select { |k, v| !CLIENT_COOKIES.include?(k.to_s.downcase) }.collect { |k, v| "#{k}=#{v}" }.join('; ')
   end
 end

data/lib/httparty/decompressor.rb

@@ -0,0 +1,102 @@
+# frozen_string_literal: true
+
+module HTTParty
+  # Decompresses the response body based on the Content-Encoding header.
+  #
+  # Net::HTTP automatically decompresses Content-Encoding values "gzip" and "deflate".
+  # This class will handle "br" (Brotli) and "compress" (LZW) if the requisite
+  # gems are installed. Otherwise, it returns nil if the body data cannot be
+  # decompressed.
+  #
+  # @abstract Read the HTTP Compression section for more information.
+  class Decompressor
+
+    # "gzip" and "deflate" are handled by Net::HTTP
+    # hence they do not need to be handled by HTTParty
+    SupportedEncodings = {
+      'none'     => :none,
+      'identity' => :none,
+      'br'       => :brotli,
+      'compress' => :lzw,
+      'zstd'     => :zstd
+    }.freeze
+
+    # The response body of the request
+    # @return [String]
+    attr_reader :body
+
+    # The Content-Encoding algorithm used to encode the body
+    # @return [Symbol] e.g. :gzip
+    attr_reader :encoding
+
+    # @param [String] body - the response body of the request
+    # @param [Symbol] encoding - the Content-Encoding algorithm used to encode the body
+    def initialize(body, encoding)
+      @body = body
+      @encoding = encoding
+    end
+
+    # Perform decompression on the response body
+    # @return [String] the decompressed body
+    # @return [nil] when the response body is nil or cannot decompressed
+    def decompress
+      return nil if body.nil?
+      return body if encoding.nil? || encoding.strip.empty?
+
+      if supports_encoding?
+        decompress_supported_encoding
+      else
+        nil
+      end
+    end
+
+    protected
+
+    def supports_encoding?
+      SupportedEncodings.keys.include?(encoding)
+    end
+
+    def decompress_supported_encoding
+      method = SupportedEncodings[encoding]
+      if respond_to?(method, true)
+        send(method)
+      else
+        raise NotImplementedError, "#{self.class.name} has not implemented a decompression method for #{encoding.inspect} encoding."
+      end
+    end
+
+    def none
+      body
+    end
+
+    def brotli
+      return nil unless defined?(::Brotli)
+      begin
+        ::Brotli.inflate(body)
+      rescue StandardError
+        nil
+      end
+    end
+
+    def lzw
+      begin
+        if defined?(::LZWS::String)
+          ::LZWS::String.decompress(body)
+        elsif defined?(::LZW::Simple)
+          ::LZW::Simple.new.decompress(body)
+        end
+      rescue StandardError
+        nil
+      end
+    end
+
+    def zstd
+      return nil unless defined?(::Zstd)
+      begin
+        ::Zstd.decompress(body)
+      rescue StandardError
+        nil
+      end
+    end
+  end
+end

data/lib/httparty/exceptions.rb

@@ -1,16 +1,42 @@
+# frozen_string_literal: true
+
 module HTTParty
-  # @abstact Exceptions raised by HTTParty inherit from Error
+  COMMON_NETWORK_ERRORS = [
+    EOFError,
+    Errno::ECONNABORTED,
+    Errno::ECONNREFUSED,
+    Errno::ECONNRESET,
+    Errno::EHOSTUNREACH,
+    Errno::EINVAL,
+    Errno::ENETUNREACH,
+    Errno::ENOTSOCK,
+    Errno::EPIPE,
+    Errno::ETIMEDOUT,
+    Net::HTTPBadResponse,
+    Net::HTTPHeaderSyntaxError,
+    Net::ProtocolError,
+    Net::ReadTimeout,
+    OpenSSL::SSL::SSLError,
+    SocketError,
+    Timeout::Error # Also covers subclasses like Net::OpenTimeout
+  ].freeze
+
+  # @abstract Exceptions raised by HTTParty inherit from Error
   class Error < StandardError; end
 
-  # Exception raised when you attempt to set a non-existant format
-  class UnsupportedFormat < Error; end
+  # @abstract Exceptions raised by HTTParty inherit from this because it is funny
+  # and if you don't like fun you should be using a different library.
+  class Foul < Error; end
+
+  # Exception raised when you attempt to set a non-existent format
+  class UnsupportedFormat < Foul; end
 
   # Exception raised when using a URI scheme other than HTTP or HTTPS
-  class UnsupportedURIScheme < Error; end
+  class UnsupportedURIScheme < Foul; end
 
   # @abstract Exceptions which inherit from ResponseError contain the Net::HTTP
   # response object accessible via the {#response} method.
-  class ResponseError < Error
+  class ResponseError < Foul
     # Returns the response of the last request
     # @return [Net::HTTPResponse] A subclass of Net::HTTPResponse, e.g.
     # Net::HTTPOK
@@ -20,10 +46,21 @@ module HTTParty
     # @param [Net::HTTPResponse]
     def initialize(response)
       @response = response
+      super(response)
     end
   end
 
   # Exception that is raised when request has redirected too many times.
   # Calling {#response} returns the Net:HTTP response object.
   class RedirectionTooDeep < ResponseError; end
+
+  # Exception that is raised when request redirects and location header is present more than once
+  class DuplicateLocationHeader < ResponseError; end
+
+  # Exception that is raised when common network errors occur.
+  class NetworkError < Foul; end
+
+  # Exception that is raised when an absolute URI is used that doesn't match
+  # the configured base_uri, which could indicate an SSRF attempt.
+  class UnsafeURIError < Foul; end
 end