httparty 0.18.1 → 0.24.0

data/Gemfile

@@ -1,8 +1,10 @@
 source 'https://rubygems.org'
 gemspec
 
+gem 'base64'
 gem 'rake'
 gem 'mongrel',  '1.2.0.pre2'
+gem 'json'
 
 group :development do
   gem 'guard'
@@ -11,6 +13,7 @@ group :development do
 end
 
 group :test do
+  gem 'rexml'
   gem 'rspec',    '~> 3.4'
   gem 'simplecov', require: false
   gem 'aruba'

data/Guardfile

@@ -1,7 +1,8 @@
 rspec_options = {
-  version:        1,
   all_after_pass: false,
-  all_on_start:   false
+  all_on_start: false,
+  failed_mode: :keep,
+  cmd: 'bundle exec rspec',
 }
 
 guard 'rspec', rspec_options do

data/README.md

@@ -1,8 +1,8 @@
 # httparty
 
-[![Build Status](https://travis-ci.org/jnunemaker/httparty.svg?branch=master)](https://travis-ci.org/jnunemaker/httparty)
+[![CI](https://github.com/jnunemaker/httparty/actions/workflows/ci.yml/badge.svg)](https://github.com/jnunemaker/httparty/actions/workflows/ci.yml)
 
-Makes http fun again!  Ain't no party like a httparty, because a httparty don't stop.
+Makes http fun again! Ain't no party like a httparty, because a httparty don't stop.
 
 ## Install
 
@@ -12,9 +12,8 @@ gem install httparty
 
 ## Requirements
 
-* Ruby 2.0.0 or higher
-* multi_xml
-* You like to party!
+- Ruby 2.7.0 or higher
+- You like to party!
 
 ## Examples
 
@@ -47,7 +46,7 @@ puts stack_exchange.questions
 puts stack_exchange.users
 ```
 
-See the [examples directory](http://github.com/jnunemaker/httparty/tree/master/examples) for even more goodies.
+See the [examples directory](http://github.com/jnunemaker/httparty/tree/main/examples) for even more goodies.
 
 ## Command Line Interface
 
@@ -64,18 +63,17 @@ httparty "https://api.stackexchange.com/2.2/questions?site=stackoverflow"
 
 ## Help and Docs
 
-* [Docs](https://github.com/jnunemaker/httparty/tree/master/docs)
-* https://groups.google.com/forum/#!forum/httparty-gem
-* https://www.rubydoc.info/github/jnunemaker/httparty
-* http://stackoverflow.com/questions/tagged/httparty
+- [Docs](https://github.com/jnunemaker/httparty/tree/main/docs)
+- https://github.com/jnunemaker/httparty/discussions
+- https://www.rubydoc.info/github/jnunemaker/httparty
 
 ## Contributing
 
-* Fork the project.
-* Run `bundle`
-* Run `bundle exec rake`
-* Make your feature addition or bug fix.
-* Add tests for it. This is important so I don't break it in a future version unintentionally.
-* Run `bundle exec rake` (No, REALLY :))
-* Commit, do not mess with rakefile, version, or history. (if you want to have your own version, that is fine but bump version in a commit by itself in another branch so I can ignore when I pull)
-* Send me a pull request. Bonus points for topic branches.
+- Fork the project.
+- Run `bundle`
+- Run `bundle exec rake`
+- Make your feature addition or bug fix.
+- Add tests for it. This is important so I don't break it in a future version unintentionally.
+- Run `bundle exec rake` (No, REALLY :))
+- Commit, do not mess with rakefile, version, or history. (if you want to have your own version, that is fine but bump version in a commit by itself in another branch so I can ignore when I pull)
+- Send me a pull request. Bonus points for topic branches.

data/docs/README.md

@@ -14,6 +14,20 @@ response = HTTParty.get('http://example.com', format: :plain)
 JSON.parse response, symbolize_names: true
 ```
 
+## Posting JSON
+When using Content Type `application/json` with `POST`, `PUT` or `PATCH` requests, the body should be a string of valid JSON:
+
+```ruby
+# With written JSON
+HTTParty.post('http://example.com', body: "{\"foo\":\"bar\"}", headers: { 'Content-Type' => 'application/json' })
+
+# Using JSON.generate
+HTTParty.post('http://example.com', body: JSON.generate({ foo: 'bar' }), headers: { 'Content-Type' => 'application/json' })
+
+# Using object.to_json
+HTTParty.post('http://example.com', body: { foo: 'bar' }.to_json, headers: { 'Content-Type' => 'application/json' })
+```
+
 ## Working with SSL
 
 You can use this guide to work with SSL certificates.
@@ -22,6 +36,8 @@ You can use this guide to work with SSL certificates.
 
 ```ruby
 # Use this example if you are using a pem file
+# - cert.pem must contain the content of a PEM file having the private key appended (separated from the cert by a newline \n)
+# - Use an empty string for the password if the cert is not password protected
 
 class Client
   include HTTParty
@@ -70,7 +86,7 @@ class Client
 end
 ```
 
-You can also include this options with the call:
+You can also include all of these options with the call:
 
 ```ruby
 class Client
@@ -86,7 +102,7 @@ end
 
 ### Avoid SSL verification
 
-In some cases you may want to skip SSL verification, because the entity that issue the certificate is not a valid one, but you still want to work with it. You can achieve this through:
+In some cases you may want to skip SSL verification, because the entity that issued the certificate is not a valid one, but you still want to work with it. You can achieve this through:
 
 ```ruby
 # Skips SSL certificate verification
@@ -104,3 +120,74 @@ class Client
   end
 end
 ```
+
+### HTTP Compression
+
+The `Accept-Encoding` request header and `Content-Encoding` response header
+are used to control compression (gzip, etc.) over the wire. Refer to
+[RFC-2616](https://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html) for details.
+(For clarity: these headers are **not** used for character encoding i.e. `utf-8`
+which is specified in the `Accept` and `Content-Type` headers.)
+
+Unless you have specific requirements otherwise, we recommend to **not** set
+set the `Accept-Encoding` header on HTTParty requests. In this case, `Net::HTTP`
+will set a sensible default compression scheme and automatically decompress the response.
+
+If you explicitly set `Accept-Encoding`, there be dragons:
+
+* If the HTTP response `Content-Encoding` received on the wire is `gzip` or `deflate`,
+  `Net::HTTP` will automatically decompress it, and will omit `Content-Encoding`
+  from your `HTTParty::Response` headers.
+
+* For the following encodings, HTTParty will automatically decompress them if you include
+  the required gem into your project. Similar to above, if decompression succeeds,
+  `Content-Encoding` will be omitted from your `HTTParty::Response` headers.
+  **Warning:** Support for these encodings is experimental and not fully battle-tested.
+
+  | Content-Encoding | Required Gem |
+  | --- | --- |
+  | `br` (Brotli)      | [brotli](https://rubygems.org/gems/brotli) |
+  | `compress` (LZW)   | [ruby-lzws](https://rubygems.org/gems/ruby-lzws) |
+  | `zstd` (Zstandard) | [zstd-ruby](https://rubygems.org/gems/zstd-ruby) |
+
+* For other encodings, `HTTParty::Response#body` will return the raw uncompressed byte string,
+  and you'll need to inspect the `Content-Encoding` response header and decompress it yourself.
+  In this case, `HTTParty::Response#parsed_response` will be `nil`.
+
+* Lastly, you may use the `skip_decompression` option to disable all automatic decompression
+  and always get `HTTParty::Response#body` in its raw form along with the `Content-Encoding` header.
+
+```ruby
+# Accept-Encoding=gzip,deflate can be safely assumed to be auto-decompressed
+
+res = HTTParty.get('https://example.com/test.json', headers: { 'Accept-Encoding' => 'gzip,deflate,identity' })
+JSON.parse(res.body) # safe
+
+
+# Accept-Encoding=br,compress requires third-party gems
+
+require 'brotli'
+require 'lzws'
+require 'zstd-ruby'
+res = HTTParty.get('https://example.com/test.json', headers: { 'Accept-Encoding' => 'br,compress,zstd' })
+JSON.parse(res.body)
+
+
+# Accept-Encoding=* may return unhandled Content-Encoding
+
+res = HTTParty.get('https://example.com/test.json', headers: { 'Accept-Encoding' => '*' })
+encoding = res.headers['Content-Encoding']
+if encoding
+JSON.parse(your_decompression_handling(res.body, encoding))
+else
+# Content-Encoding not present implies decompressed
+JSON.parse(res.body)
+end
+
+
+# Gimme the raw data!
+
+res = HTTParty.get('https://example.com/test.json', skip_decompression: true)
+encoding = res.headers['Content-Encoding']
+JSON.parse(your_decompression_handling(res.body, encoding))
+```

data/examples/README.md

@@ -84,3 +84,6 @@
 
 * [Accessing x509 Peer Certificate](peer_cert.rb)
   * Provides access to the server's TLS certificate
+
+* [Accessing IDNs](idn.rb)
+  * Uses a `get` request with an International domain names, which are Urls with emojis and non-ASCII characters such as accented letters.

data/examples/aaws.rb

@@ -1,5 +1,7 @@
 require 'rubygems'
 require 'active_support'
+require 'active_support/core_ext/hash'
+require 'active_support/core_ext/string'
 
 dir = File.expand_path(File.join(File.dirname(__FILE__), '..', 'lib'))
 require File.join(dir, 'httparty')
@@ -13,14 +15,16 @@ module AAWS
     default_params Service: 'AWSECommerceService', Operation: 'ItemSearch', SearchIndex: 'Books'
 
     def initialize(key)
-      self.class.default_params AWSAccessKeyId: key
+      @auth = { AWSAccessKeyId: key }
     end
 
     def search(options = {})
       raise ArgumentError, 'You must search for something' if options[:query].blank?
 
       # amazon uses nasty camelized query params
-      options[:query] = options[:query].inject({}) { |h, q| h[q[0].to_s.camelize] = q[1]; h }
+      options[:query] = options[:query]
+        .reverse_merge(@auth)
+        .transform_keys { |k| k.to_s.camelize }
 
       # make a request and return the items (NOTE: this doesn't handle errors at this point)
       self.class.get('/onca/xml', options)['ItemSearchResponse']['Items']

data/examples/idn.rb

@@ -0,0 +1,10 @@
+dir = File.expand_path(File.join(File.dirname(__FILE__), '..', 'lib'))
+require File.join(dir, 'httparty')
+require 'pp'
+
+class Idn
+    include HTTParty
+    uri_adapter Addressable::URI
+end
+
+pp Idn.get("https://i❤️.ws/emojidomain/💎?format=json")

data/examples/party_foul_mode.rb

@@ -0,0 +1,90 @@
+require 'httparty'
+
+class APIClient
+  include HTTParty
+  base_uri 'api.example.com'
+
+  def self.fetch_user(id)
+    begin
+      get("/users/#{id}", foul: true)
+    rescue HTTParty::NetworkError => e
+      handle_network_error(e)
+    rescue HTTParty::ResponseError => e
+      handle_api_error(e)
+    end
+  end
+
+  private
+
+  def self.handle_network_error(error)
+    case error.cause
+    when Errno::ECONNREFUSED
+      {
+        error: :server_down,
+        message: "The API server appears to be down",
+        details: error.message
+      }
+    when Net::OpenTimeout, Timeout::Error
+      {
+        error: :timeout,
+        message: "The request timed out",
+        details: error.message
+      }
+    when SocketError
+      {
+        error: :network_error,
+        message: "Could not connect to the API server",
+        details: error.message
+      }
+    when OpenSSL::SSL::SSLError
+      {
+        error: :ssl_error,
+        message: "SSL certificate verification failed",
+        details: error.message
+      }
+    else
+      {
+        error: :unknown_network_error,
+        message: "An unexpected network error occurred",
+        details: error.message
+      }
+    end
+  end
+
+  def self.handle_api_error(error)
+    {
+      error: :api_error,
+      message: "API returned error #{error.response.code}",
+      details: error.response.body
+    }
+  end
+end
+
+# Example usage:
+
+# 1. When server is down
+result = APIClient.fetch_user(123)
+puts "Server down example:"
+puts result.inspect
+puts
+
+# 2. When request times out
+result = APIClient.fetch_user(456)
+puts "Timeout example:"
+puts result.inspect
+puts
+
+# 3. When SSL error occurs
+result = APIClient.fetch_user(789)
+puts "SSL error example:"
+puts result.inspect
+puts
+
+# 4. Simple example without a wrapper class
+begin
+  HTTParty.get('https://api.example.com/users', foul: true)
+rescue HTTParty::Foul => e
+  puts "Direct usage example:"
+  puts "Error type: #{e.cause.class}"
+  puts "Error message: #{e.message}"
+end

data/httparty.gemspec

@@ -12,11 +12,13 @@ Gem::Specification.new do |s|
   s.homepage    = "https://github.com/jnunemaker/httparty"
   s.summary     = 'Makes http fun! Also, makes consuming restful web services dead easy.'
   s.description = 'Makes http fun! Also, makes consuming restful web services dead easy.'
+  s.metadata["changelog_uri"] = 'https://github.com/jnunemaker/httparty/releases'
 
-  s.required_ruby_version     = '>= 2.0.0'
+  s.required_ruby_version     = '>= 2.7.0'
 
+  s.add_dependency 'csv'
   s.add_dependency 'multi_xml', ">= 0.5.2"
-  s.add_dependency('mime-types', "~> 3.0")
+  s.add_dependency 'mini_mime', ">= 1.0.0"
 
   # If this line is removed, all hard partying will cease.
   s.post_install_message = "When you HTTParty, you must party hard!"

data/lib/httparty/connection_adapter.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module HTTParty
   # Default connection adapter that returns a new Net::HTTP each time
   #
@@ -117,10 +119,7 @@ module HTTParty
       if add_timeout?(options[:timeout])
         http.open_timeout = options[:timeout]
         http.read_timeout = options[:timeout]
-
-        from_ruby_version('2.6.0', option: :write_timeout, warn: false) do
-          http.write_timeout = options[:timeout]
-        end
+        http.write_timeout = options[:timeout]
       end
 
       if add_timeout?(options[:read_timeout])
@@ -132,15 +131,11 @@ module HTTParty
       end
 
       if add_timeout?(options[:write_timeout])
-        from_ruby_version('2.6.0', option: :write_timeout) do
-          http.write_timeout = options[:write_timeout]
-        end
+        http.write_timeout = options[:write_timeout]
       end
 
       if add_max_retries?(options[:max_retries])
-        from_ruby_version('2.5.0', option: :max_retries) do
-          http.max_retries = options[:max_retries]
-        end
+        http.max_retries = options[:max_retries]
       end
 
       if options[:debug_output]
@@ -155,15 +150,11 @@ module HTTParty
       #
       # @see https://bugs.ruby-lang.org/issues/6617
       if options[:local_host]
-        from_ruby_version('2.0.0', option: :local_host) do
-          http.local_host = options[:local_host]
-        end
+        http.local_host = options[:local_host]
       end
 
       if options[:local_port]
-        from_ruby_version('2.0.0', option: :local_port) do
-          http.local_port = options[:local_port]
-        end
+        http.local_port = options[:local_port]
       end
 
       http
@@ -171,14 +162,6 @@ module HTTParty
 
     private
 
-    def from_ruby_version(ruby_version, option: nil, warn: true)
-      if RUBY_VERSION >= ruby_version
-        yield
-      elsif warn
-        Kernel.warn("Warning: option #{ option } requires Ruby version #{ ruby_version } or later")
-      end
-    end
-
     def add_timeout?(timeout)
       timeout && (timeout.is_a?(Integer) || timeout.is_a?(Float))
     end
@@ -221,7 +204,7 @@ module HTTParty
         # Note: options[:pem] must contain the content of a PEM file having the private key appended
         if options[:pem]
           http.cert = OpenSSL::X509::Certificate.new(options[:pem])
-          http.key = OpenSSL::PKey::RSA.new(options[:pem], options[:pem_password])
+          http.key = OpenSSL::PKey.read(options[:pem], options[:pem_password])
           http.verify_mode = verify_ssl_certificate? ? OpenSSL::SSL::VERIFY_PEER : OpenSSL::SSL::VERIFY_NONE
         end
 

data/lib/httparty/cookie_hash.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 class HTTParty::CookieHash < Hash #:nodoc:
   CLIENT_COOKIES = %w(path expires domain path secure httponly samesite)
 
@@ -16,6 +18,6 @@ class HTTParty::CookieHash < Hash #:nodoc:
   end
 
   def to_cookie_string
-    select { |k, v| !CLIENT_COOKIES.include?(k.to_s.downcase) }.collect { |k, v| "#{k}=#{v}" }.join("; ")
+    select { |k, v| !CLIENT_COOKIES.include?(k.to_s.downcase) }.collect { |k, v| "#{k}=#{v}" }.join('; ')
   end
 end

data/lib/httparty/decompressor.rb

@@ -0,0 +1,102 @@
+# frozen_string_literal: true
+
+module HTTParty
+  # Decompresses the response body based on the Content-Encoding header.
+  #
+  # Net::HTTP automatically decompresses Content-Encoding values "gzip" and "deflate".
+  # This class will handle "br" (Brotli) and "compress" (LZW) if the requisite
+  # gems are installed. Otherwise, it returns nil if the body data cannot be
+  # decompressed.
+  #
+  # @abstract Read the HTTP Compression section for more information.
+  class Decompressor
+
+    # "gzip" and "deflate" are handled by Net::HTTP
+    # hence they do not need to be handled by HTTParty
+    SupportedEncodings = {
+      'none'     => :none,
+      'identity' => :none,
+      'br'       => :brotli,
+      'compress' => :lzw,
+      'zstd'     => :zstd
+    }.freeze
+
+    # The response body of the request
+    # @return [String]
+    attr_reader :body
+
+    # The Content-Encoding algorithm used to encode the body
+    # @return [Symbol] e.g. :gzip
+    attr_reader :encoding
+
+    # @param [String] body - the response body of the request
+    # @param [Symbol] encoding - the Content-Encoding algorithm used to encode the body
+    def initialize(body, encoding)
+      @body = body
+      @encoding = encoding
+    end
+
+    # Perform decompression on the response body
+    # @return [String] the decompressed body
+    # @return [nil] when the response body is nil or cannot decompressed
+    def decompress
+      return nil if body.nil?
+      return body if encoding.nil? || encoding.strip.empty?
+
+      if supports_encoding?
+        decompress_supported_encoding
+      else
+        nil
+      end
+    end
+
+    protected
+
+    def supports_encoding?
+      SupportedEncodings.keys.include?(encoding)
+    end
+
+    def decompress_supported_encoding
+      method = SupportedEncodings[encoding]
+      if respond_to?(method, true)
+        send(method)
+      else
+        raise NotImplementedError, "#{self.class.name} has not implemented a decompression method for #{encoding.inspect} encoding."
+      end
+    end
+
+    def none
+      body
+    end
+
+    def brotli
+      return nil unless defined?(::Brotli)
+      begin
+        ::Brotli.inflate(body)
+      rescue StandardError
+        nil
+      end
+    end
+
+    def lzw
+      begin
+        if defined?(::LZWS::String)
+          ::LZWS::String.decompress(body)
+        elsif defined?(::LZW::Simple)
+          ::LZW::Simple.new.decompress(body)
+        end
+      rescue StandardError
+        nil
+      end
+    end
+
+    def zstd
+      return nil unless defined?(::Zstd)
+      begin
+        ::Zstd.decompress(body)
+      rescue StandardError
+        nil
+      end
+    end
+  end
+end

data/lib/httparty/exceptions.rb

@@ -1,16 +1,42 @@
+# frozen_string_literal: true
+
 module HTTParty
-  # @abstact Exceptions raised by HTTParty inherit from Error
+  COMMON_NETWORK_ERRORS = [
+    EOFError,
+    Errno::ECONNABORTED,
+    Errno::ECONNREFUSED,
+    Errno::ECONNRESET,
+    Errno::EHOSTUNREACH,
+    Errno::EINVAL,
+    Errno::ENETUNREACH,
+    Errno::ENOTSOCK,
+    Errno::EPIPE,
+    Errno::ETIMEDOUT,
+    Net::HTTPBadResponse,
+    Net::HTTPHeaderSyntaxError,
+    Net::ProtocolError,
+    Net::ReadTimeout,
+    OpenSSL::SSL::SSLError,
+    SocketError,
+    Timeout::Error # Also covers subclasses like Net::OpenTimeout
+  ].freeze
+
+  # @abstract Exceptions raised by HTTParty inherit from Error
   class Error < StandardError; end
 
+  # @abstract Exceptions raised by HTTParty inherit from this because it is funny
+  # and if you don't like fun you should be using a different library.
+  class Foul < Error; end
+
   # Exception raised when you attempt to set a non-existent format
-  class UnsupportedFormat < Error; end
+  class UnsupportedFormat < Foul; end
 
   # Exception raised when using a URI scheme other than HTTP or HTTPS
-  class UnsupportedURIScheme < Error; end
+  class UnsupportedURIScheme < Foul; end
 
   # @abstract Exceptions which inherit from ResponseError contain the Net::HTTP
   # response object accessible via the {#response} method.
-  class ResponseError < Error
+  class ResponseError < Foul
     # Returns the response of the last request
     # @return [Net::HTTPResponse] A subclass of Net::HTTPResponse, e.g.
     # Net::HTTPOK
@@ -30,4 +56,11 @@ module HTTParty
 
   # Exception that is raised when request redirects and location header is present more than once
   class DuplicateLocationHeader < ResponseError; end
+
+  # Exception that is raised when common network errors occur.
+  class NetworkError < Foul; end
+
+  # Exception that is raised when an absolute URI is used that doesn't match
+  # the configured base_uri, which could indicate an SSRF attempt.
+  class UnsafeURIError < Foul; end
 end

data/lib/httparty/hash_conversions.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'erb'
 
 module HTTParty
@@ -26,8 +28,8 @@ module HTTParty
     def self.normalize_param(key, value)
       normalized_keys = normalize_keys(key, value)
 
-      normalized_keys.flatten.each_slice(2).inject('') do |string, (k, v)|
-        string + "#{ERB::Util.url_encode(k)}=#{ERB::Util.url_encode(v.to_s)}&"
+      normalized_keys.flatten.each_slice(2).inject(''.dup) do |string, (k, v)|
+        string << "#{ERB::Util.url_encode(k)}=#{ERB::Util.url_encode(v.to_s)}&"
       end
     end
 

data/lib/httparty/headers_processor.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module HTTParty
   class HeadersProcessor
     attr_reader :headers, :options

data/lib/httparty/logger/apache_formatter.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module HTTParty
   module Logger
     class ApacheFormatter #:nodoc:
@@ -26,11 +28,11 @@ module HTTParty
       end
 
       def current_time
-        Time.now.strftime("%Y-%m-%d %H:%M:%S %z")
+        Time.now.strftime('%Y-%m-%d %H:%M:%S %z')
       end
 
       def http_method
-        request.http_method.name.split("::").last.upcase
+        request.http_method.name.split('::').last.upcase
       end
 
       def path