httparty 0.19.0 → 0.24.0

data/Gemfile

@@ -1,8 +1,10 @@
 source 'https://rubygems.org'
 gemspec
 
+gem 'base64'
 gem 'rake'
 gem 'mongrel',  '1.2.0.pre2'
+gem 'json'
 
 group :development do
   gem 'guard'
@@ -11,6 +13,7 @@ group :development do
 end
 
 group :test do
+  gem 'rexml'
   gem 'rspec',    '~> 3.4'
   gem 'simplecov', require: false
   gem 'aruba'

data/Guardfile

@@ -1,7 +1,8 @@
 rspec_options = {
-  version:        1,
   all_after_pass: false,
-  all_on_start:   false
+  all_on_start: false,
+  failed_mode: :keep,
+  cmd: 'bundle exec rspec',
 }
 
 guard 'rspec', rspec_options do

data/README.md

@@ -1,8 +1,8 @@
 # httparty
 
-[![Build Status](https://travis-ci.org/jnunemaker/httparty.svg?branch=master)](https://travis-ci.org/jnunemaker/httparty)
+[![CI](https://github.com/jnunemaker/httparty/actions/workflows/ci.yml/badge.svg)](https://github.com/jnunemaker/httparty/actions/workflows/ci.yml)
 
-Makes http fun again!  Ain't no party like a httparty, because a httparty don't stop.
+Makes http fun again! Ain't no party like a httparty, because a httparty don't stop.
 
 ## Install
 
@@ -12,9 +12,8 @@ gem install httparty
 
 ## Requirements
 
-* Ruby 2.0.0 or higher
-* multi_xml
-* You like to party!
+- Ruby 2.7.0 or higher
+- You like to party!
 
 ## Examples
 
@@ -47,7 +46,8 @@ puts stack_exchange.questions
 puts stack_exchange.users
 ```
 
-See the [examples directory](http://github.com/jnunemaker/httparty/tree/master/examples) for even more goodies.
+See the [examples directory](http://github.com/jnunemaker/httparty/tree/main/examples) for even more goodies.
+
 ## Command Line Interface
 
 httparty also includes the executable `httparty` which can be
@@ -63,17 +63,17 @@ httparty "https://api.stackexchange.com/2.2/questions?site=stackoverflow"
 
 ## Help and Docs
 
-* [Docs](https://github.com/jnunemaker/httparty/tree/master/docs)
-* https://github.com/jnunemaker/httparty/discussions
-* https://www.rubydoc.info/github/jnunemaker/httparty
+- [Docs](https://github.com/jnunemaker/httparty/tree/main/docs)
+- https://github.com/jnunemaker/httparty/discussions
+- https://www.rubydoc.info/github/jnunemaker/httparty
 
 ## Contributing
 
-* Fork the project.
-* Run `bundle`
-* Run `bundle exec rake`
-* Make your feature addition or bug fix.
-* Add tests for it. This is important so I don't break it in a future version unintentionally.
-* Run `bundle exec rake` (No, REALLY :))
-* Commit, do not mess with rakefile, version, or history. (if you want to have your own version, that is fine but bump version in a commit by itself in another branch so I can ignore when I pull)
-* Send me a pull request. Bonus points for topic branches.
+- Fork the project.
+- Run `bundle`
+- Run `bundle exec rake`
+- Make your feature addition or bug fix.
+- Add tests for it. This is important so I don't break it in a future version unintentionally.
+- Run `bundle exec rake` (No, REALLY :))
+- Commit, do not mess with rakefile, version, or history. (if you want to have your own version, that is fine but bump version in a commit by itself in another branch so I can ignore when I pull)
+- Send me a pull request. Bonus points for topic branches.

data/docs/README.md

@@ -14,6 +14,20 @@ response = HTTParty.get('http://example.com', format: :plain)
 JSON.parse response, symbolize_names: true
 ```
 
+## Posting JSON
+When using Content Type `application/json` with `POST`, `PUT` or `PATCH` requests, the body should be a string of valid JSON:
+
+```ruby
+# With written JSON
+HTTParty.post('http://example.com', body: "{\"foo\":\"bar\"}", headers: { 'Content-Type' => 'application/json' })
+
+# Using JSON.generate
+HTTParty.post('http://example.com', body: JSON.generate({ foo: 'bar' }), headers: { 'Content-Type' => 'application/json' })
+
+# Using object.to_json
+HTTParty.post('http://example.com', body: { foo: 'bar' }.to_json, headers: { 'Content-Type' => 'application/json' })
+```
+
 ## Working with SSL
 
 You can use this guide to work with SSL certificates.
@@ -22,6 +36,8 @@ You can use this guide to work with SSL certificates.
 
 ```ruby
 # Use this example if you are using a pem file
+# - cert.pem must contain the content of a PEM file having the private key appended (separated from the cert by a newline \n)
+# - Use an empty string for the password if the cert is not password protected
 
 class Client
   include HTTParty
@@ -123,11 +139,16 @@ If you explicitly set `Accept-Encoding`, there be dragons:
   `Net::HTTP` will automatically decompress it, and will omit `Content-Encoding`
   from your `HTTParty::Response` headers.
 
-* For encodings `br` (Brotli) or `compress` (LZW), HTTParty will automatically
-  decompress if you include the `brotli` or `ruby-lzws` gems respectively into your project.
+* For the following encodings, HTTParty will automatically decompress them if you include
+  the required gem into your project. Similar to above, if decompression succeeds,
+  `Content-Encoding` will be omitted from your `HTTParty::Response` headers.
   **Warning:** Support for these encodings is experimental and not fully battle-tested.
-  Similar to above, if decompression succeeds, `Content-Encoding` will be omitted
-  from your `HTTParty::Response` headers.
+
+  | Content-Encoding | Required Gem |
+  | --- | --- |
+  | `br` (Brotli)      | [brotli](https://rubygems.org/gems/brotli) |
+  | `compress` (LZW)   | [ruby-lzws](https://rubygems.org/gems/ruby-lzws) |
+  | `zstd` (Zstandard) | [zstd-ruby](https://rubygems.org/gems/zstd-ruby) |
 
 * For other encodings, `HTTParty::Response#body` will return the raw uncompressed byte string,
   and you'll need to inspect the `Content-Encoding` response header and decompress it yourself.
@@ -147,7 +168,8 @@ JSON.parse(res.body) # safe
 
 require 'brotli'
 require 'lzws'
-res = HTTParty.get('https://example.com/test.json', headers: { 'Accept-Encoding' => 'br,compress' })
+require 'zstd-ruby'
+res = HTTParty.get('https://example.com/test.json', headers: { 'Accept-Encoding' => 'br,compress,zstd' })
 JSON.parse(res.body)
 
 

data/examples/party_foul_mode.rb

@@ -0,0 +1,90 @@
+require 'httparty'
+
+class APIClient
+  include HTTParty
+  base_uri 'api.example.com'
+
+  def self.fetch_user(id)
+    begin
+      get("/users/#{id}", foul: true)
+    rescue HTTParty::NetworkError => e
+      handle_network_error(e)
+    rescue HTTParty::ResponseError => e
+      handle_api_error(e)
+    end
+  end
+
+  private
+
+  def self.handle_network_error(error)
+    case error.cause
+    when Errno::ECONNREFUSED
+      {
+        error: :server_down,
+        message: "The API server appears to be down",
+        details: error.message
+      }
+    when Net::OpenTimeout, Timeout::Error
+      {
+        error: :timeout,
+        message: "The request timed out",
+        details: error.message
+      }
+    when SocketError
+      {
+        error: :network_error,
+        message: "Could not connect to the API server",
+        details: error.message
+      }
+    when OpenSSL::SSL::SSLError
+      {
+        error: :ssl_error,
+        message: "SSL certificate verification failed",
+        details: error.message
+      }
+    else
+      {
+        error: :unknown_network_error,
+        message: "An unexpected network error occurred",
+        details: error.message
+      }
+    end
+  end
+
+  def self.handle_api_error(error)
+    {
+      error: :api_error,
+      message: "API returned error #{error.response.code}",
+      details: error.response.body
+    }
+  end
+end
+
+# Example usage:
+
+# 1. When server is down
+result = APIClient.fetch_user(123)
+puts "Server down example:"
+puts result.inspect
+puts
+
+# 2. When request times out
+result = APIClient.fetch_user(456)
+puts "Timeout example:"
+puts result.inspect
+puts
+
+# 3. When SSL error occurs
+result = APIClient.fetch_user(789)
+puts "SSL error example:"
+puts result.inspect
+puts
+
+# 4. Simple example without a wrapper class
+begin
+  HTTParty.get('https://api.example.com/users', foul: true)
+rescue HTTParty::Foul => e
+  puts "Direct usage example:"
+  puts "Error type: #{e.cause.class}"
+  puts "Error message: #{e.message}"
+end

data/httparty.gemspec

@@ -12,11 +12,13 @@ Gem::Specification.new do |s|
   s.homepage    = "https://github.com/jnunemaker/httparty"
   s.summary     = 'Makes http fun! Also, makes consuming restful web services dead easy.'
   s.description = 'Makes http fun! Also, makes consuming restful web services dead easy.'
+  s.metadata["changelog_uri"] = 'https://github.com/jnunemaker/httparty/releases'
 
-  s.required_ruby_version     = '>= 2.0.0'
+  s.required_ruby_version     = '>= 2.7.0'
 
+  s.add_dependency 'csv'
   s.add_dependency 'multi_xml', ">= 0.5.2"
-  s.add_dependency('mime-types', "~> 3.0")
+  s.add_dependency 'mini_mime', ">= 1.0.0"
 
   # If this line is removed, all hard partying will cease.
   s.post_install_message = "When you HTTParty, you must party hard!"

data/lib/httparty/connection_adapter.rb

@@ -119,10 +119,7 @@ module HTTParty
       if add_timeout?(options[:timeout])
         http.open_timeout = options[:timeout]
         http.read_timeout = options[:timeout]
-
-        from_ruby_version('2.6.0', option: :write_timeout, warn: false) do
-          http.write_timeout = options[:timeout]
-        end
+        http.write_timeout = options[:timeout]
       end
 
       if add_timeout?(options[:read_timeout])
@@ -134,15 +131,11 @@ module HTTParty
       end
 
       if add_timeout?(options[:write_timeout])
-        from_ruby_version('2.6.0', option: :write_timeout) do
-          http.write_timeout = options[:write_timeout]
-        end
+        http.write_timeout = options[:write_timeout]
       end
 
       if add_max_retries?(options[:max_retries])
-        from_ruby_version('2.5.0', option: :max_retries) do
-          http.max_retries = options[:max_retries]
-        end
+        http.max_retries = options[:max_retries]
       end
 
       if options[:debug_output]
@@ -157,15 +150,11 @@ module HTTParty
       #
       # @see https://bugs.ruby-lang.org/issues/6617
       if options[:local_host]
-        from_ruby_version('2.0.0', option: :local_host) do
-          http.local_host = options[:local_host]
-        end
+        http.local_host = options[:local_host]
       end
 
       if options[:local_port]
-        from_ruby_version('2.0.0', option: :local_port) do
-          http.local_port = options[:local_port]
-        end
+        http.local_port = options[:local_port]
       end
 
       http
@@ -173,14 +162,6 @@ module HTTParty
 
     private
 
-    def from_ruby_version(ruby_version, option: nil, warn: true)
-      if RUBY_VERSION >= ruby_version
-        yield
-      elsif warn
-        Kernel.warn("Warning: option #{ option } requires Ruby version #{ ruby_version } or later")
-      end
-    end
-
     def add_timeout?(timeout)
       timeout && (timeout.is_a?(Integer) || timeout.is_a?(Float))
     end
@@ -223,7 +204,7 @@ module HTTParty
         # Note: options[:pem] must contain the content of a PEM file having the private key appended
         if options[:pem]
           http.cert = OpenSSL::X509::Certificate.new(options[:pem])
-          http.key = OpenSSL::PKey::RSA.new(options[:pem], options[:pem_password])
+          http.key = OpenSSL::PKey.read(options[:pem], options[:pem_password])
           http.verify_mode = verify_ssl_certificate? ? OpenSSL::SSL::VERIFY_PEER : OpenSSL::SSL::VERIFY_NONE
         end
 

data/lib/httparty/decompressor.rb

@@ -17,7 +17,8 @@ module HTTParty
       'none'     => :none,
       'identity' => :none,
       'br'       => :brotli,
-      'compress' => :lzw
+      'compress' => :lzw,
+      'zstd'     => :zstd
     }.freeze
 
     # The response body of the request
@@ -88,5 +89,14 @@ module HTTParty
         nil
       end
     end
+
+    def zstd
+      return nil unless defined?(::Zstd)
+      begin
+        ::Zstd.decompress(body)
+      rescue StandardError
+        nil
+      end
+    end
   end
 end

data/lib/httparty/exceptions.rb

@@ -1,18 +1,42 @@
 # frozen_string_literal: true
 
 module HTTParty
+  COMMON_NETWORK_ERRORS = [
+    EOFError,
+    Errno::ECONNABORTED,
+    Errno::ECONNREFUSED,
+    Errno::ECONNRESET,
+    Errno::EHOSTUNREACH,
+    Errno::EINVAL,
+    Errno::ENETUNREACH,
+    Errno::ENOTSOCK,
+    Errno::EPIPE,
+    Errno::ETIMEDOUT,
+    Net::HTTPBadResponse,
+    Net::HTTPHeaderSyntaxError,
+    Net::ProtocolError,
+    Net::ReadTimeout,
+    OpenSSL::SSL::SSLError,
+    SocketError,
+    Timeout::Error # Also covers subclasses like Net::OpenTimeout
+  ].freeze
+
   # @abstract Exceptions raised by HTTParty inherit from Error
   class Error < StandardError; end
 
+  # @abstract Exceptions raised by HTTParty inherit from this because it is funny
+  # and if you don't like fun you should be using a different library.
+  class Foul < Error; end
+
   # Exception raised when you attempt to set a non-existent format
-  class UnsupportedFormat < Error; end
+  class UnsupportedFormat < Foul; end
 
   # Exception raised when using a URI scheme other than HTTP or HTTPS
-  class UnsupportedURIScheme < Error; end
+  class UnsupportedURIScheme < Foul; end
 
   # @abstract Exceptions which inherit from ResponseError contain the Net::HTTP
   # response object accessible via the {#response} method.
-  class ResponseError < Error
+  class ResponseError < Foul
     # Returns the response of the last request
     # @return [Net::HTTPResponse] A subclass of Net::HTTPResponse, e.g.
     # Net::HTTPOK
@@ -32,4 +56,11 @@ module HTTParty
 
   # Exception that is raised when request redirects and location header is present more than once
   class DuplicateLocationHeader < ResponseError; end
+
+  # Exception that is raised when common network errors occur.
+  class NetworkError < Foul; end
+
+  # Exception that is raised when an absolute URI is used that doesn't match
+  # the configured base_uri, which could indicate an SSRF attempt.
+  class UnsafeURIError < Foul; end
 end

data/lib/httparty/hash_conversions.rb

@@ -28,7 +28,7 @@ module HTTParty
     def self.normalize_param(key, value)
       normalized_keys = normalize_keys(key, value)
 
-      normalized_keys.flatten.each_slice(2).inject(+'') do |string, (k, v)|
+      normalized_keys.flatten.each_slice(2).inject(''.dup) do |string, (k, v)|
         string << "#{ERB::Util.url_encode(k)}=#{ERB::Util.url_encode(v.to_s)}&"
       end
     end

data/lib/httparty/logger/curl_formatter.rb

@@ -22,7 +22,7 @@ module HTTParty
         log_request
         log_response
 
-        logger.public_send level, messages.join('\n')
+        logger.public_send level, messages.join("\n")
       end
 
       private

data/lib/httparty/logger/logstash_formatter.rb

@@ -24,6 +24,7 @@ module HTTParty
       attr_reader :request, :response
 
       def logstash_message
+        require 'json'
         {
           '@timestamp' => current_time,
           '@version' => 1,

data/lib/httparty/module_inheritable_attributes.rb

@@ -29,7 +29,7 @@ module HTTParty
         @mattr_inheritable_attrs += args
 
         args.each do |arg|
-          module_eval %(class << self; attr_accessor :#{arg} end)
+          singleton_class.attr_accessor(arg)
         end
 
         @mattr_inheritable_attrs
@@ -42,14 +42,12 @@ module HTTParty
           subclass.instance_variable_set(ivar, instance_variable_get(ivar).clone)
 
           if instance_variable_get(ivar).respond_to?(:merge)
-            method = <<-EOM
+            subclass.class_eval <<~RUBY, __FILE__, __LINE__ + 1
               def self.#{inheritable_attribute}
                 duplicate = ModuleInheritableAttributes.hash_deep_dup(#{ivar})
                 #{ivar} = superclass.#{inheritable_attribute}.merge(duplicate)
               end
-            EOM
-
-            subclass.class_eval method
+            RUBY
           end
         end
       end

data/lib/httparty/parser.rb

@@ -118,16 +118,19 @@ module HTTParty
     protected
 
     def xml
+      require 'multi_xml'
       MultiXml.parse(body)
     end
 
     UTF8_BOM = "\xEF\xBB\xBF"
 
     def json
+      require 'json'
       JSON.parse(body, :quirks_mode => true, :allow_nan => true)
     end
 
     def csv
+      require 'csv'
       CSV.parse(body)
     end
 

data/lib/httparty/request/body.rb

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 
 require_relative 'multipart_boundary'
+require_relative 'streaming_multipart_body'
 
 module HTTParty
   class Request
@@ -30,25 +31,48 @@ module HTTParty
         params.respond_to?(:to_hash) && (force_multipart || has_file?(params))
       end
 
+      def streaming?
+        multipart? && has_file?(params)
+      end
+
+      def to_stream
+        return nil unless streaming?
+        StreamingMultipartBody.new(prepared_parts, boundary)
+      end
+
+      def prepared_parts
+        normalized_params = params.flat_map { |key, value| HashConversions.normalize_keys(key, value) }
+        normalized_params.map do |key, value|
+          [key, value, file?(value)]
+        end
+      end
+
       private
 
+      # https://html.spec.whatwg.org/#multipart-form-data
+      MULTIPART_FORM_DATA_REPLACEMENT_TABLE = {
+        '"'  => '%22',
+        "\r" => '%0D',
+        "\n" => '%0A'
+      }.freeze
+
       def generate_multipart
         normalized_params = params.flat_map { |key, value| HashConversions.normalize_keys(key, value) }
 
-        multipart = normalized_params.inject(+'') do |memo, (key, value)|
-          memo << "--#{boundary}#{NEWLINE}"
-          memo << %(Content-Disposition: form-data; name="#{key}")
+        multipart = normalized_params.inject(''.b) do |memo, (key, value)|
+          memo << "--#{boundary}#{NEWLINE}".b
+          memo << %(Content-Disposition: form-data; name="#{key}").b
           # value.path is used to support ActionDispatch::Http::UploadedFile
           # https://github.com/jnunemaker/httparty/pull/585
-          memo << %(; filename="#{file_name(value)}") if file?(value)
-          memo << NEWLINE
-          memo << "Content-Type: #{content_type(value)}#{NEWLINE}" if file?(value)
-          memo << NEWLINE
+          memo << %(; filename="#{file_name(value).gsub(/["\r\n]/, MULTIPART_FORM_DATA_REPLACEMENT_TABLE)}").b if file?(value)
+          memo << NEWLINE.b
+          memo << "Content-Type: #{content_type(value)}#{NEWLINE}".b if file?(value)
+          memo << NEWLINE.b
           memo << content_body(value)
-          memo << NEWLINE
+          memo << NEWLINE.b
         end
 
-        multipart << "--#{boundary}--#{NEWLINE}"
+        multipart << "--#{boundary}--#{NEWLINE}".b
       end
 
       def has_file?(value)
@@ -76,16 +100,19 @@ module HTTParty
       def content_body(object)
         if file?(object)
           object = (file = object).read
+          object.force_encoding(Encoding::BINARY) if object.respond_to?(:force_encoding)
           file.rewind if file.respond_to?(:rewind)
+          object.to_s
+        else
+          object.to_s.b
         end
-
-        object.to_s
       end
 
       def content_type(object)
         return object.content_type if object.respond_to?(:content_type)
-        mime = MIME::Types.type_for(object.path)
-        mime.empty? ? 'application/octet-stream' : mime[0].content_type
+        require 'mini_mime'
+        mime = MiniMime.lookup_by_filename(object.path)
+        mime ? mime.content_type : 'application/octet-stream'
       end
 
       def file_name(object)