httparty 0.21.0 → 0.24.0

data/Gemfile

@@ -1,6 +1,7 @@
 source 'https://rubygems.org'
 gemspec
 
+gem 'base64'
 gem 'rake'
 gem 'mongrel',  '1.2.0.pre2'
 gem 'json'

data/README.md

@@ -2,7 +2,7 @@
 
 [![CI](https://github.com/jnunemaker/httparty/actions/workflows/ci.yml/badge.svg)](https://github.com/jnunemaker/httparty/actions/workflows/ci.yml)
 
-Makes http fun again!  Ain't no party like a httparty, because a httparty don't stop.
+Makes http fun again! Ain't no party like a httparty, because a httparty don't stop.
 
 ## Install
 
@@ -12,9 +12,8 @@ gem install httparty
 
 ## Requirements
 
-* Ruby 2.3.0 or higher
-* multi_xml
-* You like to party!
+- Ruby 2.7.0 or higher
+- You like to party!
 
 ## Examples
 
@@ -47,7 +46,8 @@ puts stack_exchange.questions
 puts stack_exchange.users
 ```
 
-See the [examples directory](http://github.com/jnunemaker/httparty/tree/master/examples) for even more goodies.
+See the [examples directory](http://github.com/jnunemaker/httparty/tree/main/examples) for even more goodies.
+
 ## Command Line Interface
 
 httparty also includes the executable `httparty` which can be
@@ -63,17 +63,17 @@ httparty "https://api.stackexchange.com/2.2/questions?site=stackoverflow"
 
 ## Help and Docs
 
-* [Docs](https://github.com/jnunemaker/httparty/tree/master/docs)
-* https://github.com/jnunemaker/httparty/discussions
-* https://www.rubydoc.info/github/jnunemaker/httparty
+- [Docs](https://github.com/jnunemaker/httparty/tree/main/docs)
+- https://github.com/jnunemaker/httparty/discussions
+- https://www.rubydoc.info/github/jnunemaker/httparty
 
 ## Contributing
 
-* Fork the project.
-* Run `bundle`
-* Run `bundle exec rake`
-* Make your feature addition or bug fix.
-* Add tests for it. This is important so I don't break it in a future version unintentionally.
-* Run `bundle exec rake` (No, REALLY :))
-* Commit, do not mess with rakefile, version, or history. (if you want to have your own version, that is fine but bump version in a commit by itself in another branch so I can ignore when I pull)
-* Send me a pull request. Bonus points for topic branches.
+- Fork the project.
+- Run `bundle`
+- Run `bundle exec rake`
+- Make your feature addition or bug fix.
+- Add tests for it. This is important so I don't break it in a future version unintentionally.
+- Run `bundle exec rake` (No, REALLY :))
+- Commit, do not mess with rakefile, version, or history. (if you want to have your own version, that is fine but bump version in a commit by itself in another branch so I can ignore when I pull)
+- Send me a pull request. Bonus points for topic branches.

data/docs/README.md

@@ -36,6 +36,8 @@ You can use this guide to work with SSL certificates.
 
 ```ruby
 # Use this example if you are using a pem file
+# - cert.pem must contain the content of a PEM file having the private key appended (separated from the cert by a newline \n)
+# - Use an empty string for the password if the cert is not password protected
 
 class Client
   include HTTParty

data/examples/party_foul_mode.rb

@@ -0,0 +1,90 @@
+require 'httparty'
+
+class APIClient
+  include HTTParty
+  base_uri 'api.example.com'
+
+  def self.fetch_user(id)
+    begin
+      get("/users/#{id}", foul: true)
+    rescue HTTParty::NetworkError => e
+      handle_network_error(e)
+    rescue HTTParty::ResponseError => e
+      handle_api_error(e)
+    end
+  end
+
+  private
+
+  def self.handle_network_error(error)
+    case error.cause
+    when Errno::ECONNREFUSED
+      {
+        error: :server_down,
+        message: "The API server appears to be down",
+        details: error.message
+      }
+    when Net::OpenTimeout, Timeout::Error
+      {
+        error: :timeout,
+        message: "The request timed out",
+        details: error.message
+      }
+    when SocketError
+      {
+        error: :network_error,
+        message: "Could not connect to the API server",
+        details: error.message
+      }
+    when OpenSSL::SSL::SSLError
+      {
+        error: :ssl_error,
+        message: "SSL certificate verification failed",
+        details: error.message
+      }
+    else
+      {
+        error: :unknown_network_error,
+        message: "An unexpected network error occurred",
+        details: error.message
+      }
+    end
+  end
+
+  def self.handle_api_error(error)
+    {
+      error: :api_error,
+      message: "API returned error #{error.response.code}",
+      details: error.response.body
+    }
+  end
+end
+
+# Example usage:
+
+# 1. When server is down
+result = APIClient.fetch_user(123)
+puts "Server down example:"
+puts result.inspect
+puts
+
+# 2. When request times out
+result = APIClient.fetch_user(456)
+puts "Timeout example:"
+puts result.inspect
+puts
+
+# 3. When SSL error occurs
+result = APIClient.fetch_user(789)
+puts "SSL error example:"
+puts result.inspect
+puts
+
+# 4. Simple example without a wrapper class
+begin
+  HTTParty.get('https://api.example.com/users', foul: true)
+rescue HTTParty::Foul => e
+  puts "Direct usage example:"
+  puts "Error type: #{e.cause.class}"
+  puts "Error message: #{e.message}"
+end

data/httparty.gemspec

@@ -12,9 +12,11 @@ Gem::Specification.new do |s|
   s.homepage    = "https://github.com/jnunemaker/httparty"
   s.summary     = 'Makes http fun! Also, makes consuming restful web services dead easy.'
   s.description = 'Makes http fun! Also, makes consuming restful web services dead easy.'
+  s.metadata["changelog_uri"] = 'https://github.com/jnunemaker/httparty/releases'
 
-  s.required_ruby_version     = '>= 2.3.0'
+  s.required_ruby_version     = '>= 2.7.0'
 
+  s.add_dependency 'csv'
   s.add_dependency 'multi_xml', ">= 0.5.2"
   s.add_dependency 'mini_mime', ">= 1.0.0"
 

data/lib/httparty/connection_adapter.rb

@@ -119,10 +119,7 @@ module HTTParty
       if add_timeout?(options[:timeout])
         http.open_timeout = options[:timeout]
         http.read_timeout = options[:timeout]
-
-        from_ruby_version('2.6.0', option: :write_timeout, warn: false) do
-          http.write_timeout = options[:timeout]
-        end
+        http.write_timeout = options[:timeout]
       end
 
       if add_timeout?(options[:read_timeout])
@@ -134,15 +131,11 @@ module HTTParty
       end
 
       if add_timeout?(options[:write_timeout])
-        from_ruby_version('2.6.0', option: :write_timeout) do
-          http.write_timeout = options[:write_timeout]
-        end
+        http.write_timeout = options[:write_timeout]
       end
 
       if add_max_retries?(options[:max_retries])
-        from_ruby_version('2.5.0', option: :max_retries) do
-          http.max_retries = options[:max_retries]
-        end
+        http.max_retries = options[:max_retries]
       end
 
       if options[:debug_output]
@@ -157,15 +150,11 @@ module HTTParty
       #
       # @see https://bugs.ruby-lang.org/issues/6617
       if options[:local_host]
-        from_ruby_version('2.0.0', option: :local_host) do
-          http.local_host = options[:local_host]
-        end
+        http.local_host = options[:local_host]
       end
 
       if options[:local_port]
-        from_ruby_version('2.0.0', option: :local_port) do
-          http.local_port = options[:local_port]
-        end
+        http.local_port = options[:local_port]
       end
 
       http
@@ -173,14 +162,6 @@ module HTTParty
 
     private
 
-    def from_ruby_version(ruby_version, option: nil, warn: true)
-      if RUBY_VERSION >= ruby_version
-        yield
-      elsif warn
-        Kernel.warn("Warning: option #{ option } requires Ruby version #{ ruby_version } or later")
-      end
-    end
-
     def add_timeout?(timeout)
       timeout && (timeout.is_a?(Integer) || timeout.is_a?(Float))
     end

data/lib/httparty/exceptions.rb

@@ -1,18 +1,42 @@
 # frozen_string_literal: true
 
 module HTTParty
+  COMMON_NETWORK_ERRORS = [
+    EOFError,
+    Errno::ECONNABORTED,
+    Errno::ECONNREFUSED,
+    Errno::ECONNRESET,
+    Errno::EHOSTUNREACH,
+    Errno::EINVAL,
+    Errno::ENETUNREACH,
+    Errno::ENOTSOCK,
+    Errno::EPIPE,
+    Errno::ETIMEDOUT,
+    Net::HTTPBadResponse,
+    Net::HTTPHeaderSyntaxError,
+    Net::ProtocolError,
+    Net::ReadTimeout,
+    OpenSSL::SSL::SSLError,
+    SocketError,
+    Timeout::Error # Also covers subclasses like Net::OpenTimeout
+  ].freeze
+
   # @abstract Exceptions raised by HTTParty inherit from Error
   class Error < StandardError; end
 
+  # @abstract Exceptions raised by HTTParty inherit from this because it is funny
+  # and if you don't like fun you should be using a different library.
+  class Foul < Error; end
+
   # Exception raised when you attempt to set a non-existent format
-  class UnsupportedFormat < Error; end
+  class UnsupportedFormat < Foul; end
 
   # Exception raised when using a URI scheme other than HTTP or HTTPS
-  class UnsupportedURIScheme < Error; end
+  class UnsupportedURIScheme < Foul; end
 
   # @abstract Exceptions which inherit from ResponseError contain the Net::HTTP
   # response object accessible via the {#response} method.
-  class ResponseError < Error
+  class ResponseError < Foul
     # Returns the response of the last request
     # @return [Net::HTTPResponse] A subclass of Net::HTTPResponse, e.g.
     # Net::HTTPOK
@@ -32,4 +56,11 @@ module HTTParty
 
   # Exception that is raised when request redirects and location header is present more than once
   class DuplicateLocationHeader < ResponseError; end
+
+  # Exception that is raised when common network errors occur.
+  class NetworkError < Foul; end
+
+  # Exception that is raised when an absolute URI is used that doesn't match
+  # the configured base_uri, which could indicate an SSRF attempt.
+  class UnsafeURIError < Foul; end
 end

data/lib/httparty/logger/logstash_formatter.rb

@@ -24,6 +24,7 @@ module HTTParty
       attr_reader :request, :response
 
       def logstash_message
+        require 'json'
         {
           '@timestamp' => current_time,
           '@version' => 1,

data/lib/httparty/module_inheritable_attributes.rb

@@ -29,7 +29,7 @@ module HTTParty
         @mattr_inheritable_attrs += args
 
         args.each do |arg|
-          module_eval %(class << self; attr_accessor :#{arg} end)
+          singleton_class.attr_accessor(arg)
         end
 
         @mattr_inheritable_attrs
@@ -42,14 +42,12 @@ module HTTParty
           subclass.instance_variable_set(ivar, instance_variable_get(ivar).clone)
 
           if instance_variable_get(ivar).respond_to?(:merge)
-            method = <<-EOM
+            subclass.class_eval <<~RUBY, __FILE__, __LINE__ + 1
               def self.#{inheritable_attribute}
                 duplicate = ModuleInheritableAttributes.hash_deep_dup(#{ivar})
                 #{ivar} = superclass.#{inheritable_attribute}.merge(duplicate)
               end
-            EOM
-
-            subclass.class_eval method
+            RUBY
           end
         end
       end

data/lib/httparty/parser.rb

@@ -118,16 +118,19 @@ module HTTParty
     protected
 
     def xml
+      require 'multi_xml'
       MultiXml.parse(body)
     end
 
     UTF8_BOM = "\xEF\xBB\xBF"
 
     def json
+      require 'json'
       JSON.parse(body, :quirks_mode => true, :allow_nan => true)
     end
 
     def csv
+      require 'csv'
       CSV.parse(body)
     end
 

data/lib/httparty/request/body.rb

@@ -1,6 +1,7 @@
 # frozen_string_literal: true
 
 require_relative 'multipart_boundary'
+require_relative 'streaming_multipart_body'
 
 module HTTParty
   class Request
@@ -30,6 +31,22 @@ module HTTParty
         params.respond_to?(:to_hash) && (force_multipart || has_file?(params))
       end
 
+      def streaming?
+        multipart? && has_file?(params)
+      end
+
+      def to_stream
+        return nil unless streaming?
+        StreamingMultipartBody.new(prepared_parts, boundary)
+      end
+
+      def prepared_parts
+        normalized_params = params.flat_map { |key, value| HashConversions.normalize_keys(key, value) }
+        normalized_params.map do |key, value|
+          [key, value, file?(value)]
+        end
+      end
+
       private
 
       # https://html.spec.whatwg.org/#multipart-form-data
@@ -42,20 +59,20 @@ module HTTParty
       def generate_multipart
         normalized_params = params.flat_map { |key, value| HashConversions.normalize_keys(key, value) }
 
-        multipart = normalized_params.inject(''.dup) do |memo, (key, value)|
-          memo << "--#{boundary}#{NEWLINE}"
-          memo << %(Content-Disposition: form-data; name="#{key}")
+        multipart = normalized_params.inject(''.b) do |memo, (key, value)|
+          memo << "--#{boundary}#{NEWLINE}".b
+          memo << %(Content-Disposition: form-data; name="#{key}").b
           # value.path is used to support ActionDispatch::Http::UploadedFile
           # https://github.com/jnunemaker/httparty/pull/585
-          memo << %(; filename="#{file_name(value).gsub(/["\r\n]/, MULTIPART_FORM_DATA_REPLACEMENT_TABLE)}") if file?(value)
-          memo << NEWLINE
-          memo << "Content-Type: #{content_type(value)}#{NEWLINE}" if file?(value)
-          memo << NEWLINE
+          memo << %(; filename="#{file_name(value).gsub(/["\r\n]/, MULTIPART_FORM_DATA_REPLACEMENT_TABLE)}").b if file?(value)
+          memo << NEWLINE.b
+          memo << "Content-Type: #{content_type(value)}#{NEWLINE}".b if file?(value)
+          memo << NEWLINE.b
           memo << content_body(value)
-          memo << NEWLINE
+          memo << NEWLINE.b
         end
 
-        multipart << "--#{boundary}--#{NEWLINE}"
+        multipart << "--#{boundary}--#{NEWLINE}".b
       end
 
       def has_file?(value)
@@ -83,14 +100,17 @@ module HTTParty
       def content_body(object)
         if file?(object)
           object = (file = object).read
+          object.force_encoding(Encoding::BINARY) if object.respond_to?(:force_encoding)
           file.rewind if file.respond_to?(:rewind)
+          object.to_s
+        else
+          object.to_s.b
         end
-
-        object.to_s
       end
 
       def content_type(object)
         return object.content_type if object.respond_to?(:content_type)
+        require 'mini_mime'
         mime = MiniMime.lookup_by_filename(object.path)
         mime ? mime.content_type : 'application/octet-stream'
       end

data/lib/httparty/request/streaming_multipart_body.rb

@@ -0,0 +1,188 @@
+# frozen_string_literal: true
+
+module HTTParty
+  class Request
+    class StreamingMultipartBody
+      NEWLINE = "\r\n"
+      CHUNK_SIZE = 64 * 1024 # 64 KB chunks
+
+      def initialize(parts, boundary)
+        @parts = parts
+        @boundary = boundary
+        @part_index = 0
+        @state = :header
+        @current_file = nil
+        @header_buffer = nil
+        @header_offset = 0
+        @footer_sent = false
+      end
+
+      def size
+        @size ||= calculate_size
+      end
+
+      def read(length = nil, outbuf = nil)
+        outbuf = outbuf ? outbuf.replace(''.b) : ''.b
+
+        return read_all(outbuf) if length.nil?
+
+        while outbuf.bytesize < length
+          chunk = read_chunk(length - outbuf.bytesize)
+          break if chunk.nil?
+          outbuf << chunk
+        end
+
+        outbuf.empty? ? nil : outbuf
+      end
+
+      def rewind
+        @part_index = 0
+        @state = :header
+        @current_file = nil
+        @header_buffer = nil
+        @header_offset = 0
+        @footer_sent = false
+        @parts.each do |_key, value, _is_file|
+          value.rewind if value.respond_to?(:rewind)
+        end
+      end
+
+      private
+
+      def read_all(outbuf)
+        while (chunk = read_chunk(CHUNK_SIZE))
+          outbuf << chunk
+        end
+        outbuf.empty? ? nil : outbuf
+      end
+
+      def read_chunk(max_length)
+        loop do
+          return nil if @part_index >= @parts.size && @footer_sent
+
+          if @part_index >= @parts.size
+            @footer_sent = true
+            return "--#{@boundary}--#{NEWLINE}".b
+          end
+
+          key, value, is_file = @parts[@part_index]
+
+          case @state
+          when :header
+            chunk = read_header_chunk(key, value, is_file, max_length)
+            return chunk if chunk
+
+          when :body
+            chunk = read_body_chunk(value, is_file, max_length)
+            return chunk if chunk
+
+          when :newline
+            @state = :header
+            @part_index += 1
+            return NEWLINE.b
+          end
+        end
+      end
+
+      def read_header_chunk(key, value, is_file, max_length)
+        if @header_buffer.nil?
+          @header_buffer = build_part_header(key, value, is_file)
+          @header_offset = 0
+        end
+
+        remaining = @header_buffer.bytesize - @header_offset
+        if remaining > 0
+          chunk_size = [remaining, max_length].min
+          chunk = @header_buffer.byteslice(@header_offset, chunk_size)
+          @header_offset += chunk_size
+          return chunk
+        end
+
+        @header_buffer = nil
+        @header_offset = 0
+        @state = :body
+        nil
+      end
+
+      def read_body_chunk(value, is_file, max_length)
+        if is_file
+          chunk = read_file_chunk(value, max_length)
+          if chunk
+            return chunk
+          else
+            @current_file = nil
+            @state = :newline
+            return nil
+          end
+        else
+          @state = :newline
+          return value.to_s.b
+        end
+      end
+
+      def read_file_chunk(file, max_length)
+        chunk_size = [max_length, CHUNK_SIZE].min
+        chunk = file.read(chunk_size)
+        return nil if chunk.nil?
+        chunk.force_encoding(Encoding::BINARY) if chunk.respond_to?(:force_encoding)
+        chunk
+      end
+
+      def build_part_header(key, value, is_file)
+        header = "--#{@boundary}#{NEWLINE}".b
+        header << %(Content-Disposition: form-data; name="#{key}").b
+        if is_file
+          header << %(; filename="#{file_name(value).gsub(/["\r\n]/, replacement_table)}").b
+          header << NEWLINE.b
+          header << "Content-Type: #{content_type(value)}#{NEWLINE}".b
+        end
+        header << NEWLINE.b
+        header
+      end
+
+      def calculate_size
+        total = 0
+        @parts.each do |key, value, is_file|
+          total += build_part_header(key, value, is_file).bytesize
+          total += content_size(value, is_file)
+          total += NEWLINE.bytesize
+        end
+        total += "--#{@boundary}--#{NEWLINE}".bytesize
+        total
+      end
+
+      def content_size(value, is_file)
+        if is_file
+          if value.respond_to?(:size)
+            value.size
+          elsif value.respond_to?(:stat)
+            value.stat.size
+          else
+            value.read.bytesize.tap { value.rewind }
+          end
+        else
+          value.to_s.b.bytesize
+        end
+      end
+
+      def content_type(object)
+        return object.content_type if object.respond_to?(:content_type)
+        require 'mini_mime'
+        mime = MiniMime.lookup_by_filename(object.path)
+        mime ? mime.content_type : 'application/octet-stream'
+      end
+
+      def file_name(object)
+        object.respond_to?(:original_filename) ? object.original_filename : File.basename(object.path)
+      end
+
+      def replacement_table
+        @replacement_table ||= {
+          '"'  => '%22',
+          "\r" => '%0D',
+          "\n" => '%0A'
+        }.freeze
+      end
+    end
+  end
+end