httparty 0.17.3 → 0.22.0

data/lib/httparty/parser.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module HTTParty
   # The default parser used by HTTParty, supports xml, json, html, csv and
   # plain text.
@@ -101,7 +103,7 @@ module HTTParty
     # @return [nil] when the response body is nil, an empty string, spaces only or "null"
     def parse
       return nil if body.nil?
-      return nil if body == "null"
+      return nil if body == 'null'
       return nil if body.valid_encoding? && body.strip.empty?
       if body.valid_encoding? && body.encoding == Encoding::UTF_8
         @body = body.gsub(/\A#{UTF8_BOM}/, '')
@@ -116,16 +118,19 @@ module HTTParty
     protected
 
     def xml
+      require 'multi_xml'
       MultiXml.parse(body)
     end
 
-    UTF8_BOM = "\xEF\xBB\xBF".freeze
+    UTF8_BOM = "\xEF\xBB\xBF"
 
     def json
+      require 'json'
       JSON.parse(body, :quirks_mode => true, :allow_nan => true)
     end
 
     def csv
+      require 'csv'
       CSV.parse(body)
     end
 
@@ -142,9 +147,11 @@ module HTTParty
     end
 
     def parse_supported_format
-      send(format)
-    rescue NoMethodError => e
-      raise NotImplementedError, "#{self.class.name} has not implemented a parsing method for the #{format.inspect} format.", e.backtrace
+      if respond_to?(format, true)
+        send(format)
+      else
+        raise NotImplementedError, "#{self.class.name} has not implemented a parsing method for the #{format.inspect} format."
+      end
     end
   end
 end

data/lib/httparty/request/body.rb

@@ -1,8 +1,13 @@
+# frozen_string_literal: true
+
 require_relative 'multipart_boundary'
 
 module HTTParty
   class Request
     class Body
+      NEWLINE = "\r\n"
+      private_constant :NEWLINE
+
       def initialize(params, query_string_normalizer: nil, force_multipart: false)
         @params = params
         @query_string_normalizer = query_string_normalizer
@@ -27,23 +32,30 @@ module HTTParty
 
       private
 
+      # https://html.spec.whatwg.org/#multipart-form-data
+      MULTIPART_FORM_DATA_REPLACEMENT_TABLE = {
+        '"'  => '%22',
+        "\r" => '%0D',
+        "\n" => '%0A'
+      }.freeze
+
       def generate_multipart
         normalized_params = params.flat_map { |key, value| HashConversions.normalize_keys(key, value) }
 
-        multipart = normalized_params.inject('') do |memo, (key, value)|
-          memo += "--#{boundary}\r\n"
-          memo += %(Content-Disposition: form-data; name="#{key}")
+        multipart = normalized_params.inject(''.dup) do |memo, (key, value)|
+          memo << "--#{boundary}#{NEWLINE}"
+          memo << %(Content-Disposition: form-data; name="#{key}")
           # value.path is used to support ActionDispatch::Http::UploadedFile
           # https://github.com/jnunemaker/httparty/pull/585
-          memo += %(; filename="#{file_name(value)}") if file?(value)
-          memo += "\r\n"
-          memo += "Content-Type: #{content_type(value)}\r\n" if file?(value)
-          memo += "\r\n"
-          memo += file?(value) ? value.read : value.to_s
-          memo += "\r\n"
+          memo << %(; filename="#{file_name(value).gsub(/["\r\n]/, MULTIPART_FORM_DATA_REPLACEMENT_TABLE)}") if file?(value)
+          memo << NEWLINE
+          memo << "Content-Type: #{content_type(value)}#{NEWLINE}" if file?(value)
+          memo << NEWLINE
+          memo << content_body(value)
+          memo << NEWLINE
         end
 
-        multipart += "--#{boundary}--\r\n"
+        multipart << "--#{boundary}--#{NEWLINE}"
       end
 
       def has_file?(value)
@@ -68,10 +80,20 @@ module HTTParty
         end
       end
 
+      def content_body(object)
+        if file?(object)
+          object = (file = object).read
+          file.rewind if file.respond_to?(:rewind)
+        end
+
+        object.to_s
+      end
+
       def content_type(object)
         return object.content_type if object.respond_to?(:content_type)
-        mime = MIME::Types.type_for(object.path)
-        mime.empty? ? 'application/octet-stream' : mime[0].content_type
+        require 'mini_mime'
+        mime = MiniMime.lookup_by_filename(object.path)
+        mime ? mime.content_type : 'application/octet-stream'
       end
 
       def file_name(object)

data/lib/httparty/request/multipart_boundary.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'securerandom'
 
 module HTTParty

data/lib/httparty/request.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'erb'
 
 module HTTParty
@@ -44,6 +46,15 @@ module HTTParty
       end.flatten.join('&')
     end
 
+    def self._load(data)
+      http_method, path, options, last_response, last_uri, raw_request = Marshal.load(data)
+      instance = new(http_method, path, options)
+      instance.last_response = last_response
+      instance.last_uri = last_uri
+      instance.instance_variable_set("@raw_request", raw_request)
+      instance
+    end
+
     attr_accessor :http_method, :options, :last_response, :redirect, :last_uri
     attr_reader :path
 
@@ -87,11 +98,11 @@ module HTTParty
     end
 
     def uri
-      if redirect && path.relative? && path.path[0] != "/"
-        last_uri_host = @last_uri.path.gsub(/[^\/]+$/, "")
+      if redirect && path.relative? && path.path[0] != '/'
+        last_uri_host = @last_uri.path.gsub(/[^\/]+$/, '')
 
-        path.path = "/#{path.path}" if last_uri_host[-1] != "/"
-        path.path = last_uri_host + path.path
+        path.path = "/#{path.path}" if last_uri_host[-1] != '/'
+        path.path = "#{last_uri_host}#{path.path}"
       end
 
       if path.relative? && path.host
@@ -117,7 +128,7 @@ module HTTParty
     def base_uri
       if redirect
         base_uri = "#{@last_uri.scheme}://#{@last_uri.host}"
-        base_uri += ":#{@last_uri.port}" if @last_uri.port != 80
+        base_uri = "#{base_uri}:#{@last_uri.port}" if @last_uri.port != 80
         base_uri
       else
         options[:base_uri] && HTTParty.normalize_base_uri(options[:base_uri])
@@ -173,6 +184,13 @@ module HTTParty
       @raw_request.body
     end
 
+    def _dump(_level)
+      opts = options.dup
+      opts.delete(:logger)
+      opts.delete(:parser) if opts[:parser] && opts[:parser].is_a?(Proc)
+      Marshal.dump([http_method, path, opts, last_response, @last_uri, @raw_request])
+    end
+
     private
 
     def http
@@ -204,22 +222,15 @@ module HTTParty
     end
 
     def setup_raw_request
-      @raw_request = http_method.new(request_uri(uri))
-      @raw_request.body_stream = options[:body_stream] if options[:body_stream]
-
       if options[:headers].respond_to?(:to_hash)
         headers_hash = options[:headers].to_hash
-
-        @raw_request.initialize_http_header(headers_hash)
-        # If the caller specified a header of 'Accept-Encoding', assume they want to
-        # deal with encoding of content. Disable the internal logic in Net:HTTP
-        # that handles encoding, if the platform supports it.
-        if @raw_request.respond_to?(:decode_content) && (headers_hash.key?('Accept-Encoding') || headers_hash.key?('accept-encoding'))
-          # Using the '[]=' sets decode_content to false
-          @raw_request['accept-encoding'] = @raw_request['accept-encoding']
-        end
+      else
+        headers_hash = nil
       end
 
+      @raw_request = http_method.new(request_uri(uri), headers_hash)
+      @raw_request.body_stream = options[:body_stream] if options[:body_stream]
+
       if options[:body]
         body = Body.new(
           options[:body],
@@ -234,6 +245,8 @@ module HTTParty
         @raw_request.body = body.call
       end
 
+      @raw_request.instance_variable_set(:@decode_content, decompress_content?)
+
       if options[:basic_auth] && send_authorization_header?
         @raw_request.basic_auth(username, password)
         @credentials_sent = true
@@ -245,6 +258,10 @@ module HTTParty
       !!options[:digest_auth]
     end
 
+    def decompress_content?
+      !options[:skip_decompression]
+    end
+
     def response_unauthorized?
       !!last_response && last_response.code == '401'
     end
@@ -268,7 +285,7 @@ module HTTParty
         query_string_parts << options[:query] unless options[:query].nil?
       end
 
-      query_string_parts.reject!(&:empty?) unless query_string_parts == [""]
+      query_string_parts.reject!(&:empty?) unless query_string_parts == ['']
       query_string_parts.size > 0 ? query_string_parts.join('&') : nil
     end
 
@@ -276,37 +293,55 @@ module HTTParty
       options[:assume_utf16_is_big_endian]
     end
 
-    def handle_response(body, &block)
+    def handle_response(raw_body, &block)
       if response_redirects?
-        options[:limit] -= 1
-        if options[:logger]
-          logger = HTTParty::Logger.build(options[:logger], options[:log_level], options[:log_format])
-          logger.format(self, last_response)
-        end
-        self.path = last_response['location']
-        self.redirect = true
-        if last_response.class == Net::HTTPSeeOther
-          unless options[:maintain_method_across_redirects] && options[:resend_on_redirect]
-            self.http_method = Net::HTTP::Get
-          end
-        elsif last_response.code != '307' && last_response.code != '308'
-          unless options[:maintain_method_across_redirects]
-            self.http_method = Net::HTTP::Get
+        handle_redirection(&block)
+      else
+        raw_body ||= last_response.body
+
+        body = decompress(raw_body, last_response['content-encoding']) unless raw_body.nil?
+
+        unless body.nil?
+          body = encode_text(body, last_response['content-type'])
+
+          if decompress_content?
+            last_response.delete('content-encoding')
+            raw_body = body
           end
         end
-        capture_cookies(last_response)
-        perform(&block)
-      else
-        body ||= last_response.body
-        body = body.nil? ? body : encode_text(body, last_response['content-type'])
-        Response.new(self, last_response, lambda { parse_response(body) }, body: body)
+
+        Response.new(self, last_response, lambda { parse_response(body) }, body: raw_body)
       end
     end
 
+    def handle_redirection(&block)
+      options[:limit] -= 1
+      if options[:logger]
+        logger = HTTParty::Logger.build(options[:logger], options[:log_level], options[:log_format])
+        logger.format(self, last_response)
+      end
+      self.path       = last_response['location']
+      self.redirect   = true
+      if last_response.class == Net::HTTPSeeOther
+        unless options[:maintain_method_across_redirects] && options[:resend_on_redirect]
+          self.http_method = Net::HTTP::Get
+        end
+      elsif last_response.code != '307' && last_response.code != '308'
+        unless options[:maintain_method_across_redirects]
+          self.http_method = Net::HTTP::Get
+        end
+      end
+      if http_method == Net::HTTP::Get
+        clear_body
+      end
+      capture_cookies(last_response)
+      perform(&block)
+    end
+
     def handle_host_redirection
       check_duplicate_location_header
       redirect_path = options[:uri_adapter].parse(last_response['location']).normalize
-      return if redirect_path.relative? || path.host == redirect_path.host
+      return if redirect_path.relative? || path.host == redirect_path.host || uri.host == redirect_path.host
       @changed_hosts = true
     end
 
@@ -334,6 +369,14 @@ module HTTParty
       parser.call(body, format)
     end
 
+    # Some Web Application Firewalls reject incoming GET requests that have a body
+    # if we redirect, and the resulting verb is GET then we will clear the body that
+    # may be left behind from the initiating request
+    def clear_body
+      options[:body] = nil
+      @raw_request.body = nil
+    end
+
     def capture_cookies(response)
       return unless response['Set-Cookie']
       cookies_hash = HTTParty::CookieHash.new
@@ -375,6 +418,10 @@ module HTTParty
       end
     end
 
+    def decompress(body, encoding)
+      Decompressor.new(body, encoding).decompress
+    end
+
     def encode_text(text, content_type)
       TextEncoder.new(
         text,

data/lib/httparty/response/headers.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'delegate'
 
 module HTTParty

data/lib/httparty/response.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module HTTParty
   class Response < Object
     def self.underscore(string)
@@ -49,14 +51,14 @@ module HTTParty
     end
 
     def inspect
-      inspect_id = ::Kernel::format "%x", (object_id * 2)
+      inspect_id = ::Kernel::format '%x', (object_id * 2)
       %(#<#{self.class}:0x#{inspect_id} parsed_response=#{parsed_response.inspect}, @response=#{response.inspect}, @headers=#{headers.inspect}>)
     end
 
     CODES_TO_OBJ = ::Net::HTTPResponse::CODE_CLASS_TO_OBJ.merge ::Net::HTTPResponse::CODE_TO_OBJ
 
     CODES_TO_OBJ.each do |response_code, klass|
-      name = klass.name.sub("Net::HTTP", '')
+      name = klass.name.sub('Net::HTTP', '')
       name = "#{underscore(name)}?".to_sym
 
       define_method(name) do
@@ -65,12 +67,12 @@ module HTTParty
     end
 
     # Support old multiple_choice? method from pre 2.0.0 era.
-    if ::RUBY_VERSION >= "2.0.0" && ::RUBY_PLATFORM != "java"
+    if ::RUBY_PLATFORM != 'java'
       alias_method :multiple_choice?, :multiple_choices?
     end
 
     # Support old status codes method from pre 2.6.0 era.
-    if ::RUBY_VERSION >= "2.6.0" && ::RUBY_PLATFORM != "java"
+    if ::RUBY_PLATFORM != 'java'
       alias_method :gateway_time_out?,                :gateway_timeout?
       alias_method :request_entity_too_large?,        :payload_too_large?
       alias_method :request_time_out?,                :request_timeout?
@@ -131,7 +133,7 @@ module HTTParty
     end
 
     def throw_exception
-      if @request.options[:raise_on] && @request.options[:raise_on].include?(code)
+      if @request.options[:raise_on].to_a.detect { |c| code.to_s.match(/#{c.to_s}/) }
         ::Kernel.raise ::HTTParty::ResponseError.new(@response), "Code #{code} - #{body}"
       end
     end

data/lib/httparty/response_fragment.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'delegate'
 
 module HTTParty

data/lib/httparty/text_encoder.rb

@@ -1,9 +1,11 @@
+# frozen_string_literal: true
+
 module HTTParty
   class TextEncoder
     attr_reader :text, :content_type, :assume_utf16_is_big_endian
 
     def initialize(text, assume_utf16_is_big_endian: true, content_type: nil)
-      @text = text.dup
+      @text = +text
       @content_type = content_type
       @assume_utf16_is_big_endian = assume_utf16_is_big_endian
     end
@@ -33,16 +35,16 @@ module HTTParty
     def encode_utf_16
       if text.bytesize >= 2
         if text.getbyte(0) == 0xFF && text.getbyte(1) == 0xFE
-          return text.force_encoding("UTF-16LE")
+          return text.force_encoding('UTF-16LE')
         elsif text.getbyte(0) == 0xFE && text.getbyte(1) == 0xFF
-          return text.force_encoding("UTF-16BE")
+          return text.force_encoding('UTF-16BE')
         end
       end
 
       if assume_utf16_is_big_endian # option
-        text.force_encoding("UTF-16BE")
+        text.force_encoding('UTF-16BE')
       else
-        text.force_encoding("UTF-16LE")
+        text.force_encoding('UTF-16LE')
       end
     end
 

data/lib/httparty/utils.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module HTTParty
   module Utils
     def self.stringify_keys(hash)

data/lib/httparty/version.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module HTTParty
-  VERSION = "0.17.3"
+  VERSION = '0.22.0'
 end

data/lib/httparty.rb

@@ -1,12 +1,8 @@
+# frozen_string_literal: true
+
 require 'pathname'
 require 'net/http'
-require 'net/https'
 require 'uri'
-require 'zlib'
-require 'multi_xml'
-require 'mime/types'
-require 'json'
-require 'csv'
 
 require 'httparty/module_inheritable_attributes'
 require 'httparty/cookie_hash'
@@ -16,6 +12,7 @@ require 'httparty/connection_adapter'
 require 'httparty/logger/logger'
 require 'httparty/request/body'
 require 'httparty/response_fragment'
+require 'httparty/decompressor'
 require 'httparty/text_encoder'
 require 'httparty/headers_processor'
 
@@ -26,8 +23,8 @@ module HTTParty
     base.send :include, ModuleInheritableAttributes
     base.send(:mattr_inheritable, :default_options)
     base.send(:mattr_inheritable, :default_cookies)
-    base.instance_variable_set("@default_options", {})
-    base.instance_variable_set("@default_cookies", CookieHash.new)
+    base.instance_variable_set(:@default_options, {})
+    base.instance_variable_set(:@default_cookies, CookieHash.new)
   end
 
   # == Common Request Options
@@ -41,11 +38,11 @@ module HTTParty
   # [:+limit+:] Maximum number of redirects to follow. Takes precedences over :+no_follow+.
   # [:+query+:] Query string, or an object that responds to #to_hash representing it. Normalized according to the same rules as :+body+. If you specify this on a POST, you must use an object which responds to #to_hash. See also HTTParty::ClassMethods.default_params.
   # [:+timeout+:] Timeout for opening connection and reading data.
-  # [:+local_host:] Local address to bind to before connecting.
-  # [:+local_port:] Local port to bind to before connecting.
-  # [:+body_stream:] Allow streaming to a REST server to specify a body_stream.
-  # [:+stream_body:] Allow for streaming large files without loading them into memory.
-  # [:+multipart:] Force content-type to be multipart
+  # [:+local_host+:] Local address to bind to before connecting.
+  # [:+local_port+:] Local port to bind to before connecting.
+  # [:+body_stream+:] Allow streaming to a REST server to specify a body_stream.
+  # [:+stream_body+:] Allow for streaming large files without loading them into memory.
+  # [:+multipart+:] Force content-type to be multipart
   #
   # There are also another set of options with names corresponding to various class methods. The methods in question are those that let you set a class-wide default, and the options override the defaults on a request-by-request basis. Those options are:
   # * :+base_uri+: see HTTParty::ClassMethods.base_uri.
@@ -81,7 +78,7 @@ module HTTParty
     #
     #   class Foo
     #     include HTTParty
-    #     raise_on [404, 500]
+    #     raise_on [404, 500, '5[0-9]*']
     #   end
     def raise_on(codes = [])
       default_options[:raise_on] = *codes
@@ -399,6 +396,22 @@ module HTTParty
       default_options[:ssl_version] = version
     end
 
+    # Deactivate automatic decompression of the response body.
+    # This will require you to explicitly handle body decompression
+    # by inspecting the Content-Encoding response header.
+    #
+    # Refer to docs/README.md "HTTP Compression" section for
+    # further details.
+    #
+    # @example
+    #   class Foo
+    #     include HTTParty
+    #     skip_decompression
+    #   end
+    def skip_decompression(value = true)
+      default_options[:skip_decompression] = !!value
+    end
+
     # Allows setting of SSL ciphers to use.  This only works in Ruby 1.9+.
     # You can get a list of valid specific ciphers from OpenSSL::Cipher.ciphers.
     # You also can specify a cipher suite here, listed here at openssl.org:
@@ -573,6 +586,13 @@ module HTTParty
       perform_request Net::HTTP::Unlock, path, options, &block
     end
 
+    def build_request(http_method, path, options = {})
+      options = ModuleInheritableAttributes.hash_deep_dup(default_options).merge(options)
+      HeadersProcessor.new(headers, options).call
+      process_cookies(options)
+      Request.new(http_method, path, options)
+    end
+
     attr_reader :default_options
 
     private
@@ -588,16 +608,13 @@ module HTTParty
     end
 
     def perform_request(http_method, path, options, &block) #:nodoc:
-      options = ModuleInheritableAttributes.hash_deep_dup(default_options).merge(options)
-      HeadersProcessor.new(headers, options).call
-      process_cookies(options)
-      Request.new(http_method, path, options).perform(&block)
+      build_request(http_method, path, options).perform(&block)
     end
 
     def process_cookies(options) #:nodoc:
       return unless options[:cookies] || default_cookies.any?
       options[:headers] ||= headers.dup
-      options[:headers]["cookie"] = cookies.merge(options.delete(:cookies) || {}).to_cookie_string
+      options[:headers]['cookie'] = cookies.merge(options.delete(:cookies) || {}).to_cookie_string
     end
 
     def validate_format
@@ -658,6 +675,10 @@ module HTTParty
   def self.options(*args, &block)
     Basement.options(*args, &block)
   end
+
+  def self.build_request(*args, &block)
+    Basement.build_request(*args, &block)
+  end
 end
 
 require 'httparty/hash_conversions'

data/script/release

@@ -18,9 +18,9 @@ gem_name=httparty
 rm -rf $gem_name-*.gem
 gem build -q $gem_name.gemspec
 
-# Make sure we're on the master branch.
-(git branch | grep -q '* master') || {
-  echo "Only release from the master branch."
+# Make sure we're on the main branch.
+(git branch | grep -q '* main') || {
+  echo "Only release from the main branch."
   exit 1
 }
 
@@ -39,4 +39,4 @@ git fetch -t origin
 
 # Tag it and bag it.
 gem push $gem_name-*.gem && git tag "$tag" &&
-  git push origin master && git push origin "$tag"
+  git push origin main && git push origin "$tag"