httparty 0.16.2 → 0.20.0

data/lib/httparty/parser.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module HTTParty
   # The default parser used by HTTParty, supports xml, json, html, csv and
   # plain text.
@@ -101,7 +103,7 @@ module HTTParty
     # @return [nil] when the response body is nil, an empty string, spaces only or "null"
     def parse
       return nil if body.nil?
-      return nil if body == "null"
+      return nil if body == 'null'
       return nil if body.valid_encoding? && body.strip.empty?
       if body.valid_encoding? && body.encoding == Encoding::UTF_8
         @body = body.gsub(/\A#{UTF8_BOM}/, '')
@@ -119,7 +121,7 @@ module HTTParty
       MultiXml.parse(body)
     end
 
-    UTF8_BOM = "\xEF\xBB\xBF".freeze
+    UTF8_BOM = "\xEF\xBB\xBF"
 
     def json
       JSON.parse(body, :quirks_mode => true, :allow_nan => true)
@@ -142,9 +144,11 @@ module HTTParty
     end
 
     def parse_supported_format
-      send(format)
-    rescue NoMethodError => e
-      raise NotImplementedError, "#{self.class.name} has not implemented a parsing method for the #{format.inspect} format.", e.backtrace
+      if respond_to?(format, true)
+        send(format)
+      else
+        raise NotImplementedError, "#{self.class.name} has not implemented a parsing method for the #{format.inspect} format."
+      end
     end
   end
 end

data/lib/httparty/request/body.rb

@@ -1,11 +1,17 @@
+# frozen_string_literal: true
+
 require_relative 'multipart_boundary'
 
 module HTTParty
   class Request
     class Body
-      def initialize(params, query_string_normalizer: nil)
+      NEWLINE = "\r\n"
+      private_constant :NEWLINE
+
+      def initialize(params, query_string_normalizer: nil, force_multipart: false)
         @params = params
         @query_string_normalizer = query_string_normalizer
+        @force_multipart = force_multipart
       end
 
       def call
@@ -21,7 +27,7 @@ module HTTParty
       end
 
       def multipart?
-        params.respond_to?(:to_hash) && has_file?(params.to_hash)
+        params.respond_to?(:to_hash) && (force_multipart || has_file?(params))
       end
 
       private
@@ -29,40 +35,34 @@ module HTTParty
       def generate_multipart
         normalized_params = params.flat_map { |key, value| HashConversions.normalize_keys(key, value) }
 
-        multipart = normalized_params.inject('') do |memo, (key, value)|
-          memo += "--#{boundary}\r\n"
-          memo += %(Content-Disposition: form-data; name="#{key}")
+        multipart = normalized_params.inject(''.dup) do |memo, (key, value)|
+          memo << "--#{boundary}#{NEWLINE}"
+          memo << %(Content-Disposition: form-data; name="#{key}")
           # value.path is used to support ActionDispatch::Http::UploadedFile
           # https://github.com/jnunemaker/httparty/pull/585
-          memo += %(; filename="#{File.basename(value.path)}") if file?(value)
-          memo += "\r\n"
-          memo += "Content-Type: application/octet-stream\r\n" if file?(value)
-          memo += "\r\n"
-          memo += file?(value) ? value.read : value.to_s
-          memo += "\r\n"
+          memo << %(; filename="#{file_name(value)}") if file?(value)
+          memo << NEWLINE
+          memo << "Content-Type: #{content_type(value)}#{NEWLINE}" if file?(value)
+          memo << NEWLINE
+          memo << content_body(value)
+          memo << NEWLINE
         end
 
-        multipart += "--#{boundary}--\r\n"
+        multipart << "--#{boundary}--#{NEWLINE}"
       end
 
-      def has_file?(hash)
-        hash.detect do |key, value|
-          if value.respond_to?(:to_hash) || includes_hash?(value)
-            has_file?(value)
-          elsif value.respond_to?(:to_ary)
-            value.any? { |e| file?(e) }
-          else
-            file?(value)
-          end
+      def has_file?(value)
+        if value.respond_to?(:to_hash)
+          value.to_hash.any? { |_, v| has_file?(v) }
+        elsif value.respond_to?(:to_ary)
+          value.to_ary.any? { |v| has_file?(v) }
+        else
+          file?(value)
         end
       end
 
       def file?(object)
-        object.respond_to?(:path) && object.respond_to?(:read) # add memoization
-      end
-
-      def includes_hash?(object)
-        object.respond_to?(:to_ary) && object.any? { |e| e.respond_to?(:hash) }
+        object.respond_to?(:path) && object.respond_to?(:read)
       end
 
       def normalize_query(query)
@@ -73,7 +73,26 @@ module HTTParty
         end
       end
 
-      attr_reader :params, :query_string_normalizer
+      def content_body(object)
+        if file?(object)
+          object = (file = object).read
+          file.rewind if file.respond_to?(:rewind)
+        end
+
+        object.to_s
+      end
+
+      def content_type(object)
+        return object.content_type if object.respond_to?(:content_type)
+        mime = MIME::Types.type_for(object.path)
+        mime.empty? ? 'application/octet-stream' : mime[0].content_type
+      end
+
+      def file_name(object)
+        object.respond_to?(:original_filename) ? object.original_filename : File.basename(object.path)
+      end
+
+      attr_reader :params, :query_string_normalizer, :force_multipart
     end
   end
 end

data/lib/httparty/request/multipart_boundary.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'securerandom'
 
 module HTTParty

data/lib/httparty/request.rb

@@ -1,5 +1,6 @@
+# frozen_string_literal: true
+
 require 'erb'
-require 'httparty/request/body'
 
 module HTTParty
   class Request #:nodoc:
@@ -14,6 +15,8 @@ module HTTParty
       Net::HTTP::Move,
       Net::HTTP::Copy,
       Net::HTTP::Mkcol,
+      Net::HTTP::Lock,
+      Net::HTTP::Unlock,
     ]
 
     SupportedURISchemes  = ['http', 'https', 'webcal', nil]
@@ -43,6 +46,11 @@ module HTTParty
       end.flatten.join('&')
     end
 
+    def self._load(data)
+      http_method, path, options = Marshal.load(data)
+      new(http_method, path, options)
+    end
+
     attr_accessor :http_method, :options, :last_response, :redirect, :last_uri
     attr_reader :path
 
@@ -70,7 +78,7 @@ module HTTParty
       @path = if uri.is_a?(uri_adapter)
         uri
       elsif String.try_convert(uri)
-        uri_adapter.parse uri
+        uri_adapter.parse(uri).normalize
       else
         raise ArgumentError,
           "bad argument (expected #{uri_adapter} object or URI string)"
@@ -86,17 +94,17 @@ module HTTParty
     end
 
     def uri
-      if redirect && path.relative? && path.path[0] != "/"
-        last_uri_host = @last_uri.path.gsub(/[^\/]+$/, "")
+      if redirect && path.relative? && path.path[0] != '/'
+        last_uri_host = @last_uri.path.gsub(/[^\/]+$/, '')
 
-        path.path = "/#{path.path}" if last_uri_host[-1] != "/"
-        path.path = last_uri_host + path.path
+        path.path = "/#{path.path}" if last_uri_host[-1] != '/'
+        path.path = "#{last_uri_host}#{path.path}"
       end
 
       if path.relative? && path.host
-        new_uri = options[:uri_adapter].parse("#{@last_uri.scheme}:#{path}")
+        new_uri = options[:uri_adapter].parse("#{@last_uri.scheme}:#{path}").normalize
       elsif path.relative?
-        new_uri = options[:uri_adapter].parse("#{base_uri}#{path}")
+        new_uri = options[:uri_adapter].parse("#{base_uri}#{path}").normalize
       else
         new_uri = path.clone
       end
@@ -116,7 +124,7 @@ module HTTParty
     def base_uri
       if redirect
         base_uri = "#{@last_uri.scheme}://#{@last_uri.host}"
-        base_uri += ":#{@last_uri.port}" if @last_uri.port != 80
+        base_uri = "#{base_uri}:#{@last_uri.port}" if @last_uri.port != 80
         base_uri
       else
         options[:base_uri] && HTTParty.normalize_base_uri(options[:base_uri])
@@ -139,21 +147,22 @@ module HTTParty
       validate
       setup_raw_request
       chunked_body = nil
+      current_http = http
 
-      self.last_response = http.request(@raw_request) do |http_response|
+      self.last_response = current_http.request(@raw_request) do |http_response|
         if block
           chunks = []
 
           http_response.read_body do |fragment|
-            chunks << fragment unless options[:stream_body]
-            block.call(fragment)
+            encoded_fragment = encode_text(fragment, http_response['content-type'])
+            chunks << encoded_fragment if !options[:stream_body]
+            block.call ResponseFragment.new(encoded_fragment, http_response, current_http)
           end
 
           chunked_body = chunks.join
         end
       end
 
-
       handle_host_redirection if response_redirects?
       result = handle_unauthorized
       result ||= handle_response(chunked_body, &block)
@@ -171,6 +180,13 @@ module HTTParty
       @raw_request.body
     end
 
+    def _dump(_level)
+      opts = options.dup
+      opts.delete(:logger)
+      opts.delete(:parser) if opts[:parser] && opts[:parser].is_a?(Proc)
+      Marshal.dump([http_method, path, opts])
+    end
+
     private
 
     def http
@@ -202,24 +218,22 @@ module HTTParty
     end
 
     def setup_raw_request
-      @raw_request = http_method.new(request_uri(uri))
-      @raw_request.body_stream = options[:body_stream] if options[:body_stream]
-
       if options[:headers].respond_to?(:to_hash)
         headers_hash = options[:headers].to_hash
-
-        @raw_request.initialize_http_header(headers_hash)
-        # If the caller specified a header of 'Accept-Encoding', assume they want to
-        # deal with encoding of content. Disable the internal logic in Net:HTTP
-        # that handles encoding, if the platform supports it.
-        if @raw_request.respond_to?(:decode_content) && (headers_hash.key?('Accept-Encoding') || headers_hash.key?('accept-encoding'))
-          # Using the '[]=' sets decode_content to false
-          @raw_request['accept-encoding'] = @raw_request['accept-encoding']
-        end
+      else
+        headers_hash = nil
       end
 
+      @raw_request = http_method.new(request_uri(uri), headers_hash)
+      @raw_request.body_stream = options[:body_stream] if options[:body_stream]
+
       if options[:body]
-        body = Body.new(options[:body], query_string_normalizer: query_string_normalizer)
+        body = Body.new(
+          options[:body],
+          query_string_normalizer: query_string_normalizer,
+          force_multipart: options[:multipart]
+        )
+
         if body.multipart?
           content_type = "multipart/form-data; boundary=#{body.boundary}"
           @raw_request['Content-Type'] = content_type
@@ -227,6 +241,8 @@ module HTTParty
         @raw_request.body = body.call
       end
 
+      @raw_request.instance_variable_set(:@decode_content, decompress_content?)
+
       if options[:basic_auth] && send_authorization_header?
         @raw_request.basic_auth(username, password)
         @credentials_sent = true
@@ -238,6 +254,10 @@ module HTTParty
       !!options[:digest_auth]
     end
 
+    def decompress_content?
+      !options[:skip_decompression]
+    end
+
     def response_unauthorized?
       !!last_response && last_response.code == '401'
     end
@@ -261,79 +281,15 @@ module HTTParty
         query_string_parts << options[:query] unless options[:query].nil?
       end
 
-      query_string_parts.reject!(&:empty?) unless query_string_parts == [""]
+      query_string_parts.reject!(&:empty?) unless query_string_parts == ['']
       query_string_parts.size > 0 ? query_string_parts.join('&') : nil
     end
 
-    def get_charset
-      content_type = last_response["content-type"]
-      if content_type.nil?
-        return nil
-      end
-
-      if content_type =~ /;\s*charset\s*=\s*([^=,;"\s]+)/i
-        return $1
-      end
-
-      if content_type =~ /;\s*charset\s*=\s*"((\\.|[^\\"])+)"/i
-        return $1.gsub(/\\(.)/, '\1')
-      end
-
-      nil
-    end
-
-    def encode_with_ruby_encoding(body, charset)
-      # NOTE: This will raise an argument error if the
-      # charset does not exist
-      encoding = Encoding.find(charset)
-      body.force_encoding(encoding.to_s)
-    rescue ArgumentError
-      body
-    end
-
     def assume_utf16_is_big_endian
       options[:assume_utf16_is_big_endian]
     end
 
-    def encode_utf_16(body)
-      if body.bytesize >= 2
-        if body.getbyte(0) == 0xFF && body.getbyte(1) == 0xFE
-          return body.force_encoding("UTF-16LE")
-        elsif body.getbyte(0) == 0xFE && body.getbyte(1) == 0xFF
-          return body.force_encoding("UTF-16BE")
-        end
-      end
-
-      if assume_utf16_is_big_endian
-        body.force_encoding("UTF-16BE")
-      else
-        body.force_encoding("UTF-16LE")
-      end
-    end
-
-    def _encode_body(body)
-      charset = get_charset
-
-      if charset.nil?
-        return body
-      end
-
-      if "utf-16".casecmp(charset) == 0
-        encode_utf_16(body)
-      else
-        encode_with_ruby_encoding(body, charset)
-      end
-    end
-
-    def encode_body(body)
-      if "".respond_to?(:encoding)
-        _encode_body(body)
-      else
-        body
-      end
-    end
-
-    def handle_response(body, &block)
+    def handle_response(raw_body, &block)
       if response_redirects?
         options[:limit] -= 1
         if options[:logger]
@@ -354,15 +310,26 @@ module HTTParty
         capture_cookies(last_response)
         perform(&block)
       else
-        body ||= last_response.body
-        body = body.nil? ? body : encode_body(body)
-        Response.new(self, last_response, lambda { parse_response(body) }, body: body)
+        raw_body ||= last_response.body
+
+        body = decompress(raw_body, last_response['content-encoding']) unless raw_body.nil?
+
+        unless body.nil?
+          body = encode_text(body, last_response['content-type'])
+
+          if decompress_content?
+            last_response.delete('content-encoding')
+            raw_body = body
+          end
+        end
+
+        Response.new(self, last_response, lambda { parse_response(body) }, body: raw_body)
       end
     end
 
     def handle_host_redirection
       check_duplicate_location_header
-      redirect_path = options[:uri_adapter].parse last_response['location']
+      redirect_path = options[:uri_adapter].parse(last_response['location']).normalize
       return if redirect_path.relative? || path.host == redirect_path.host
       @changed_hosts = true
     end
@@ -431,5 +398,17 @@ module HTTParty
         @credentials_sent = true
       end
     end
+
+    def decompress(body, encoding)
+      Decompressor.new(body, encoding).decompress
+    end
+
+    def encode_text(text, content_type)
+      TextEncoder.new(
+        text,
+        content_type: content_type,
+        assume_utf16_is_big_endian: assume_utf16_is_big_endian
+      ).call
+    end
   end
 end

data/lib/httparty/response/headers.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'delegate'
 
 module HTTParty
@@ -22,10 +24,10 @@ module HTTParty
       end
 
       def ==(other)
-        if other.is_a?(::Net::HTTPHeader) 
+        if other.is_a?(::Net::HTTPHeader)
           @header == other.instance_variable_get(:@header)
         elsif other.is_a?(Hash)
-          @header == other || @header == Headers.new(other).instance_variable_get(:@header)           
+          @header == other || @header == Headers.new(other).instance_variable_get(:@header)
         end
       end
     end

data/lib/httparty/response.rb

@@ -1,9 +1,17 @@
+# frozen_string_literal: true
+
 module HTTParty
   class Response < Object
     def self.underscore(string)
       string.gsub(/([A-Z]+)([A-Z][a-z])/, '\1_\2').gsub(/([a-z])([A-Z])/, '\1_\2').downcase
     end
 
+    def self._load(data)
+      req, resp, parsed_resp, resp_body = Marshal.load(data)
+
+      new(req, resp, -> { parsed_resp }, body: resp_body)
+    end
+
     attr_reader :request, :response, :body, :headers
 
     def initialize(request, response, parsed_block, options = {})
@@ -14,7 +22,11 @@ module HTTParty
       @headers      = Headers.new(response.to_hash)
 
       if request.options[:logger]
-        logger = ::HTTParty::Logger.build(request.options[:logger], request.options[:log_level], request.options[:log_format])
+        logger = ::HTTParty::Logger.build(
+          request.options[:logger],
+          request.options[:log_level],
+          request.options[:log_format]
+        )
         logger.format(request, self)
       end
 
@@ -29,20 +41,24 @@ module HTTParty
       response.code.to_i
     end
 
+    def http_version
+      response.http_version
+    end
+
     def tap
       yield self
       self
     end
 
     def inspect
-      inspect_id = ::Kernel::format "%x", (object_id * 2)
+      inspect_id = ::Kernel::format '%x', (object_id * 2)
       %(#<#{self.class}:0x#{inspect_id} parsed_response=#{parsed_response.inspect}, @response=#{response.inspect}, @headers=#{headers.inspect}>)
     end
 
     CODES_TO_OBJ = ::Net::HTTPResponse::CODE_CLASS_TO_OBJ.merge ::Net::HTTPResponse::CODE_TO_OBJ
 
     CODES_TO_OBJ.each do |response_code, klass|
-      name = klass.name.sub("Net::HTTP", '')
+      name = klass.name.sub('Net::HTTP', '')
       name = "#{underscore(name)}?".to_sym
 
       define_method(name) do
@@ -51,18 +67,28 @@ module HTTParty
     end
 
     # Support old multiple_choice? method from pre 2.0.0 era.
-    if ::RUBY_VERSION >= "2.0.0" && ::RUBY_PLATFORM != "java"
+    if ::RUBY_VERSION >= '2.0.0' && ::RUBY_PLATFORM != 'java'
       alias_method :multiple_choice?, :multiple_choices?
     end
 
+    # Support old status codes method from pre 2.6.0 era.
+    if ::RUBY_VERSION >= '2.6.0' && ::RUBY_PLATFORM != 'java'
+      alias_method :gateway_time_out?,                :gateway_timeout?
+      alias_method :request_entity_too_large?,        :payload_too_large?
+      alias_method :request_time_out?,                :request_timeout?
+      alias_method :request_uri_too_long?,            :uri_too_long?
+      alias_method :requested_range_not_satisfiable?, :range_not_satisfiable?
+    end
+
     def nil?
+      warn_about_nil_deprecation
       response.nil? || response.body.nil? || response.body.empty?
     end
 
-    def to_s      
+    def to_s
       if !response.nil? && !response.body.nil? && response.body.respond_to?(:to_s)
         response.body.to_s
-      else 
+      else
         inspect
       end
     end
@@ -80,7 +106,7 @@ module HTTParty
         parsed_response.display(port)
       elsif !response.nil? && !response.body.nil? && response.body.respond_to?(:display)
         response.body.display(port)
-      else 
+      else
         port.write(inspect)
       end
     end
@@ -89,7 +115,11 @@ module HTTParty
       return true if super
       parsed_response.respond_to?(name) || response.respond_to?(name)
     end
-    
+
+    def _dump(_level)
+      Marshal.dump([request, response, parsed_response, body])
+    end
+
     protected
 
     def method_missing(name, *args, &block)
@@ -107,6 +137,19 @@ module HTTParty
         ::Kernel.raise ::HTTParty::ResponseError.new(@response), "Code #{code} - #{body}"
       end
     end
+
+    private
+
+    def warn_about_nil_deprecation
+      trace_line = caller.reject { |line| line.include?('httparty') }.first
+      warning = "[DEPRECATION] HTTParty will no longer override `response#nil?`. " \
+        "This functionality will be removed in future versions. " \
+        "Please, add explicit check `response.body.nil? || response.body.empty?`. " \
+        "For more info refer to: https://github.com/jnunemaker/httparty/issues/568\n" \
+        "#{trace_line}"
+
+      warn(warning)
+    end
   end
 end
 

data/lib/httparty/response_fragment.rb

@@ -0,0 +1,21 @@
+# frozen_string_literal: true
+
+require 'delegate'
+
+module HTTParty
+  # Allow access to http_response and code by delegation on fragment
+  class ResponseFragment < SimpleDelegator
+    attr_reader :http_response, :connection
+
+    def code
+      @http_response.code.to_i
+    end
+
+    def initialize(fragment, http_response, connection)
+      @fragment = fragment
+      @http_response = http_response
+      @connection = connection
+      super fragment
+    end
+  end
+end

data/lib/httparty/text_encoder.rb

@@ -0,0 +1,72 @@
+# frozen_string_literal: true
+
+module HTTParty
+  class TextEncoder
+    attr_reader :text, :content_type, :assume_utf16_is_big_endian
+
+    def initialize(text, assume_utf16_is_big_endian: true, content_type: nil)
+      @text = text.dup
+      @content_type = content_type
+      @assume_utf16_is_big_endian = assume_utf16_is_big_endian
+    end
+
+    def call
+      if can_encode?
+        encoded_text
+      else
+        text
+      end
+    end
+
+    private
+
+    def can_encode?
+      ''.respond_to?(:encoding) && charset
+    end
+
+    def encoded_text
+      if 'utf-16'.casecmp(charset) == 0
+        encode_utf_16
+      else
+        encode_with_ruby_encoding
+      end
+    end
+
+    def encode_utf_16
+      if text.bytesize >= 2
+        if text.getbyte(0) == 0xFF && text.getbyte(1) == 0xFE
+          return text.force_encoding('UTF-16LE')
+        elsif text.getbyte(0) == 0xFE && text.getbyte(1) == 0xFF
+          return text.force_encoding('UTF-16BE')
+        end
+      end
+
+      if assume_utf16_is_big_endian # option
+        text.force_encoding('UTF-16BE')
+      else
+        text.force_encoding('UTF-16LE')
+      end
+    end
+
+    def encode_with_ruby_encoding
+      # NOTE: This will raise an argument error if the
+      # charset does not exist
+      encoding = Encoding.find(charset)
+      text.force_encoding(encoding.to_s)
+    rescue ArgumentError
+      text
+    end
+
+    def charset
+      return nil if content_type.nil?
+
+      if (matchdata = content_type.match(/;\s*charset\s*=\s*([^=,;"\s]+)/i))
+        return matchdata.captures.first
+      end
+
+      if (matchdata = content_type.match(/;\s*charset\s*=\s*"((\\.|[^\\"])+)"/i))
+        return matchdata.captures.first.gsub(/\\(.)/, '\1')
+      end
+    end
+  end
+end