httparty 0.15.6 → 0.21.0

data/lib/httparty/net_digest_auth.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'digest/md5'
 require 'net/http'
 
@@ -14,11 +16,11 @@ module Net
 
       authenticator.authorization_header.each do |v|
         add_field('Authorization', v)
-      end 
+      end
 
       authenticator.cookie_header.each do |v|
         add_field('Cookie', v)
-      end      
+      end
     end
 
     class DigestAuthenticator
@@ -44,12 +46,9 @@ module Net
         header << %(algorithm="#{@response['algorithm']}") if algorithm_present?
 
         if qop_present?
-          fields = [
-            %(cnonce="#{@cnonce}"),
-            %(qop="#{@response['qop']}"),
-            "nc=00000001"
-          ]
-          fields.each { |field| header << field }
+          header << %(cnonce="#{@cnonce}")
+          header << %(qop="#{@response['qop']}")
+          header << 'nc=00000001'
         end
 
         header << %(opaque="#{@response['opaque']}") if opaque_present?
@@ -64,7 +63,8 @@ module Net
 
       def parse(response_header)
         header = response_header['www-authenticate']
-                 .gsub(/qop=(auth(?:-int)?)/, 'qop="\\1"')
+
+        header = header.gsub(/qop=(auth(?:-int)?)/, 'qop="\\1"')
 
         header =~ /Digest (.*)/
         params = {}
@@ -97,13 +97,13 @@ module Net
       end
 
       def random
-        format "%x", (Time.now.to_i + rand(65535))
+        format '%x', (Time.now.to_i + rand(65535))
       end
 
       def request_digest
         a = [md5(a1), @response['nonce'], md5(a2)]
-        a.insert(2, "00000001", @cnonce, @response['qop']) if qop_present?
-        md5(a.join(":"))
+        a.insert(2, '00000001', @cnonce, @response['qop']) if qop_present?
+        md5(a.join(':'))
       end
 
       def md5(str)
@@ -113,11 +113,11 @@ module Net
       def algorithm_present?
         @response.key?('algorithm') && !@response['algorithm'].empty?
       end
-      
+
       def use_md5_sess?
         algorithm_present? && @response['algorithm'] == 'MD5-sess'
       end
-      
+
       def a1
         a1_user_realm_pwd =  [@username, @response['realm'], @password].join(':')
         if use_md5_sess?
@@ -128,7 +128,7 @@ module Net
       end
 
       def a2
-        [@method, @path].join(":")
+        [@method, @path].join(':')
       end
     end
   end

data/lib/httparty/parser.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module HTTParty
   # The default parser used by HTTParty, supports xml, json, html, csv and
   # plain text.
@@ -38,16 +40,18 @@ module HTTParty
   # @abstract Read the Custom Parsers section for more information.
   class Parser
     SupportedFormats = {
-      'text/xml'               => :xml,
-      'application/xml'        => :xml,
-      'application/json'       => :json,
-      'text/json'              => :json,
-      'application/javascript' => :plain,
-      'text/javascript'        => :plain,
-      'text/html'              => :html,
-      'text/plain'             => :plain,
-      'text/csv'               => :csv,
-      'application/csv'        => :csv,
+      'text/xml'                    => :xml,
+      'application/xml'             => :xml,
+      'application/json'            => :json,
+      'application/vnd.api+json'    => :json,
+      'application/hal+json'        => :json,
+      'text/json'                   => :json,
+      'application/javascript'      => :plain,
+      'text/javascript'             => :plain,
+      'text/html'                   => :html,
+      'text/plain'                  => :plain,
+      'text/csv'                    => :csv,
+      'application/csv'             => :csv,
       'text/comma-separated-values' => :csv
     }
 
@@ -99,7 +103,7 @@ module HTTParty
     # @return [nil] when the response body is nil, an empty string, spaces only or "null"
     def parse
       return nil if body.nil?
-      return nil if body == "null"
+      return nil if body == 'null'
       return nil if body.valid_encoding? && body.strip.empty?
       if body.valid_encoding? && body.encoding == Encoding::UTF_8
         @body = body.gsub(/\A#{UTF8_BOM}/, '')
@@ -117,7 +121,7 @@ module HTTParty
       MultiXml.parse(body)
     end
 
-    UTF8_BOM = "\xEF\xBB\xBF".freeze
+    UTF8_BOM = "\xEF\xBB\xBF"
 
     def json
       JSON.parse(body, :quirks_mode => true, :allow_nan => true)
@@ -140,9 +144,11 @@ module HTTParty
     end
 
     def parse_supported_format
-      send(format)
-    rescue NoMethodError => e
-      raise NotImplementedError, "#{self.class.name} has not implemented a parsing method for the #{format.inspect} format.", e.backtrace
+      if respond_to?(format, true)
+        send(format)
+      else
+        raise NotImplementedError, "#{self.class.name} has not implemented a parsing method for the #{format.inspect} format."
+      end
     end
   end
 end

data/lib/httparty/request/body.rb

@@ -0,0 +1,105 @@
+# frozen_string_literal: true
+
+require_relative 'multipart_boundary'
+
+module HTTParty
+  class Request
+    class Body
+      NEWLINE = "\r\n"
+      private_constant :NEWLINE
+
+      def initialize(params, query_string_normalizer: nil, force_multipart: false)
+        @params = params
+        @query_string_normalizer = query_string_normalizer
+        @force_multipart = force_multipart
+      end
+
+      def call
+        if params.respond_to?(:to_hash)
+          multipart? ? generate_multipart : normalize_query(params)
+        else
+          params
+        end
+      end
+
+      def boundary
+        @boundary ||= MultipartBoundary.generate
+      end
+
+      def multipart?
+        params.respond_to?(:to_hash) && (force_multipart || has_file?(params))
+      end
+
+      private
+
+      # https://html.spec.whatwg.org/#multipart-form-data
+      MULTIPART_FORM_DATA_REPLACEMENT_TABLE = {
+        '"'  => '%22',
+        "\r" => '%0D',
+        "\n" => '%0A'
+      }.freeze
+
+      def generate_multipart
+        normalized_params = params.flat_map { |key, value| HashConversions.normalize_keys(key, value) }
+
+        multipart = normalized_params.inject(''.dup) do |memo, (key, value)|
+          memo << "--#{boundary}#{NEWLINE}"
+          memo << %(Content-Disposition: form-data; name="#{key}")
+          # value.path is used to support ActionDispatch::Http::UploadedFile
+          # https://github.com/jnunemaker/httparty/pull/585
+          memo << %(; filename="#{file_name(value).gsub(/["\r\n]/, MULTIPART_FORM_DATA_REPLACEMENT_TABLE)}") if file?(value)
+          memo << NEWLINE
+          memo << "Content-Type: #{content_type(value)}#{NEWLINE}" if file?(value)
+          memo << NEWLINE
+          memo << content_body(value)
+          memo << NEWLINE
+        end
+
+        multipart << "--#{boundary}--#{NEWLINE}"
+      end
+
+      def has_file?(value)
+        if value.respond_to?(:to_hash)
+          value.to_hash.any? { |_, v| has_file?(v) }
+        elsif value.respond_to?(:to_ary)
+          value.to_ary.any? { |v| has_file?(v) }
+        else
+          file?(value)
+        end
+      end
+
+      def file?(object)
+        object.respond_to?(:path) && object.respond_to?(:read)
+      end
+
+      def normalize_query(query)
+        if query_string_normalizer
+          query_string_normalizer.call(query)
+        else
+          HashConversions.to_params(query)
+        end
+      end
+
+      def content_body(object)
+        if file?(object)
+          object = (file = object).read
+          file.rewind if file.respond_to?(:rewind)
+        end
+
+        object.to_s
+      end
+
+      def content_type(object)
+        return object.content_type if object.respond_to?(:content_type)
+        mime = MiniMime.lookup_by_filename(object.path)
+        mime ? mime.content_type : 'application/octet-stream'
+      end
+
+      def file_name(object)
+        object.respond_to?(:original_filename) ? object.original_filename : File.basename(object.path)
+      end
+
+      attr_reader :params, :query_string_normalizer, :force_multipart
+    end
+  end
+end

data/lib/httparty/request/multipart_boundary.rb

@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+require 'securerandom'
+
+module HTTParty
+  class Request
+    class MultipartBoundary
+      def self.generate
+        "------------------------#{SecureRandom.urlsafe_base64(12)}"
+      end
+    end
+  end
+end

data/lib/httparty/request.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'erb'
 
 module HTTParty
@@ -13,6 +15,8 @@ module HTTParty
       Net::HTTP::Move,
       Net::HTTP::Copy,
       Net::HTTP::Mkcol,
+      Net::HTTP::Lock,
+      Net::HTTP::Unlock,
     ]
 
     SupportedURISchemes  = ['http', 'https', 'webcal', nil]
@@ -42,6 +46,15 @@ module HTTParty
       end.flatten.join('&')
     end
 
+    def self._load(data)
+      http_method, path, options, last_response, last_uri, raw_request = Marshal.load(data)
+      instance = new(http_method, path, options)
+      instance.last_response = last_response
+      instance.last_uri = last_uri
+      instance.instance_variable_set("@raw_request", raw_request)
+      instance
+    end
+
     attr_accessor :http_method, :options, :last_response, :redirect, :last_uri
     attr_reader :path
 
@@ -69,7 +82,7 @@ module HTTParty
       @path = if uri.is_a?(uri_adapter)
         uri
       elsif String.try_convert(uri)
-        uri_adapter.parse uri
+        uri_adapter.parse(uri).normalize
       else
         raise ArgumentError,
           "bad argument (expected #{uri_adapter} object or URI string)"
@@ -85,17 +98,17 @@ module HTTParty
     end
 
     def uri
-      if redirect && path.relative? && path.path[0] != "/"
-        last_uri_host = @last_uri.path.gsub(/[^\/]+$/, "")
+      if redirect && path.relative? && path.path[0] != '/'
+        last_uri_host = @last_uri.path.gsub(/[^\/]+$/, '')
 
-        path.path = "/#{path.path}" if last_uri_host[-1] != "/"
-        path.path = last_uri_host + path.path
+        path.path = "/#{path.path}" if last_uri_host[-1] != '/'
+        path.path = "#{last_uri_host}#{path.path}"
       end
 
       if path.relative? && path.host
-        new_uri = options[:uri_adapter].parse("#{@last_uri.scheme}:#{path}")
+        new_uri = options[:uri_adapter].parse("#{@last_uri.scheme}:#{path}").normalize
       elsif path.relative?
-        new_uri = options[:uri_adapter].parse("#{base_uri}#{path}")
+        new_uri = options[:uri_adapter].parse("#{base_uri}#{path}").normalize
       else
         new_uri = path.clone
       end
@@ -115,7 +128,7 @@ module HTTParty
     def base_uri
       if redirect
         base_uri = "#{@last_uri.scheme}://#{@last_uri.host}"
-        base_uri += ":#{@last_uri.port}" if @last_uri.port != 80
+        base_uri = "#{base_uri}:#{@last_uri.port}" if @last_uri.port != 80
         base_uri
       else
         options[:base_uri] && HTTParty.normalize_base_uri(options[:base_uri])
@@ -138,21 +151,22 @@ module HTTParty
       validate
       setup_raw_request
       chunked_body = nil
+      current_http = http
 
-      self.last_response = http.request(@raw_request) do |http_response|
+      self.last_response = current_http.request(@raw_request) do |http_response|
         if block
           chunks = []
 
           http_response.read_body do |fragment|
-            chunks << fragment unless options[:stream_body]
-            block.call(fragment)
+            encoded_fragment = encode_text(fragment, http_response['content-type'])
+            chunks << encoded_fragment if !options[:stream_body]
+            block.call ResponseFragment.new(encoded_fragment, http_response, current_http)
           end
 
           chunked_body = chunks.join
         end
       end
 
-
       handle_host_redirection if response_redirects?
       result = handle_unauthorized
       result ||= handle_response(chunked_body, &block)
@@ -170,16 +184,19 @@ module HTTParty
       @raw_request.body
     end
 
+    def _dump(_level)
+      opts = options.dup
+      opts.delete(:logger)
+      opts.delete(:parser) if opts[:parser] && opts[:parser].is_a?(Proc)
+      Marshal.dump([http_method, path, opts, last_response, @last_uri, @raw_request])
+    end
+
     private
 
     def http
       connection_adapter.call(uri, options)
     end
 
-    def body
-      options[:body].respond_to?(:to_hash) ? normalize_query(options[:body]) : options[:body]
-    end
-
     def credentials
       (options[:basic_auth] || options[:digest_auth]).to_hash
     end
@@ -205,21 +222,31 @@ module HTTParty
     end
 
     def setup_raw_request
-      @raw_request = http_method.new(request_uri(uri))
-      @raw_request.body = body if body
-      @raw_request.body_stream = options[:body_stream] if options[:body_stream]
       if options[:headers].respond_to?(:to_hash)
         headers_hash = options[:headers].to_hash
+      else
+        headers_hash = nil
+      end
+
+      @raw_request = http_method.new(request_uri(uri), headers_hash)
+      @raw_request.body_stream = options[:body_stream] if options[:body_stream]
+
+      if options[:body]
+        body = Body.new(
+          options[:body],
+          query_string_normalizer: query_string_normalizer,
+          force_multipart: options[:multipart]
+        )
 
-        @raw_request.initialize_http_header(headers_hash)
-        # If the caller specified a header of 'Accept-Encoding', assume they want to
-        # deal with encoding of content. Disable the internal logic in Net:HTTP
-        # that handles encoding, if the platform supports it.
-        if @raw_request.respond_to?(:decode_content) && (headers_hash.key?('Accept-Encoding') || headers_hash.key?('accept-encoding'))
-          # Using the '[]=' sets decode_content to false
-          @raw_request['accept-encoding'] = @raw_request['accept-encoding']
+        if body.multipart?
+          content_type = "multipart/form-data; boundary=#{body.boundary}"
+          @raw_request['Content-Type'] = content_type
         end
+        @raw_request.body = body.call
       end
+
+      @raw_request.instance_variable_set(:@decode_content, decompress_content?)
+
       if options[:basic_auth] && send_authorization_header?
         @raw_request.basic_auth(username, password)
         @credentials_sent = true
@@ -231,6 +258,10 @@ module HTTParty
       !!options[:digest_auth]
     end
 
+    def decompress_content?
+      !options[:skip_decompression]
+    end
+
     def response_unauthorized?
       !!last_response && last_response.code == '401'
     end
@@ -254,77 +285,15 @@ module HTTParty
         query_string_parts << options[:query] unless options[:query].nil?
       end
 
-      query_string_parts.reject!(&:empty?) unless query_string_parts == [""]
+      query_string_parts.reject!(&:empty?) unless query_string_parts == ['']
       query_string_parts.size > 0 ? query_string_parts.join('&') : nil
     end
 
-    def get_charset
-      content_type = last_response["content-type"]
-      if content_type.nil?
-        return nil
-      end
-
-      if content_type =~ /;\s*charset\s*=\s*([^=,;"\s]+)/i
-        return $1
-      end
-
-      if content_type =~ /;\s*charset\s*=\s*"((\\.|[^\\"])+)"/i
-        return $1.gsub(/\\(.)/, '\1')
-      end
-
-      nil
-    end
-
-    def encode_with_ruby_encoding(body, charset)
-      encoding = Encoding.find(charset)
-      body.force_encoding(charset)
-    rescue ArgumentError
-      body
-    end
-
     def assume_utf16_is_big_endian
       options[:assume_utf16_is_big_endian]
     end
 
-    def encode_utf_16(body)
-      if body.bytesize >= 2
-        if body.getbyte(0) == 0xFF && body.getbyte(1) == 0xFE
-          return body.force_encoding("UTF-16LE")
-        elsif body.getbyte(0) == 0xFE && body.getbyte(1) == 0xFF
-          return body.force_encoding("UTF-16BE")
-        end
-      end
-
-      if assume_utf16_is_big_endian
-        body.force_encoding("UTF-16BE")
-      else
-        body.force_encoding("UTF-16LE")
-      end
-    end
-
-    def _encode_body(body)
-      charset = get_charset
-
-      if charset.nil?
-        return body
-      end
-
-      if "utf-16".casecmp(charset) == 0
-        encode_utf_16(body)
-      else
-        encode_with_ruby_encoding(body, charset)
-      end
-    end
-
-    def encode_body(body)
-      if "".respond_to?(:encoding)
-        _encode_body(body)
-      else
-        body
-      end
-    end
-
-    def handle_response(body, &block)
+    def handle_response(raw_body, &block)
       if response_redirects?
         options[:limit] -= 1
         if options[:logger]
@@ -345,15 +314,26 @@ module HTTParty
         capture_cookies(last_response)
         perform(&block)
       else
-        body ||= last_response.body
-        body = body.nil? ? body : encode_body(body)
-        Response.new(self, last_response, lambda { parse_response(body) }, body: body)
+        raw_body ||= last_response.body
+
+        body = decompress(raw_body, last_response['content-encoding']) unless raw_body.nil?
+
+        unless body.nil?
+          body = encode_text(body, last_response['content-type'])
+
+          if decompress_content?
+            last_response.delete('content-encoding')
+            raw_body = body
+          end
+        end
+
+        Response.new(self, last_response, lambda { parse_response(body) }, body: raw_body)
       end
     end
 
     def handle_host_redirection
       check_duplicate_location_header
-      redirect_path = options[:uri_adapter].parse last_response['location']
+      redirect_path = options[:uri_adapter].parse(last_response['location']).normalize
       return if redirect_path.relative? || path.host == redirect_path.host
       @changed_hosts = true
     end
@@ -422,5 +402,17 @@ module HTTParty
         @credentials_sent = true
       end
     end
+
+    def decompress(body, encoding)
+      Decompressor.new(body, encoding).decompress
+    end
+
+    def encode_text(text, content_type)
+      TextEncoder.new(
+        text,
+        content_type: content_type,
+        assume_utf16_is_big_endian: assume_utf16_is_big_endian
+      ).call
+    end
   end
 end

data/lib/httparty/response/headers.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'delegate'
 
 module HTTParty
@@ -22,10 +24,10 @@ module HTTParty
       end
 
       def ==(other)
-        if other.is_a?(::Net::HTTPHeader) 
+        if other.is_a?(::Net::HTTPHeader)
           @header == other.instance_variable_get(:@header)
         elsif other.is_a?(Hash)
-          @header == other || @header == Headers.new(other).instance_variable_get(:@header)           
+          @header == other || @header == Headers.new(other).instance_variable_get(:@header)
         end
       end
     end