httparty 0.14.0 → 0.20.0

data/lib/httparty/net_digest_auth.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'digest/md5'
 require 'net/http'
 
@@ -12,13 +14,13 @@ module Net
         response
       )
 
-      @header['Authorization'] = authenticator.authorization_header
-      @header['cookie'] = append_cookies(authenticator) if response['Set-Cookie']
-    end
+      authenticator.authorization_header.each do |v|
+        add_field('Authorization', v)
+      end
 
-    def append_cookies(authenticator)
-      cookies = @header['cookie'] ? @header['cookie'] : []
-      cookies.concat(authenticator.cookie_header)
+      authenticator.cookie_header.each do |v|
+        add_field('Cookie', v)
+      end
     end
 
     class DigestAuthenticator
@@ -44,12 +46,9 @@ module Net
         header << %(algorithm="#{@response['algorithm']}") if algorithm_present?
 
         if qop_present?
-          fields = [
-            %(cnonce="#{@cnonce}"),
-            %(qop="#{@response['qop']}"),
-            "nc=00000001"
-          ]
-          fields.each { |field| header << field }
+          header << %(cnonce="#{@cnonce}")
+          header << %(qop="#{@response['qop']}")
+          header << 'nc=00000001'
         end
 
         header << %(opaque="#{@response['opaque']}") if opaque_present?
@@ -64,7 +63,8 @@ module Net
 
       def parse(response_header)
         header = response_header['www-authenticate']
-                 .gsub(/qop=(auth(?:-int)?)/, 'qop="\\1"')
+
+        header = header.gsub(/qop=(auth(?:-int)?)/, 'qop="\\1"')
 
         header =~ /Digest (.*)/
         params = {}
@@ -97,13 +97,13 @@ module Net
       end
 
       def random
-        format "%x", (Time.now.to_i + rand(65535))
+        format '%x', (Time.now.to_i + rand(65535))
       end
 
       def request_digest
         a = [md5(a1), @response['nonce'], md5(a2)]
-        a.insert(2, "00000001", @cnonce, @response['qop']) if qop_present?
-        md5(a.join(":"))
+        a.insert(2, '00000001', @cnonce, @response['qop']) if qop_present?
+        md5(a.join(':'))
       end
 
       def md5(str)
@@ -113,11 +113,11 @@ module Net
       def algorithm_present?
         @response.key?('algorithm') && !@response['algorithm'].empty?
       end
-      
+
       def use_md5_sess?
         algorithm_present? && @response['algorithm'] == 'MD5-sess'
       end
-      
+
       def a1
         a1_user_realm_pwd =  [@username, @response['realm'], @password].join(':')
         if use_md5_sess?
@@ -128,7 +128,7 @@ module Net
       end
 
       def a2
-        [@method, @path].join(":")
+        [@method, @path].join(':')
       end
     end
   end

data/lib/httparty/parser.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module HTTParty
   # The default parser used by HTTParty, supports xml, json, html, csv and
   # plain text.
@@ -38,16 +40,18 @@ module HTTParty
   # @abstract Read the Custom Parsers section for more information.
   class Parser
     SupportedFormats = {
-      'text/xml'               => :xml,
-      'application/xml'        => :xml,
-      'application/json'       => :json,
-      'text/json'              => :json,
-      'application/javascript' => :plain,
-      'text/javascript'        => :plain,
-      'text/html'              => :html,
-      'text/plain'             => :plain,
-      'text/csv'               => :csv,
-      'application/csv'        => :csv,
+      'text/xml'                    => :xml,
+      'application/xml'             => :xml,
+      'application/json'            => :json,
+      'application/vnd.api+json'    => :json,
+      'application/hal+json'        => :json,
+      'text/json'                   => :json,
+      'application/javascript'      => :plain,
+      'text/javascript'             => :plain,
+      'text/html'                   => :html,
+      'text/plain'                  => :plain,
+      'text/csv'                    => :csv,
+      'application/csv'             => :csv,
       'text/comma-separated-values' => :csv
     }
 
@@ -99,8 +103,11 @@ module HTTParty
     # @return [nil] when the response body is nil, an empty string, spaces only or "null"
     def parse
       return nil if body.nil?
-      return nil if body == "null"
+      return nil if body == 'null'
       return nil if body.valid_encoding? && body.strip.empty?
+      if body.valid_encoding? && body.encoding == Encoding::UTF_8
+        @body = body.gsub(/\A#{UTF8_BOM}/, '')
+      end
       if supports_format?
         parse_supported_format
       else
@@ -114,6 +121,8 @@ module HTTParty
       MultiXml.parse(body)
     end
 
+    UTF8_BOM = "\xEF\xBB\xBF"
+
     def json
       JSON.parse(body, :quirks_mode => true, :allow_nan => true)
     end
@@ -135,9 +144,11 @@ module HTTParty
     end
 
     def parse_supported_format
-      send(format)
-    rescue NoMethodError => e
-      raise NotImplementedError, "#{self.class.name} has not implemented a parsing method for the #{format.inspect} format.", e.backtrace
+      if respond_to?(format, true)
+        send(format)
+      else
+        raise NotImplementedError, "#{self.class.name} has not implemented a parsing method for the #{format.inspect} format."
+      end
     end
   end
 end

data/lib/httparty/request/body.rb

@@ -0,0 +1,98 @@
+# frozen_string_literal: true
+
+require_relative 'multipart_boundary'
+
+module HTTParty
+  class Request
+    class Body
+      NEWLINE = "\r\n"
+      private_constant :NEWLINE
+
+      def initialize(params, query_string_normalizer: nil, force_multipart: false)
+        @params = params
+        @query_string_normalizer = query_string_normalizer
+        @force_multipart = force_multipart
+      end
+
+      def call
+        if params.respond_to?(:to_hash)
+          multipart? ? generate_multipart : normalize_query(params)
+        else
+          params
+        end
+      end
+
+      def boundary
+        @boundary ||= MultipartBoundary.generate
+      end
+
+      def multipart?
+        params.respond_to?(:to_hash) && (force_multipart || has_file?(params))
+      end
+
+      private
+
+      def generate_multipart
+        normalized_params = params.flat_map { |key, value| HashConversions.normalize_keys(key, value) }
+
+        multipart = normalized_params.inject(''.dup) do |memo, (key, value)|
+          memo << "--#{boundary}#{NEWLINE}"
+          memo << %(Content-Disposition: form-data; name="#{key}")
+          # value.path is used to support ActionDispatch::Http::UploadedFile
+          # https://github.com/jnunemaker/httparty/pull/585
+          memo << %(; filename="#{file_name(value)}") if file?(value)
+          memo << NEWLINE
+          memo << "Content-Type: #{content_type(value)}#{NEWLINE}" if file?(value)
+          memo << NEWLINE
+          memo << content_body(value)
+          memo << NEWLINE
+        end
+
+        multipart << "--#{boundary}--#{NEWLINE}"
+      end
+
+      def has_file?(value)
+        if value.respond_to?(:to_hash)
+          value.to_hash.any? { |_, v| has_file?(v) }
+        elsif value.respond_to?(:to_ary)
+          value.to_ary.any? { |v| has_file?(v) }
+        else
+          file?(value)
+        end
+      end
+
+      def file?(object)
+        object.respond_to?(:path) && object.respond_to?(:read)
+      end
+
+      def normalize_query(query)
+        if query_string_normalizer
+          query_string_normalizer.call(query)
+        else
+          HashConversions.to_params(query)
+        end
+      end
+
+      def content_body(object)
+        if file?(object)
+          object = (file = object).read
+          file.rewind if file.respond_to?(:rewind)
+        end
+
+        object.to_s
+      end
+
+      def content_type(object)
+        return object.content_type if object.respond_to?(:content_type)
+        mime = MIME::Types.type_for(object.path)
+        mime.empty? ? 'application/octet-stream' : mime[0].content_type
+      end
+
+      def file_name(object)
+        object.respond_to?(:original_filename) ? object.original_filename : File.basename(object.path)
+      end
+
+      attr_reader :params, :query_string_normalizer, :force_multipart
+    end
+  end
+end

data/lib/httparty/request/multipart_boundary.rb

@@ -0,0 +1,13 @@
+# frozen_string_literal: true
+
+require 'securerandom'
+
+module HTTParty
+  class Request
+    class MultipartBoundary
+      def self.generate
+        "------------------------#{SecureRandom.urlsafe_base64(12)}"
+      end
+    end
+  end
+end

data/lib/httparty/request.rb

@@ -1,3 +1,7 @@
+# frozen_string_literal: true
+
+require 'erb'
+
 module HTTParty
   class Request #:nodoc:
     SupportedHTTPMethods = [
@@ -11,6 +15,8 @@ module HTTParty
       Net::HTTP::Move,
       Net::HTTP::Copy,
       Net::HTTP::Mkcol,
+      Net::HTTP::Lock,
+      Net::HTTP::Unlock,
     ]
 
     SupportedURISchemes  = ['http', 'https', 'webcal', nil]
@@ -27,10 +33,31 @@ module HTTParty
       end.flatten.join('&')
     end
 
+    JSON_API_QUERY_STRING_NORMALIZER = proc do |query|
+      Array(query).sort_by { |a| a[0].to_s }.map do |key, value|
+        if value.nil?
+          key.to_s
+        elsif value.respond_to?(:to_ary)
+          values = value.to_ary.map{|v| ERB::Util.url_encode(v.to_s)}
+          "#{key}=#{values.join(',')}"
+        else
+          HashConversions.to_params(key => value)
+        end
+      end.flatten.join('&')
+    end
+
+    def self._load(data)
+      http_method, path, options = Marshal.load(data)
+      new(http_method, path, options)
+    end
+
     attr_accessor :http_method, :options, :last_response, :redirect, :last_uri
     attr_reader :path
 
     def initialize(http_method, path, o = {})
+      @changed_hosts = false
+      @credentials_sent = false
+
       self.http_method = http_method
       self.options = {
         limit: o.delete(:no_follow) ? 1 : 5,
@@ -51,7 +78,7 @@ module HTTParty
       @path = if uri.is_a?(uri_adapter)
         uri
       elsif String.try_convert(uri)
-        uri_adapter.parse uri
+        uri_adapter.parse(uri).normalize
       else
         raise ArgumentError,
           "bad argument (expected #{uri_adapter} object or URI string)"
@@ -67,14 +94,20 @@ module HTTParty
     end
 
     def uri
-      if redirect && path.relative? && path.path[0] != "/"
-        last_uri_host = @last_uri.path.gsub(/[^\/]+$/, "")
+      if redirect && path.relative? && path.path[0] != '/'
+        last_uri_host = @last_uri.path.gsub(/[^\/]+$/, '')
 
-        path.path = "/#{path.path}" if last_uri_host[-1] != "/"
-        path.path = last_uri_host + path.path
+        path.path = "/#{path.path}" if last_uri_host[-1] != '/'
+        path.path = "#{last_uri_host}#{path.path}"
       end
 
-      new_uri = path.relative? ? options[:uri_adapter].parse("#{base_uri}#{path}") : path.clone
+      if path.relative? && path.host
+        new_uri = options[:uri_adapter].parse("#{@last_uri.scheme}:#{path}").normalize
+      elsif path.relative?
+        new_uri = options[:uri_adapter].parse("#{base_uri}#{path}").normalize
+      else
+        new_uri = path.clone
+      end
 
       # avoid double query string on redirects [#12]
       unless redirect
@@ -91,7 +124,7 @@ module HTTParty
     def base_uri
       if redirect
         base_uri = "#{@last_uri.scheme}://#{@last_uri.host}"
-        base_uri += ":#{@last_uri.port}" if @last_uri.port != 80
+        base_uri = "#{base_uri}:#{@last_uri.port}" if @last_uri.port != 80
         base_uri
       else
         options[:base_uri] && HTTParty.normalize_base_uri(options[:base_uri])
@@ -114,39 +147,52 @@ module HTTParty
       validate
       setup_raw_request
       chunked_body = nil
+      current_http = http
 
-      self.last_response = http.request(@raw_request) do |http_response|
+      self.last_response = current_http.request(@raw_request) do |http_response|
         if block
           chunks = []
 
           http_response.read_body do |fragment|
-            chunks << fragment unless options[:stream_body]
-            block.call(fragment)
+            encoded_fragment = encode_text(fragment, http_response['content-type'])
+            chunks << encoded_fragment if !options[:stream_body]
+            block.call ResponseFragment.new(encoded_fragment, http_response, current_http)
           end
 
           chunked_body = chunks.join
         end
       end
 
-      handle_deflation unless http_method == Net::HTTP::Head
       handle_host_redirection if response_redirects?
-      handle_response(chunked_body, &block)
+      result = handle_unauthorized
+      result ||= handle_response(chunked_body, &block)
+      result
+    end
+
+    def handle_unauthorized(&block)
+      return unless digest_auth? && response_unauthorized? && response_has_digest_auth_challenge?
+      return if @credentials_sent
+      @credentials_sent = true
+      perform(&block)
     end
 
     def raw_body
       @raw_request.body
     end
 
+    def _dump(_level)
+      opts = options.dup
+      opts.delete(:logger)
+      opts.delete(:parser) if opts[:parser] && opts[:parser].is_a?(Proc)
+      Marshal.dump([http_method, path, opts])
+    end
+
     private
 
     def http
       connection_adapter.call(uri, options)
     end
 
-    def body
-      options[:body].respond_to?(:to_hash) ? normalize_query(options[:body]) : options[:body]
-    end
-
     def credentials
       (options[:basic_auth] || options[:digest_auth]).to_hash
     end
@@ -172,106 +218,78 @@ module HTTParty
     end
 
     def setup_raw_request
-      @raw_request = http_method.new(request_uri(uri))
-      @raw_request.body = body if body
-      @raw_request.body_stream = options[:body_stream] if options[:body_stream]
-      @raw_request.initialize_http_header(options[:headers].to_hash) if options[:headers].respond_to?(:to_hash)
-      @raw_request.basic_auth(username, password) if options[:basic_auth] && send_authorization_header?
-      setup_digest_auth if options[:digest_auth]
-    end
-
-    def setup_digest_auth
-      auth_request = http_method.new(uri.request_uri)
-      auth_request.initialize_http_header(options[:headers].to_hash) if options[:headers].respond_to?(:to_hash)
-      res = http.request(auth_request)
-
-      if !res['www-authenticate'].nil? && res['www-authenticate'].length > 0
-        @raw_request.digest_auth(username, password, res)
-      end
-    end
-
-    def query_string(uri)
-      query_string_parts = []
-      query_string_parts << uri.query unless uri.query.nil?
-
-      if options[:query].respond_to?(:to_hash)
-        query_string_parts << normalize_query(options[:default_params].merge(options[:query].to_hash))
+      if options[:headers].respond_to?(:to_hash)
+        headers_hash = options[:headers].to_hash
       else
-        query_string_parts << normalize_query(options[:default_params]) unless options[:default_params].empty?
-        query_string_parts << options[:query] unless options[:query].nil?
+        headers_hash = nil
       end
 
-      query_string_parts.reject!(&:empty?) unless query_string_parts == [""]
-      query_string_parts.size > 0 ? query_string_parts.join('&') : nil
-    end
+      @raw_request = http_method.new(request_uri(uri), headers_hash)
+      @raw_request.body_stream = options[:body_stream] if options[:body_stream]
 
-    def get_charset
-      content_type = last_response["content-type"]
-      if content_type.nil?
-        return nil
-      end
+      if options[:body]
+        body = Body.new(
+          options[:body],
+          query_string_normalizer: query_string_normalizer,
+          force_multipart: options[:multipart]
+        )
 
-      if content_type =~ /;\s*charset\s*=\s*([^=,;"\s]+)/i
-        return $1
+        if body.multipart?
+          content_type = "multipart/form-data; boundary=#{body.boundary}"
+          @raw_request['Content-Type'] = content_type
+        end
+        @raw_request.body = body.call
       end
 
-      if content_type =~ /;\s*charset\s*=\s*"((\\.|[^\\"])+)"/i
-        return $1.gsub(/\\(.)/, '\1')
-      end
+      @raw_request.instance_variable_set(:@decode_content, decompress_content?)
 
-      nil
+      if options[:basic_auth] && send_authorization_header?
+        @raw_request.basic_auth(username, password)
+        @credentials_sent = true
+      end
+      setup_digest_auth if digest_auth? && response_unauthorized? && response_has_digest_auth_challenge?
     end
 
-    def encode_with_ruby_encoding(body, charset)
-      encoding = Encoding.find(charset)
-      body.force_encoding(encoding)
-    rescue
-      body
+    def digest_auth?
+      !!options[:digest_auth]
     end
 
-    def assume_utf16_is_big_endian
-      options[:assume_utf16_is_big_endian]
+    def decompress_content?
+      !options[:skip_decompression]
     end
 
-    def encode_utf_16(body)
-      if body.bytesize >= 2
-        if body.getbyte(0) == 0xFF && body.getbyte(1) == 0xFE
-          return body.force_encoding("UTF-16LE")
-        elsif body.getbyte(0) == 0xFE && body.getbyte(1) == 0xFF
-          return body.force_encoding("UTF-16BE")
-        end
-      end
+    def response_unauthorized?
+      !!last_response && last_response.code == '401'
+    end
 
-      if assume_utf16_is_big_endian
-        body.force_encoding("UTF-16BE")
-      else
-        body.force_encoding("UTF-16LE")
-      end
+    def response_has_digest_auth_challenge?
+      !last_response['www-authenticate'].nil? && last_response['www-authenticate'].length > 0
     end
 
-    def _encode_body(body)
-      charset = get_charset
+    def setup_digest_auth
+      @raw_request.digest_auth(username, password, last_response)
+    end
 
-      if charset.nil?
-        return body
-      end
+    def query_string(uri)
+      query_string_parts = []
+      query_string_parts << uri.query unless uri.query.nil?
 
-      if "utf-16".casecmp(charset) == 0
-        encode_utf_16(body)
+      if options[:query].respond_to?(:to_hash)
+        query_string_parts << normalize_query(options[:default_params].merge(options[:query].to_hash))
       else
-        encode_with_ruby_encoding(body, charset)
+        query_string_parts << normalize_query(options[:default_params]) unless options[:default_params].empty?
+        query_string_parts << options[:query] unless options[:query].nil?
       end
+
+      query_string_parts.reject!(&:empty?) unless query_string_parts == ['']
+      query_string_parts.size > 0 ? query_string_parts.join('&') : nil
     end
 
-    def encode_body(body)
-      if "".respond_to?(:encoding)
-        _encode_body(body)
-      else
-        body
-      end
+    def assume_utf16_is_big_endian
+      options[:assume_utf16_is_big_endian]
     end
 
-    def handle_response(body, &block)
+    def handle_response(raw_body, &block)
       if response_redirects?
         options[:limit] -= 1
         if options[:logger]
@@ -292,31 +310,26 @@ module HTTParty
         capture_cookies(last_response)
         perform(&block)
       else
-        body ||= last_response.body
-        body = encode_body(body)
-        Response.new(self, last_response, lambda { parse_response(body) }, body: body)
-      end
-    end
+        raw_body ||= last_response.body
+
+        body = decompress(raw_body, last_response['content-encoding']) unless raw_body.nil?
+
+        unless body.nil?
+          body = encode_text(body, last_response['content-type'])
 
-    # Inspired by Ruby 1.9
-    def handle_deflation
-      return if response_redirects?
-      return if last_response.body.nil?
+          if decompress_content?
+            last_response.delete('content-encoding')
+            raw_body = body
+          end
+        end
 
-      case last_response["content-encoding"]
-      when "gzip", "x-gzip"
-        body_io = StringIO.new(last_response.body)
-        last_response.body.replace Zlib::GzipReader.new(body_io).read
-        last_response.delete('content-encoding')
-      when "deflate"
-        last_response.body.replace Zlib::Inflate.inflate(last_response.body)
-        last_response.delete('content-encoding')
+        Response.new(self, last_response, lambda { parse_response(body) }, body: raw_body)
       end
     end
 
     def handle_host_redirection
       check_duplicate_location_header
-      redirect_path = options[:uri_adapter].parse last_response['location']
+      redirect_path = options[:uri_adapter].parse(last_response['location']).normalize
       return if redirect_path.relative? || path.host == redirect_path.host
       @changed_hosts = true
     end
@@ -329,7 +342,7 @@ module HTTParty
     end
 
     def send_authorization_header?
-      !defined?(@changed_hosts)
+      !@changed_hosts
     end
 
     def response_redirects?
@@ -350,6 +363,7 @@ module HTTParty
       cookies_hash = HTTParty::CookieHash.new
       cookies_hash.add_cookies(options[:headers].to_hash['Cookie']) if options[:headers] && options[:headers].to_hash['Cookie']
       response.get_fields('Set-Cookie').each { |cookie| cookies_hash.add_cookies(cookie) }
+
       options[:headers] ||= {}
       options[:headers]['Cookie'] = cookies_hash.to_cookie_string
     end
@@ -381,7 +395,20 @@ module HTTParty
       if path.userinfo
         username, password = path.userinfo.split(':')
         options[:basic_auth] = {username: username, password: password}
+        @credentials_sent = true
       end
     end
+
+    def decompress(body, encoding)
+      Decompressor.new(body, encoding).decompress
+    end
+
+    def encode_text(text, content_type)
+      TextEncoder.new(
+        text,
+        content_type: content_type,
+        assume_utf16_is_big_endian: assume_utf16_is_big_endian
+      ).call
+    end
   end
 end