httparty 0.13.0 → 0.21.0

data/lib/httparty/request.rb

@@ -1,3 +1,7 @@
+# frozen_string_literal: true
+
+require 'erb'
+
 module HTTParty
   class Request #:nodoc:
     SupportedHTTPMethods = [
@@ -9,41 +13,80 @@ module HTTParty
       Net::HTTP::Head,
       Net::HTTP::Options,
       Net::HTTP::Move,
-      Net::HTTP::Copy
+      Net::HTTP::Copy,
+      Net::HTTP::Mkcol,
+      Net::HTTP::Lock,
+      Net::HTTP::Unlock,
     ]
 
-    SupportedURISchemes  = [URI::HTTP, URI::HTTPS, URI::Generic]
+    SupportedURISchemes  = ['http', 'https', 'webcal', nil]
+
+    NON_RAILS_QUERY_STRING_NORMALIZER = proc do |query|
+      Array(query).sort_by { |a| a[0].to_s }.map do |key, value|
+        if value.nil?
+          key.to_s
+        elsif value.respond_to?(:to_ary)
+          value.to_ary.map {|v| "#{key}=#{ERB::Util.url_encode(v.to_s)}"}
+        else
+          HashConversions.to_params(key => value)
+        end
+      end.flatten.join('&')
+    end
 
-    NON_RAILS_QUERY_STRING_NORMALIZER = Proc.new do |query|
+    JSON_API_QUERY_STRING_NORMALIZER = proc do |query|
       Array(query).sort_by { |a| a[0].to_s }.map do |key, value|
         if value.nil?
           key.to_s
-        elsif value.is_a?(Array)
-          value.map {|v| "#{key}=#{URI.encode(v.to_s, Regexp.new("[^#{URI::PATTERN::UNRESERVED}]"))}"}
+        elsif value.respond_to?(:to_ary)
+          values = value.to_ary.map{|v| ERB::Util.url_encode(v.to_s)}
+          "#{key}=#{values.join(',')}"
         else
           HashConversions.to_params(key => value)
         end
       end.flatten.join('&')
     end
 
+    def self._load(data)
+      http_method, path, options, last_response, last_uri, raw_request = Marshal.load(data)
+      instance = new(http_method, path, options)
+      instance.last_response = last_response
+      instance.last_uri = last_uri
+      instance.instance_variable_set("@raw_request", raw_request)
+      instance
+    end
+
     attr_accessor :http_method, :options, :last_response, :redirect, :last_uri
     attr_reader :path
 
-    def initialize(http_method, path, o={})
+    def initialize(http_method, path, o = {})
+      @changed_hosts = false
+      @credentials_sent = false
+
       self.http_method = http_method
-      self.path = path
       self.options = {
-        :limit => o.delete(:no_follow) ? 1 : 5,
-        :assume_utf16_is_big_endian => true,
-        :default_params => {},
-        :follow_redirects => true,
-        :parser => Parser,
-        :connection_adapter => ConnectionAdapter
+        limit: o.delete(:no_follow) ? 1 : 5,
+        assume_utf16_is_big_endian: true,
+        default_params: {},
+        follow_redirects: true,
+        parser: Parser,
+        uri_adapter: URI,
+        connection_adapter: ConnectionAdapter
       }.merge(o)
+      self.path = path
+      set_basic_auth_from_uri
     end
 
     def path=(uri)
-      @path = URI.parse(uri)
+      uri_adapter = options[:uri_adapter]
+
+      @path = if uri.is_a?(uri_adapter)
+        uri
+      elsif String.try_convert(uri)
+        uri_adapter.parse(uri).normalize
+      else
+        raise ArgumentError,
+          "bad argument (expected #{uri_adapter} object or URI string)"
+      end
     end
 
     def request_uri(uri)
@@ -55,14 +98,27 @@ module HTTParty
     end
 
     def uri
-      new_uri = path.relative? ? URI.parse("#{base_uri}#{path}") : path.clone
+      if redirect && path.relative? && path.path[0] != '/'
+        last_uri_host = @last_uri.path.gsub(/[^\/]+$/, '')
+
+        path.path = "/#{path.path}" if last_uri_host[-1] != '/'
+        path.path = "#{last_uri_host}#{path.path}"
+      end
+
+      if path.relative? && path.host
+        new_uri = options[:uri_adapter].parse("#{@last_uri.scheme}:#{path}").normalize
+      elsif path.relative?
+        new_uri = options[:uri_adapter].parse("#{base_uri}#{path}").normalize
+      else
+        new_uri = path.clone
+      end
 
       # avoid double query string on redirects [#12]
       unless redirect
         new_uri.query = query_string(new_uri)
       end
 
-      unless SupportedURISchemes.include? new_uri.class
+      unless SupportedURISchemes.include? new_uri.scheme
         raise UnsupportedURIScheme, "'#{new_uri}' Must be HTTP, HTTPS or Generic"
       end
 
@@ -70,7 +126,13 @@ module HTTParty
     end
 
     def base_uri
-      redirect ? "#{@last_uri.scheme}://#{@last_uri.host}" : options[:base_uri]
+      if redirect
+        base_uri = "#{@last_uri.scheme}://#{@last_uri.host}"
+        base_uri = "#{base_uri}:#{@last_uri.port}" if @last_uri.port != 80
+        base_uri
+      else
+        options[:base_uri] && HTTParty.normalize_base_uri(options[:base_uri])
+      end
     end
 
     def format
@@ -89,40 +151,54 @@ module HTTParty
       validate
       setup_raw_request
       chunked_body = nil
+      current_http = http
 
-      self.last_response = http.request(@raw_request) do |http_response|
+      self.last_response = current_http.request(@raw_request) do |http_response|
         if block
           chunks = []
 
           http_response.read_body do |fragment|
-            chunks << fragment
-            block.call(fragment)
+            encoded_fragment = encode_text(fragment, http_response['content-type'])
+            chunks << encoded_fragment if !options[:stream_body]
+            block.call ResponseFragment.new(encoded_fragment, http_response, current_http)
           end
 
           chunked_body = chunks.join
         end
       end
 
-      handle_deflation unless http_method == Net::HTTP::Head
-      handle_response(chunked_body, &block)
+      handle_host_redirection if response_redirects?
+      result = handle_unauthorized
+      result ||= handle_response(chunked_body, &block)
+      result
+    end
+
+    def handle_unauthorized(&block)
+      return unless digest_auth? && response_unauthorized? && response_has_digest_auth_challenge?
+      return if @credentials_sent
+      @credentials_sent = true
+      perform(&block)
     end
 
     def raw_body
       @raw_request.body
     end
 
+    def _dump(_level)
+      opts = options.dup
+      opts.delete(:logger)
+      opts.delete(:parser) if opts[:parser] && opts[:parser].is_a?(Proc)
+      Marshal.dump([http_method, path, opts, last_response, @last_uri, @raw_request])
+    end
+
     private
 
     def http
       connection_adapter.call(uri, options)
     end
 
-    def body
-      options[:body].is_a?(Hash) ? normalize_query(options[:body]) : options[:body]
-    end
-
     def credentials
-      options[:basic_auth] || options[:digest_auth]
+      (options[:basic_auth] || options[:digest_auth]).to_hash
     end
 
     def username
@@ -146,108 +222,78 @@ module HTTParty
     end
 
     def setup_raw_request
-      @raw_request = http_method.new(request_uri(uri))
-      @raw_request.body = body if body
-      @raw_request.initialize_http_header(options[:headers])
-      @raw_request.basic_auth(username, password) if options[:basic_auth]
-      setup_digest_auth if options[:digest_auth]
-    end
-
-    def setup_digest_auth
-      auth_request = http_method.new(uri.request_uri)
-      auth_request.initialize_http_header(options[:headers])
-      res = http.request(auth_request)
-
-      if res['www-authenticate'] != nil && res['www-authenticate'].length > 0
-        @raw_request.digest_auth(username, password, res)
-      end
-    end
-
-    def query_string(uri)
-      query_string_parts = []
-      query_string_parts << uri.query unless uri.query.nil?
-
-      if options[:query].is_a?(Hash)
-        query_string_parts << normalize_query(options[:default_params].merge(options[:query]))
+      if options[:headers].respond_to?(:to_hash)
+        headers_hash = options[:headers].to_hash
       else
-        query_string_parts << normalize_query(options[:default_params]) unless options[:default_params].empty?
-        query_string_parts << options[:query] unless options[:query].nil?
+        headers_hash = nil
       end
 
-      query_string_parts.reject!(&:empty?) unless query_string_parts == [""]
-      query_string_parts.size > 0 ? query_string_parts.join('&') : nil
-    end
+      @raw_request = http_method.new(request_uri(uri), headers_hash)
+      @raw_request.body_stream = options[:body_stream] if options[:body_stream]
 
-    def get_charset
-      content_type = last_response["content-type"]
-      if content_type.nil?
-        return nil
-      end
+      if options[:body]
+        body = Body.new(
+          options[:body],
+          query_string_normalizer: query_string_normalizer,
+          force_multipart: options[:multipart]
+        )
 
-      if content_type =~ /;\s*charset\s*=\s*([^=,;"\s]+)/i
-        return $1
-      end
-
-      if content_type =~ /;\s*charset\s*=\s*"((\\.|[^\\"])+)"/i
-        return $1.gsub(/\\(.)/, '\1')
+        if body.multipart?
+          content_type = "multipart/form-data; boundary=#{body.boundary}"
+          @raw_request['Content-Type'] = content_type
+        end
+        @raw_request.body = body.call
       end
 
-      nil
-    end
+      @raw_request.instance_variable_set(:@decode_content, decompress_content?)
 
-    def encode_with_ruby_encoding(body, charset)
-      begin
-        encoding = Encoding.find(charset)
-        body.force_encoding(encoding)
-      rescue
-        body
+      if options[:basic_auth] && send_authorization_header?
+        @raw_request.basic_auth(username, password)
+        @credentials_sent = true
       end
+      setup_digest_auth if digest_auth? && response_unauthorized? && response_has_digest_auth_challenge?
     end
 
-    def assume_utf16_is_big_endian
-      options[:assume_utf16_is_big_endian]
+    def digest_auth?
+      !!options[:digest_auth]
     end
 
-    def encode_utf_16(body)
-      if body.bytesize >= 2
-        if body.getbyte(0) == 0xFF && body.getbyte(1) == 0xFE
-          return body.force_encoding("UTF-16LE")
-        elsif body.getbyte(0) == 0xFE && body.getbyte(1) == 0xFF
-          return body.force_encoding("UTF-16BE")
-        end
-      end
+    def decompress_content?
+      !options[:skip_decompression]
+    end
 
-      if assume_utf16_is_big_endian
-        body.force_encoding("UTF-16BE")
-      else
-        body.force_encoding("UTF-16LE")
-      end
+    def response_unauthorized?
+      !!last_response && last_response.code == '401'
+    end
 
+    def response_has_digest_auth_challenge?
+      !last_response['www-authenticate'].nil? && last_response['www-authenticate'].length > 0
     end
 
-    def _encode_body(body)
-      charset = get_charset
+    def setup_digest_auth
+      @raw_request.digest_auth(username, password, last_response)
+    end
 
-      if charset.nil?
-        return body
-      end
+    def query_string(uri)
+      query_string_parts = []
+      query_string_parts << uri.query unless uri.query.nil?
 
-      if "utf-16".casecmp(charset) == 0
-        encode_utf_16(body)
+      if options[:query].respond_to?(:to_hash)
+        query_string_parts << normalize_query(options[:default_params].merge(options[:query].to_hash))
       else
-        encode_with_ruby_encoding(body, charset)
+        query_string_parts << normalize_query(options[:default_params]) unless options[:default_params].empty?
+        query_string_parts << options[:query] unless options[:query].nil?
       end
+
+      query_string_parts.reject!(&:empty?) unless query_string_parts == ['']
+      query_string_parts.size > 0 ? query_string_parts.join('&') : nil
     end
 
-    def encode_body(body)
-      if "".respond_to?(:encoding)
-        _encode_body(body)
-      else
-        body
-      end
+    def assume_utf16_is_big_endian
+      options[:assume_utf16_is_big_endian]
     end
 
-    def handle_response(body, &block)
+    def handle_response(raw_body, &block)
       if response_redirects?
         options[:limit] -= 1
         if options[:logger]
@@ -256,37 +302,58 @@ module HTTParty
         end
         self.path = last_response['location']
         self.redirect = true
-        self.http_method = Net::HTTP::Get unless options[:maintain_method_across_redirects]
+        if last_response.class == Net::HTTPSeeOther
+          unless options[:maintain_method_across_redirects] && options[:resend_on_redirect]
+            self.http_method = Net::HTTP::Get
+          end
+        elsif last_response.code != '307' && last_response.code != '308'
+          unless options[:maintain_method_across_redirects]
+            self.http_method = Net::HTTP::Get
+          end
+        end
         capture_cookies(last_response)
         perform(&block)
       else
-        body = body || last_response.body
-        body = encode_body(body)
-        Response.new(self, last_response, lambda { parse_response(body) }, :body => body)
+        raw_body ||= last_response.body
+
+        body = decompress(raw_body, last_response['content-encoding']) unless raw_body.nil?
+
+        unless body.nil?
+          body = encode_text(body, last_response['content-type'])
+
+          if decompress_content?
+            last_response.delete('content-encoding')
+            raw_body = body
+          end
+        end
+
+        Response.new(self, last_response, lambda { parse_response(body) }, body: raw_body)
       end
     end
 
-    # Inspired by Ruby 1.9
-    def handle_deflation
-      case last_response["content-encoding"]
-      when "gzip", "x-gzip"
-        body_io = StringIO.new(last_response.body)
-        last_response.body.replace Zlib::GzipReader.new(body_io).read
-        last_response.delete('content-encoding')
-      when "deflate"
-        last_response.body.replace Zlib::Inflate.inflate(last_response.body)
-        last_response.delete('content-encoding')
+    def handle_host_redirection
+      check_duplicate_location_header
+      redirect_path = options[:uri_adapter].parse(last_response['location']).normalize
+      return if redirect_path.relative? || path.host == redirect_path.host
+      @changed_hosts = true
+    end
+
+    def check_duplicate_location_header
+      location = last_response.get_fields('location')
+      if location.is_a?(Array) && location.count > 1
+        raise DuplicateLocationHeader.new(last_response)
       end
     end
 
+    def send_authorization_header?
+      !@changed_hosts
+    end
+
     def response_redirects?
       case last_response
-      when Net::HTTPMultipleChoice, # 300
-           Net::HTTPMovedPermanently, # 301
-           Net::HTTPFound, # 302
-           Net::HTTPSeeOther, # 303
-           Net::HTTPUseProxy, # 305
-           Net::HTTPTemporaryRedirect
+      when Net::HTTPNotModified # 304
+        false
+      when Net::HTTPRedirection
         options[:follow_redirects] && last_response.key?('location')
       end
     end
@@ -297,9 +364,10 @@ module HTTParty
 
     def capture_cookies(response)
       return unless response['Set-Cookie']
-      cookies_hash = HTTParty::CookieHash.new()
-      cookies_hash.add_cookies(options[:headers]['Cookie']) if options[:headers] && options[:headers]['Cookie']
+      cookies_hash = HTTParty::CookieHash.new
+      cookies_hash.add_cookies(options[:headers].to_hash['Cookie']) if options[:headers] && options[:headers].to_hash['Cookie']
       response.get_fields('Set-Cookie').each { |cookie| cookies_hash.add_cookies(cookie) }
+
       options[:headers] ||= {}
       options[:headers]['Cookie'] = cookies_hash.to_cookie_string
     end
@@ -316,15 +384,35 @@ module HTTParty
     def validate
       raise HTTParty::RedirectionTooDeep.new(last_response), 'HTTP redirects too deep' if options[:limit].to_i <= 0
       raise ArgumentError, 'only get, post, patch, put, delete, head, and options methods are supported' unless SupportedHTTPMethods.include?(http_method)
-      raise ArgumentError, ':headers must be a hash' if options[:headers] && !options[:headers].is_a?(Hash)
+      raise ArgumentError, ':headers must be a hash' if options[:headers] && !options[:headers].respond_to?(:to_hash)
       raise ArgumentError, 'only one authentication method, :basic_auth or :digest_auth may be used at a time' if options[:basic_auth] && options[:digest_auth]
-      raise ArgumentError, ':basic_auth must be a hash' if options[:basic_auth] && !options[:basic_auth].is_a?(Hash)
-      raise ArgumentError, ':digest_auth must be a hash' if options[:digest_auth] && !options[:digest_auth].is_a?(Hash)
-      raise ArgumentError, ':query must be hash if using HTTP Post' if post? && !options[:query].nil? && !options[:query].is_a?(Hash)
+      raise ArgumentError, ':basic_auth must be a hash' if options[:basic_auth] && !options[:basic_auth].respond_to?(:to_hash)
+      raise ArgumentError, ':digest_auth must be a hash' if options[:digest_auth] && !options[:digest_auth].respond_to?(:to_hash)
+      raise ArgumentError, ':query must be hash if using HTTP Post' if post? && !options[:query].nil? && !options[:query].respond_to?(:to_hash)
     end
 
     def post?
       Net::HTTP::Post == http_method
     end
+
+    def set_basic_auth_from_uri
+      if path.userinfo
+        username, password = path.userinfo.split(':')
+        options[:basic_auth] = {username: username, password: password}
+        @credentials_sent = true
+      end
+    end
+
+    def decompress(body, encoding)
+      Decompressor.new(body, encoding).decompress
+    end
+
+    def encode_text(text, content_type)
+      TextEncoder.new(
+        text,
+        content_type: content_type,
+        assume_utf16_is_big_endian: assume_utf16_is_big_endian
+      ).call
+    end
   end
 end

data/lib/httparty/response/headers.rb

@@ -1,31 +1,35 @@
+# frozen_string_literal: true
+
+require 'delegate'
+
 module HTTParty
   class Response #:nodoc:
-    class Headers
+    class Headers < ::SimpleDelegator
       include ::Net::HTTPHeader
 
-      def initialize(header = {})
-        @header = header
+      def initialize(header_values = nil)
+        @header = {}
+        if header_values
+          header_values.each_pair do |k,v|
+            if v.is_a?(Array)
+              v.each do |sub_v|
+                add_field(k, sub_v)
+              end
+            else
+              add_field(k, v)
+            end
+          end
+        end
+        super(@header)
       end
 
       def ==(other)
-        @header == other
-      end
-
-      def inspect
-        @header.inspect
-      end
-
-      def method_missing(name, *args, &block)
-        if @header.respond_to?(name)
-          @header.send(name, *args, &block)
-        else
-          super
+        if other.is_a?(::Net::HTTPHeader)
+          @header == other.instance_variable_get(:@header)
+        elsif other.is_a?(Hash)
+          @header == other || @header == Headers.new(other).instance_variable_get(:@header)
         end
       end
-
-      def respond_to?(method, include_all = false)
-        super || @header.respond_to?(method, include_all)
-      end
     end
   end
 end