rest-client 2.0.2 → 2.1.0.rc1

data/history.md

@@ -1,3 +1,36 @@
+# 2.1.0.rc1
+
+- Add a dependency on http-accept for parsing Content-Type charset headers.
+  This works around a bad memory leak introduced in Ruby 2.4.x (the leak is
+  probably a bug in MRI). (#615)
+- Use mime/types/columnar from mime-types 2.6.1+, which is leaner in memory
+  usage than the older storage model of mime-types. (#393)
+- Add `:log` option to individual requests. This allows users to set a log on a
+  per-request / per-resource basis instead of the kludgy global log. (#538)
+- Log request duration by tracking request start and end times. Make
+  `log_response` a method on the Response object, and ensure the `size` method
+  works on RawResponse objects. (#126)
+  - `# => 200 OK | text/html 1270 bytes, 0.08s`
+- Drop custom handling of compression and use built-in Net::HTTP support for
+  supported Content-Encodings like gzip and deflate. Don't set any explicit
+  `Accept-Encoding` header, rely instead on Net::HTTP defaults. (#597)
+  - Note: this changes behavior for compressed responses when using
+    `:raw_response => true`. Previously the raw response would not have been
+    uncompressed by rest-client, but now Net::HTTP will uncompress it.
+- The previous fix to avoid having Netrc username/password override an
+  Authorization header was case-sensitive and incomplete. Fix this by
+  respecting existing Authorization headers, regardless of letter case. (#550)
+- Handle ParamsArray payloads. Previously, rest-client would silently drop a
+  ParamsArray passed as the payload. Instead, automatically use
+  Payload::Multipart if the ParamsArray contains a file handle, or use
+  Payload::UrlEncoded if it doesn't. (#508)
+- Gracefully handle Payload objects (Payload::Base or subclasses) that are
+  passed as a payload argument. Previously, `Payload.generate` would wrap a
+  Payload object in Payload::Streamed, creating a pointlessly nested payload.
+  Also add a `closed?` method to Payload objects, and don't error in
+  `short_inspect` if `size` returns nil. (#603)
+- Test with an image in the public domain to avoid licensing complexity. (#607)
+
 # 2.0.2
 
 - Suppress the header override warning introduced in 2.0.1 if the value is the

data/lib/restclient.rb

@@ -2,7 +2,6 @@ require 'net/http'
 require 'openssl'
 require 'stringio'
 require 'uri'
-require 'zlib'
 
 require File.dirname(__FILE__) + '/restclient/version'
 require File.dirname(__FILE__) + '/restclient/platform'

data/lib/restclient/abstract_response.rb

@@ -5,12 +5,27 @@ module RestClient
 
   module AbstractResponse
 
-    attr_reader :net_http_res, :request
+    attr_reader :net_http_res, :request, :start_time, :end_time, :duration
 
     def inspect
       raise NotImplementedError.new('must override in subclass')
     end
 
+    # Logger from the request, potentially nil.
+    def log
+      request.log
+    end
+
+    def log_response
+      return unless log
+
+      code = net_http_res.code
+      res_name = net_http_res.class.to_s.gsub(/\ANet::HTTP/, '')
+      content_type = (net_http_res['Content-type'] || '').gsub(/;.*\z/, '')
+
+      log << "# => #{code} #{res_name} | #{content_type} #{size} bytes, #{sprintf('%.2f', duration)}s\n"
+    end
+
     # HTTP status code
     def code
       @code ||= @net_http_res.code.to_i
@@ -31,9 +46,20 @@ module RestClient
       @raw_headers ||= @net_http_res.to_hash
     end
 
-    def response_set_vars(net_http_res, request)
+    # @param [Net::HTTPResponse] net_http_res
+    # @param [RestClient::Request] request
+    # @param [Time] start_time
+    def response_set_vars(net_http_res, request, start_time)
       @net_http_res = net_http_res
       @request = request
+      @start_time = start_time
+      @end_time = Time.now
+
+      if @start_time
+        @duration = @end_time - @start_time
+      else
+        @duration = nil
+      end
 
       # prime redirection history
       history

data/lib/restclient/exceptions.rb

@@ -148,7 +148,7 @@ module RestClient
   end
 
   # Compatibility
-  class ExceptionWithResponse < Exception
+  class ExceptionWithResponse < RestClient::Exception
   end
 
   # The request failed with an error code not managed by the code
@@ -228,14 +228,14 @@ module RestClient
   # The server broke the connection prior to the request completing.  Usually
   # this means it crashed, or sometimes that your network connection was
   # severed before it could complete.
-  class ServerBrokeConnection < Exception
+  class ServerBrokeConnection < RestClient::Exception
     def initialize(message = 'Server broke connection')
       super nil, nil
       self.message = message
     end
   end
 
-  class SSLCertificateNotVerified < Exception
+  class SSLCertificateNotVerified < RestClient::Exception
     def initialize(message = 'SSL certificate not verified')
       super nil, nil
       self.message = message

data/lib/restclient/payload.rb

@@ -2,14 +2,22 @@ require 'tempfile'
 require 'securerandom'
 require 'stringio'
 
-require 'mime/types'
+begin
+  # Use mime/types/columnar if available, for reduced memory usage
+  require 'mime/types/columnar'
+rescue LoadError
+  require 'mime/types'
+end
 
 module RestClient
   module Payload
     extend self
 
     def generate(params)
-      if params.is_a?(String)
+      if params.is_a?(RestClient::Payload::Base)
+        # pass through Payload objects unchanged
+        params
+      elsif params.is_a?(String)
         Base.new(params)
       elsif params.is_a?(Hash)
         if params.delete(:multipart) == true || has_file?(params)
@@ -17,6 +25,12 @@ module RestClient
         else
           UrlEncoded.new(params)
         end
+      elsif params.is_a?(ParamsArray)
+        if _has_file?(params)
+          Multipart.new(params)
+        else
+          UrlEncoded.new(params)
+        end
       elsif params.respond_to?(:read)
         Streamed.new(params)
       else
@@ -76,12 +90,20 @@ module RestClient
         @stream.close unless @stream.closed?
       end
 
+      def closed?
+        @stream.closed?
+      end
+
       def to_s_inspect
         to_s.inspect
       end
 
       def short_inspect
-        (size > 500 ? "#{size} byte(s) length" : to_s_inspect)
+        if size && size > 500
+          "#{size} byte(s) length"
+        else
+          to_s_inspect
+        end
       end
 
     end
@@ -99,6 +121,9 @@ module RestClient
         end
       end
 
+      # TODO (breaks compatibility): ought to use mime_for() to autodetect the
+      # Content-Type for stream objects that have a filename.
+
       alias :length :size
     end
 

data/lib/restclient/raw_response.rb

@@ -13,25 +13,36 @@ module RestClient
 
     include AbstractResponse
 
-    attr_reader :file, :request
+    attr_reader :file, :request, :start_time, :end_time
 
     def inspect
       "<RestClient::RawResponse @code=#{code.inspect}, @file=#{file.inspect}, @request=#{request.inspect}>"
     end
 
-    def initialize(tempfile, net_http_res, request)
-      @net_http_res = net_http_res
+    # @param [Tempfile] tempfile The temporary file containing the body
+    # @param [Net::HTTPResponse] net_http_res
+    # @param [RestClient::Request] request
+    # @param [Time] start_time
+    def initialize(tempfile, net_http_res, request, start_time=nil)
       @file = tempfile
-      @request = request
+
+      # reopen the tempfile so we can read it
+      @file.open
+
+      response_set_vars(net_http_res, request, start_time)
     end
 
     def to_s
-      @file.open
+      body
+    end
+
+    def body
+      @file.rewind
       @file.read
     end
 
     def size
-      File.size file
+      file.size
     end
 
   end

data/lib/restclient/request.rb

@@ -1,9 +1,15 @@
 require 'tempfile'
-require 'mime/types'
 require 'cgi'
 require 'netrc'
 require 'set'
 
+begin
+  # Use mime/types/columnar if available, for reduced memory usage
+  require 'mime/types/columnar'
+rescue LoadError
+  require 'mime/types'
+end
+
 module RestClient
   # This class is used internally by RestClient to send the request, but you can also
   # call it directly if you'd like to use a method not supported by the
@@ -92,6 +98,12 @@ module RestClient
       @block_response = args[:block_response]
       @raw_response = args[:raw_response] || false
 
+      @stream_log_percent = args[:stream_log_percent] || 10
+      if @stream_log_percent <= 0 || @stream_log_percent > 100
+        raise ArgumentError.new(
+          "Invalid :stream_log_percent #{@stream_log_percent.inspect}")
+      end
+
       @proxy = args.fetch(:proxy) if args.include?(:proxy)
 
       @ssl_opts = {}
@@ -131,9 +143,10 @@ module RestClient
         end
       end
 
-      @tf = nil # If you are a raw request, this is your tempfile
+      @log = args[:log]
       @max_redirects = args[:max_redirects] || 10
       @processed_headers = make_headers headers
+      @processed_headers_lowercase = Hash[@processed_headers.map {|k, v| [k.downcase, v]}]
       @args = args
 
       @before_execution_proc = args[:before_execution_proc]
@@ -356,6 +369,13 @@ module RestClient
     #   - headers from the payload object (e.g. Content-Type, Content-Lenth)
     #   - cookie headers from #make_cookie_header
     #
+    # BUG: stringify_headers does not alter the capitalization of headers that
+    # are passed as strings, it only normalizes those passed as symbols. This
+    # behavior will probably remain for a while for compatibility, but it means
+    # that the warnings that attempt to detect accidental header overrides may
+    # not always work.
+    # https://github.com/rest-client/rest-client/issues/599
+    #
     # @param [Hash] user_headers User-provided headers to include
     #
     # @return [Hash<String, String>] A hash of HTTP headers => values
@@ -493,24 +513,6 @@ module RestClient
       cert_store
     end
 
-    def self.decode content_encoding, body
-      if (!body) || body.empty?
-        body
-      elsif content_encoding == 'gzip'
-        Zlib::GzipReader.new(StringIO.new(body)).read
-      elsif content_encoding == 'deflate'
-        begin
-          Zlib::Inflate.new.inflate body
-        rescue Zlib::DataError
-          # No luck with Zlib decompression. Let's try with raw deflate,
-          # like some broken web servers do.
-          Zlib::Inflate.new(-Zlib::MAX_WBITS).inflate body
-        end
-      else
-        body
-      end
-    end
-
     def redacted_uri
       if uri.password
         sanitized_uri = uri.dup
@@ -525,30 +527,29 @@ module RestClient
       redacted_uri.to_s
     end
 
+    # Default to the global logger if there's not a request-specific one
+    def log
+      @log || RestClient.log
+    end
+
     def log_request
-      return unless RestClient.log
+      return unless log
 
       out = []
 
       out << "RestClient.#{method} #{redacted_url.inspect}"
       out << payload.short_inspect if payload
       out << processed_headers.to_a.sort.map { |(k, v)| [k.inspect, v.inspect].join("=>") }.join(", ")
-      RestClient.log << out.join(', ') + "\n"
-    end
-
-    def log_response res
-      return unless RestClient.log
-
-      size = if @raw_response
-               File.size(@tf.path)
-             else
-               res.body.nil? ? 0 : res.body.size
-             end
-
-      RestClient.log << "# => #{res.code} #{res.class.to_s.gsub(/^Net::HTTP/, '')} | #{(res['Content-type'] || '').gsub(/;.*$/, '')} #{size} bytes\n"
+      log << out.join(', ') + "\n"
     end
 
     # Return a hash of headers whose keys are capitalized strings
+    #
+    # BUG: stringify_headers does not fix the capitalization of headers that
+    # are already Strings. Leaving this behavior as is for now for
+    # backwards compatibility.
+    # https://github.com/rest-client/rest-client/issues/599
+    #
     def stringify_headers headers
       headers.inject({}) do |result, (key, value)|
         if key.is_a? Symbol
@@ -573,10 +574,13 @@ module RestClient
       end
     end
 
+    # Default headers set by RestClient. In addition to these headers, servers
+    # will receive headers set by Net::HTTP, such as Accept-Encoding and Host.
+    #
+    # @return [Hash<Symbol, String>]
     def default_headers
       {
         :accept => '*/*',
-        :accept_encoding => 'gzip, deflate',
         :user_agent => RestClient::Platform.default_user_agent,
       }
     end
@@ -712,6 +716,9 @@ module RestClient
 
       log_request
 
+      start_time = Time.now
+      tempfile = nil
+
       net.start do |http|
         established_connection = true
 
@@ -719,10 +726,16 @@ module RestClient
           net_http_do_request(http, req, payload, &@block_response)
         else
           res = net_http_do_request(http, req, payload) { |http_response|
-            fetch_body(http_response)
+            if @raw_response
+              # fetch body into tempfile
+              tempfile = fetch_body_to_tempfile(http_response)
+            else
+              # fetch body
+              http_response.read_body
+            end
+            http_response
           }
-          log_response res
-          process_result res, & block
+          process_result(res, start_time, tempfile, &block)
         end
       end
     rescue EOFError
@@ -762,47 +775,56 @@ module RestClient
     end
 
     def setup_credentials(req)
-      req.basic_auth(user, password) if user && !headers.has_key?("Authorization")
+      if user && !@processed_headers_lowercase.include?('authorization')
+        req.basic_auth(user, password)
+      end
     end
 
-    def fetch_body(http_response)
-      if @raw_response
-        # Taken from Chef, which as in turn...
-        # Stolen from http://www.ruby-forum.com/topic/166423
-        # Kudos to _why!
-        @tf = Tempfile.new('rest-client.')
-        @tf.binmode
-        size, total = 0, http_response['Content-Length'].to_i
-        http_response.read_body do |chunk|
-          @tf.write chunk
-          size += chunk.size
-          if RestClient.log
-            if size == 0
-              RestClient.log << "%s %s done (0 length file)\n" % [@method, @url]
-            elsif total == 0
-              RestClient.log << "%s %s (zero content length)\n" % [@method, @url]
-            else
-              RestClient.log << "%s %s %d%% done (%d of %d)\n" % [@method, @url, (size * 100) / total, size, total]
+    def fetch_body_to_tempfile(http_response)
+      # Taken from Chef, which as in turn...
+      # Stolen from http://www.ruby-forum.com/topic/166423
+      # Kudos to _why!
+      tf = Tempfile.new('rest-client.')
+      tf.binmode
+
+      size = 0
+      total = http_response['Content-Length'].to_i
+      stream_log_bucket = nil
+
+      http_response.read_body do |chunk|
+        tf.write chunk
+        size += chunk.size
+        if log
+          if total == 0
+            log << "streaming %s %s (%d of unknown) [0 Content-Length]\n" % [@method.upcase, @url, size]
+          else
+            percent = (size * 100) / total
+            current_log_bucket, _ = percent.divmod(@stream_log_percent)
+            if current_log_bucket != stream_log_bucket
+              stream_log_bucket = current_log_bucket
+              log << "streaming %s %s %d%% done (%d of %d)\n" % [@method.upcase, @url, (size * 100) / total, size, total]
             end
           end
         end
-        @tf.close
-        @tf
-      else
-        http_response.read_body
       end
-      http_response
+      tf.close
+      tf
     end
 
-    def process_result res, & block
+    # @param res The Net::HTTP response object
+    # @param start_time [Time] Time of request start
+    def process_result(res, start_time, tempfile=nil, &block)
       if @raw_response
-        # We don't decode raw requests
-        response = RawResponse.new(@tf, res, self)
+        unless tempfile
+          raise ArgumentError.new('tempfile is required')
+        end
+        response = RawResponse.new(tempfile, res, self, start_time)
       else
-        decoded = Request.decode(res['content-encoding'], res.body)
-        response = Response.create(decoded, res, self)
+        response = Response.create(res.body, res, self, start_time)
       end
 
+      response.log_response
+
       if block_given?
         block.call(response, self, res, & block)
       else