rest-client 2.0.2 → 2.1.0

data/Rakefile

@@ -38,12 +38,11 @@ desc 'Regenerate authors file'
 task :authors do
   Dir.chdir(File.dirname(__FILE__)) do
     File.open('AUTHORS', 'w') do |f|
-      f.write( <<-EOM
+      f.write <<-EOM
 The Ruby REST Client would not be what it is today without the help of
 the following kind souls:
 
       EOM
-      )
     end
 
     sh 'git shortlog -s | cut -f 2 >> AUTHORS'
@@ -70,8 +69,8 @@ namespace :all do
   task :build => ['ruby:build'] + \
     WindowsPlatforms.map {|p| "windows:#{p}:build"}
 
-  desc "Create tag v#{RestClient::VERSION} and for all platforms build and push " \
-    "rest-client #{RestClient::VERSION} to Rubygems"
+  desc "Create tag v#{RestClient::VERSION} and for all platforms build and " \
+    "push rest-client #{RestClient::VERSION} to Rubygems"
   task :release => ['build', 'ruby:release'] + \
     WindowsPlatforms.map {|p| "windows:#{p}:push"}
 
@@ -130,3 +129,12 @@ Rake::RDocTask.new do |t|
   t.rdoc_files.include('README.md')
   t.rdoc_files.include('lib/*.rb')
 end
+
+############################
+
+require 'rubocop/rake_task'
+
+RuboCop::RakeTask.new(:rubocop) do |t|
+  t.options = ['--display-cop-names']
+end
+alias_task(:lint, :rubocop)

data/history.md

@@ -1,3 +1,40 @@
+# 2.1.0
+
+- Add a dependency on http-accept for parsing Content-Type charset headers.
+  This works around a bad memory leak introduced in MRI Ruby 2.4.0 and fixed in
+  Ruby 2.4.2. (#615)
+- Use mime/types/columnar from mime-types 2.6.1+, which is leaner in memory
+  usage than the older storage model of mime-types. (#393)
+- Add `:log` option to individual requests. This allows users to set a log on a
+  per-request / per-resource basis instead of the kludgy global log. (#538)
+- Log request duration by tracking request start and end times. Make
+  `log_response` a method on the Response object, and ensure the `size` method
+  works on RawResponse objects. (#126)
+  - `# => 200 OK | text/html 1270 bytes, 0.08s`
+  - Also add a new `:stream_log_percent` parameter, which is applicable only
+    when `:raw_response => true` is set. This causes progress logs to be
+    emitted only on every N% (default 10%) of the total download size rather
+    than on every chunk.
+- Drop custom handling of compression and use built-in Net::HTTP support for
+  supported Content-Encodings like gzip and deflate. Don't set any explicit
+  `Accept-Encoding` header, rely instead on Net::HTTP defaults. (#597)
+  - Note: this changes behavior for compressed responses when using
+    `:raw_response => true`. Previously the raw response would not have been
+    uncompressed by rest-client, but now Net::HTTP will uncompress it.
+- The previous fix to avoid having Netrc username/password override an
+  Authorization header was case-sensitive and incomplete. Fix this by
+  respecting existing Authorization headers, regardless of letter case. (#550)
+- Handle ParamsArray payloads. Previously, rest-client would silently drop a
+  ParamsArray passed as the payload. Instead, automatically use
+  Payload::Multipart if the ParamsArray contains a file handle, or use
+  Payload::UrlEncoded if it doesn't. (#508)
+- Gracefully handle Payload objects (Payload::Base or subclasses) that are
+  passed as a payload argument. Previously, `Payload.generate` would wrap a
+  Payload object in Payload::Streamed, creating a pointlessly nested payload.
+  Also add a `closed?` method to Payload objects, and don't error in
+  `short_inspect` if `size` returns nil. (#603)
+- Test with an image in the public domain to avoid licensing complexity. (#607)
+
 # 2.0.2
 
 - Suppress the header override warning introduced in 2.0.1 if the value is the
@@ -179,6 +216,22 @@ release:
   - Disable timeouts with :timeout => nil rather than :timeout => -1
   - Drop all Net::HTTP monkey patches
 
+# 1.6.14
+
+- This release is unchanged from 1.6.9. It was published in order to supersede
+  the malicious 1.6.10-13 versions, even for users who are still pinning to the
+  legacy 1.6.x series. All users are encouraged to upgrade to rest-client 2.x.
+
+# 1.6.10, 1.6.11, 1.6.12, 1.6.13 (CVE-2019-15224)
+
+- These versions were pushed by a malicious actor and included a backdoor permitting
+  remote code execution in Rails environments. (#713)
+- They were live for about five days before being yanked.
+
+# 1.6.9
+
+- Move rdoc to a development dependency
+
 # 1.6.8
 
 - The 1.6.x series will be the last to support Ruby 1.8.7

data/lib/restclient.rb

@@ -2,7 +2,6 @@ require 'net/http'
 require 'openssl'
 require 'stringio'
 require 'uri'
-require 'zlib'
 
 require File.dirname(__FILE__) + '/restclient/version'
 require File.dirname(__FILE__) + '/restclient/platform'

data/lib/restclient/abstract_response.rb

@@ -5,12 +5,27 @@ module RestClient
 
   module AbstractResponse
 
-    attr_reader :net_http_res, :request
+    attr_reader :net_http_res, :request, :start_time, :end_time, :duration
 
     def inspect
       raise NotImplementedError.new('must override in subclass')
     end
 
+    # Logger from the request, potentially nil.
+    def log
+      request.log
+    end
+
+    def log_response
+      return unless log
+
+      code = net_http_res.code
+      res_name = net_http_res.class.to_s.gsub(/\ANet::HTTP/, '')
+      content_type = (net_http_res['Content-type'] || '').gsub(/;.*\z/, '')
+
+      log << "# => #{code} #{res_name} | #{content_type} #{size} bytes, #{sprintf('%.2f', duration)}s\n"
+    end
+
     # HTTP status code
     def code
       @code ||= @net_http_res.code.to_i
@@ -31,9 +46,20 @@ module RestClient
       @raw_headers ||= @net_http_res.to_hash
     end
 
-    def response_set_vars(net_http_res, request)
+    # @param [Net::HTTPResponse] net_http_res
+    # @param [RestClient::Request] request
+    # @param [Time] start_time
+    def response_set_vars(net_http_res, request, start_time)
       @net_http_res = net_http_res
       @request = request
+      @start_time = start_time
+      @end_time = Time.now
+
+      if @start_time
+        @duration = @end_time - @start_time
+      else
+        @duration = nil
+      end
 
       # prime redirection history
       history

data/lib/restclient/exceptions.rb

@@ -148,7 +148,7 @@ module RestClient
   end
 
   # Compatibility
-  class ExceptionWithResponse < Exception
+  class ExceptionWithResponse < RestClient::Exception
   end
 
   # The request failed with an error code not managed by the code
@@ -228,14 +228,14 @@ module RestClient
   # The server broke the connection prior to the request completing.  Usually
   # this means it crashed, or sometimes that your network connection was
   # severed before it could complete.
-  class ServerBrokeConnection < Exception
+  class ServerBrokeConnection < RestClient::Exception
     def initialize(message = 'Server broke connection')
       super nil, nil
       self.message = message
     end
   end
 
-  class SSLCertificateNotVerified < Exception
+  class SSLCertificateNotVerified < RestClient::Exception
     def initialize(message = 'SSL certificate not verified')
       super nil, nil
       self.message = message

data/lib/restclient/payload.rb

@@ -2,14 +2,22 @@ require 'tempfile'
 require 'securerandom'
 require 'stringio'
 
-require 'mime/types'
+begin
+  # Use mime/types/columnar if available, for reduced memory usage
+  require 'mime/types/columnar'
+rescue LoadError
+  require 'mime/types'
+end
 
 module RestClient
   module Payload
     extend self
 
     def generate(params)
-      if params.is_a?(String)
+      if params.is_a?(RestClient::Payload::Base)
+        # pass through Payload objects unchanged
+        params
+      elsif params.is_a?(String)
         Base.new(params)
       elsif params.is_a?(Hash)
         if params.delete(:multipart) == true || has_file?(params)
@@ -17,6 +25,12 @@ module RestClient
         else
           UrlEncoded.new(params)
         end
+      elsif params.is_a?(ParamsArray)
+        if _has_file?(params)
+          Multipart.new(params)
+        else
+          UrlEncoded.new(params)
+        end
       elsif params.respond_to?(:read)
         Streamed.new(params)
       else
@@ -76,12 +90,20 @@ module RestClient
         @stream.close unless @stream.closed?
       end
 
+      def closed?
+        @stream.closed?
+      end
+
       def to_s_inspect
         to_s.inspect
       end
 
       def short_inspect
-        (size > 500 ? "#{size} byte(s) length" : to_s_inspect)
+        if size && size > 500
+          "#{size} byte(s) length"
+        else
+          to_s_inspect
+        end
       end
 
     end
@@ -99,6 +121,9 @@ module RestClient
         end
       end
 
+      # TODO (breaks compatibility): ought to use mime_for() to autodetect the
+      # Content-Type for stream objects that have a filename.
+
       alias :length :size
     end
 
@@ -119,7 +144,7 @@ module RestClient
       def build_stream(params)
         b = '--' + boundary
 
-        @stream = Tempfile.new("RESTClient.Stream.#{rand(1000)}")
+        @stream = Tempfile.new('rest-client.multipart.')
         @stream.binmode
         @stream.write(b + EOL)
 

data/lib/restclient/raw_response.rb

@@ -13,25 +13,36 @@ module RestClient
 
     include AbstractResponse
 
-    attr_reader :file, :request
+    attr_reader :file, :request, :start_time, :end_time
 
     def inspect
       "<RestClient::RawResponse @code=#{code.inspect}, @file=#{file.inspect}, @request=#{request.inspect}>"
     end
 
-    def initialize(tempfile, net_http_res, request)
-      @net_http_res = net_http_res
+    # @param [Tempfile] tempfile The temporary file containing the body
+    # @param [Net::HTTPResponse] net_http_res
+    # @param [RestClient::Request] request
+    # @param [Time] start_time
+    def initialize(tempfile, net_http_res, request, start_time=nil)
       @file = tempfile
-      @request = request
+
+      # reopen the tempfile so we can read it
+      @file.open
+
+      response_set_vars(net_http_res, request, start_time)
     end
 
     def to_s
-      @file.open
+      body
+    end
+
+    def body
+      @file.rewind
       @file.read
     end
 
     def size
-      File.size file
+      file.size
     end
 
   end

data/lib/restclient/request.rb

@@ -1,9 +1,15 @@
 require 'tempfile'
-require 'mime/types'
 require 'cgi'
 require 'netrc'
 require 'set'
 
+begin
+  # Use mime/types/columnar if available, for reduced memory usage
+  require 'mime/types/columnar'
+rescue LoadError
+  require 'mime/types'
+end
+
 module RestClient
   # This class is used internally by RestClient to send the request, but you can also
   # call it directly if you'd like to use a method not supported by the
@@ -22,6 +28,11 @@ module RestClient
   # * :user and :password for basic auth, will be replaced by a user/password available in the :url
   # * :block_response call the provided block with the HTTPResponse as parameter
   # * :raw_response return a low-level RawResponse instead of a Response
+  # * :log Set the log for this request only, overriding RestClient.log, if
+  #      any.
+  # * :stream_log_percent (Only relevant with :raw_response => true) Customize
+  #     the interval at which download progress is logged. Defaults to every
+  #     10% complete.
   # * :max_redirects maximum number of redirections (default to 10)
   # * :proxy An HTTP proxy URI to use for this request. Any value here
   #   (including nil) will override RestClient.proxy.
@@ -92,6 +103,12 @@ module RestClient
       @block_response = args[:block_response]
       @raw_response = args[:raw_response] || false
 
+      @stream_log_percent = args[:stream_log_percent] || 10
+      if @stream_log_percent <= 0 || @stream_log_percent > 100
+        raise ArgumentError.new(
+          "Invalid :stream_log_percent #{@stream_log_percent.inspect}")
+      end
+
       @proxy = args.fetch(:proxy) if args.include?(:proxy)
 
       @ssl_opts = {}
@@ -131,9 +148,10 @@ module RestClient
         end
       end
 
-      @tf = nil # If you are a raw request, this is your tempfile
+      @log = args[:log]
       @max_redirects = args[:max_redirects] || 10
       @processed_headers = make_headers headers
+      @processed_headers_lowercase = Hash[@processed_headers.map {|k, v| [k.downcase, v]}]
       @args = args
 
       @before_execution_proc = args[:before_execution_proc]
@@ -356,6 +374,13 @@ module RestClient
     #   - headers from the payload object (e.g. Content-Type, Content-Lenth)
     #   - cookie headers from #make_cookie_header
     #
+    # BUG: stringify_headers does not alter the capitalization of headers that
+    # are passed as strings, it only normalizes those passed as symbols. This
+    # behavior will probably remain for a while for compatibility, but it means
+    # that the warnings that attempt to detect accidental header overrides may
+    # not always work.
+    # https://github.com/rest-client/rest-client/issues/599
+    #
     # @param [Hash] user_headers User-provided headers to include
     #
     # @return [Hash<String, String>] A hash of HTTP headers => values
@@ -493,24 +518,6 @@ module RestClient
       cert_store
     end
 
-    def self.decode content_encoding, body
-      if (!body) || body.empty?
-        body
-      elsif content_encoding == 'gzip'
-        Zlib::GzipReader.new(StringIO.new(body)).read
-      elsif content_encoding == 'deflate'
-        begin
-          Zlib::Inflate.new.inflate body
-        rescue Zlib::DataError
-          # No luck with Zlib decompression. Let's try with raw deflate,
-          # like some broken web servers do.
-          Zlib::Inflate.new(-Zlib::MAX_WBITS).inflate body
-        end
-      else
-        body
-      end
-    end
-
     def redacted_uri
       if uri.password
         sanitized_uri = uri.dup
@@ -525,30 +532,29 @@ module RestClient
       redacted_uri.to_s
     end
 
+    # Default to the global logger if there's not a request-specific one
+    def log
+      @log || RestClient.log
+    end
+
     def log_request
-      return unless RestClient.log
+      return unless log
 
       out = []
 
       out << "RestClient.#{method} #{redacted_url.inspect}"
       out << payload.short_inspect if payload
       out << processed_headers.to_a.sort.map { |(k, v)| [k.inspect, v.inspect].join("=>") }.join(", ")
-      RestClient.log << out.join(', ') + "\n"
-    end
-
-    def log_response res
-      return unless RestClient.log
-
-      size = if @raw_response
-               File.size(@tf.path)
-             else
-               res.body.nil? ? 0 : res.body.size
-             end
-
-      RestClient.log << "# => #{res.code} #{res.class.to_s.gsub(/^Net::HTTP/, '')} | #{(res['Content-type'] || '').gsub(/;.*$/, '')} #{size} bytes\n"
+      log << out.join(', ') + "\n"
     end
 
     # Return a hash of headers whose keys are capitalized strings
+    #
+    # BUG: stringify_headers does not fix the capitalization of headers that
+    # are already Strings. Leaving this behavior as is for now for
+    # backwards compatibility.
+    # https://github.com/rest-client/rest-client/issues/599
+    #
     def stringify_headers headers
       headers.inject({}) do |result, (key, value)|
         if key.is_a? Symbol
@@ -573,10 +579,13 @@ module RestClient
       end
     end
 
+    # Default headers set by RestClient. In addition to these headers, servers
+    # will receive headers set by Net::HTTP, such as Accept-Encoding and Host.
+    #
+    # @return [Hash<Symbol, String>]
     def default_headers
       {
         :accept => '*/*',
-        :accept_encoding => 'gzip, deflate',
         :user_agent => RestClient::Platform.default_user_agent,
       }
     end
@@ -712,6 +721,9 @@ module RestClient
 
       log_request
 
+      start_time = Time.now
+      tempfile = nil
+
       net.start do |http|
         established_connection = true
 
@@ -719,10 +731,16 @@ module RestClient
           net_http_do_request(http, req, payload, &@block_response)
         else
           res = net_http_do_request(http, req, payload) { |http_response|
-            fetch_body(http_response)
+            if @raw_response
+              # fetch body into tempfile
+              tempfile = fetch_body_to_tempfile(http_response)
+            else
+              # fetch body
+              http_response.read_body
+            end
+            http_response
           }
-          log_response res
-          process_result res, & block
+          process_result(res, start_time, tempfile, &block)
         end
       end
     rescue EOFError
@@ -762,47 +780,56 @@ module RestClient
     end
 
     def setup_credentials(req)
-      req.basic_auth(user, password) if user && !headers.has_key?("Authorization")
+      if user && !@processed_headers_lowercase.include?('authorization')
+        req.basic_auth(user, password)
+      end
     end
 
-    def fetch_body(http_response)
-      if @raw_response
-        # Taken from Chef, which as in turn...
-        # Stolen from http://www.ruby-forum.com/topic/166423
-        # Kudos to _why!
-        @tf = Tempfile.new('rest-client.')
-        @tf.binmode
-        size, total = 0, http_response['Content-Length'].to_i
-        http_response.read_body do |chunk|
-          @tf.write chunk
-          size += chunk.size
-          if RestClient.log
-            if size == 0
-              RestClient.log << "%s %s done (0 length file)\n" % [@method, @url]
-            elsif total == 0
-              RestClient.log << "%s %s (zero content length)\n" % [@method, @url]
-            else
-              RestClient.log << "%s %s %d%% done (%d of %d)\n" % [@method, @url, (size * 100) / total, size, total]
+    def fetch_body_to_tempfile(http_response)
+      # Taken from Chef, which as in turn...
+      # Stolen from http://www.ruby-forum.com/topic/166423
+      # Kudos to _why!
+      tf = Tempfile.new('rest-client.')
+      tf.binmode
+
+      size = 0
+      total = http_response['Content-Length'].to_i
+      stream_log_bucket = nil
+
+      http_response.read_body do |chunk|
+        tf.write chunk
+        size += chunk.size
+        if log
+          if total == 0
+            log << "streaming %s %s (%d of unknown) [0 Content-Length]\n" % [@method.upcase, @url, size]
+          else
+            percent = (size * 100) / total
+            current_log_bucket, _ = percent.divmod(@stream_log_percent)
+            if current_log_bucket != stream_log_bucket
+              stream_log_bucket = current_log_bucket
+              log << "streaming %s %s %d%% done (%d of %d)\n" % [@method.upcase, @url, (size * 100) / total, size, total]
             end
           end
         end
-        @tf.close
-        @tf
-      else
-        http_response.read_body
       end
-      http_response
+      tf.close
+      tf
     end
 
-    def process_result res, & block
+    # @param res The Net::HTTP response object
+    # @param start_time [Time] Time of request start
+    def process_result(res, start_time, tempfile=nil, &block)
       if @raw_response
-        # We don't decode raw requests
-        response = RawResponse.new(@tf, res, self)
+        unless tempfile
+          raise ArgumentError.new('tempfile is required')
+        end
+        response = RawResponse.new(tempfile, res, self, start_time)
       else
-        decoded = Request.decode(res['content-encoding'], res.body)
-        response = Response.create(decoded, res, self)
+        response = Response.create(res.body, res, self, start_time)
       end
 
+      response.log_response
+
       if block_given?
         block.call(response, self, res, & block)
       else