rest-client 1.6.1 → 1.6.14

data/history.md

@@ -1,3 +1,68 @@
+# 1.6.14
+
+- This release is unchanged from 1.6.9. It was published in order to supersede
+  the malicious 1.6.10-13 versions, even for users who are still pinning to the
+  legacy 1.6.x series. All users are encouraged to upgrade to rest-client 2.x.
+
+# 1.6.10, 1.6.11, 1.6.12, 1.6.13 (CVE-2019-15224)
+
+- These versions were pushed by a malicious actor and included a backdoor permitting
+  remote code execution in Rails environments.
+- They were live for about five days before being yanked.
+  https://github.com/rest-client/rest-client/issues/713
+
+# 1.6.9
+
+- Move rdoc to a development dependency
+
+# 1.6.8
+
+- The 1.6.x series will be the last to support Ruby 1.8.7
+- Pin mime-types to < 2.0 to maintain Ruby 1.8.7 support
+- Add Gemfile, AUTHORS, add license to gemspec
+- Point homepage at https://github.com/rest-client/rest-client
+- Clean up and fix various tests and ruby warnings
+- Backport `ssl_verify_callback` functionality from 1.7.0
+
+# 1.6.7
+
+- rebuild with 1.8.7 to avoid https://github.com/rubygems/rubygems/pull/57
+
+# 1.6.6
+
+- 1.6.5 was yanked
+
+# 1.6.5
+
+- RFC6265 requires single SP after ';' for separating parameters pairs in the 'Cookie:' header (patch provided by Hiroshi Nakamura)
+- enable url parameters for all actions
+- detect file parameters in arrays
+- allow disabling the timeouts by passing -1 (patch provided by Sven Böhm)
+
+# 1.6.4
+
+- fix restclient script compatibility with 1.9.2
+- fix unlinking temp file (patch provided by Evan Smith)
+- monkeypatching ruby for http patch method (patch provided by Syl Turner)
+
+# 1.6.3
+
+- 1.6.2 was yanked
+
+# 1.6.2
+
+- add support for HEAD in resources (patch provided by tpresa)
+- fix shell for 1.9.2
+- workaround when some gem monkeypatch net/http (patch provided by Ian Warshak)
+- DELETE requests should process parameters just like GET and HEAD
+- adding :block_response parameter for manual processing
+- limit number of redirections (patch provided by Chris Dinn)
+- close and unlink the temp file created by playload (patch provided by Chris Green)
+- make gemspec Rubygems 1.8 compatible (patch provided by David Backeus)
+- added RestClient.reset_before_execution_procs (patch provided by Cloudify)
+- added PATCH method (patch provided by Jeff Remer)
+- hack for HTTP servers that use raw DEFLATE compression, see http://www.ruby-forum.com/topic/136825 (path provided by James Reeves)
+
 # 1.6.1
 
 - add response body in Exception#inspect
@@ -6,6 +71,7 @@
 - block passing in Resource#[] (patch provided by Niko Dittmann)
 - cookies set in a response should be kept in a redirect
 - HEAD requests should process parameters just like GET (patch provided by Rob Eanes)
+- exception message should never be nil (patch provided by Michael Klett)
 
 # 1.6.0
 
@@ -91,4 +157,4 @@ The only breaking change should be the exception classes, but as the new classes
 
 All changes exept the last one should be fully compatible with the previous version.
 
-NOTE: due to a dependency problem and to the last change, heroku users should update their heroku gem to >= 1.5.3 to be able to use this version.
+NOTE: due to a dependency problem and to the last change, heroku users should update their heroku gem to >= 1.5.3 to be able to use this version.

data/lib/restclient.rb

@@ -9,6 +9,8 @@ rescue LoadError => e
   raise LoadError, "no such file to load -- net/https. Try running apt-get install libopenssl-ruby"
 end
 
+require File.dirname(__FILE__) + '/restclient/version'
+require File.dirname(__FILE__) + '/restclient/platform'
 require File.dirname(__FILE__) + '/restclient/exceptions'
 require File.dirname(__FILE__) + '/restclient/request'
 require File.dirname(__FILE__) + '/restclient/abstract_response'
@@ -72,6 +74,10 @@ module RestClient
     Request.execute(:method => :post, :url => url, :payload => payload, :headers => headers, &block)
   end
 
+  def self.patch(url, payload, headers={}, &block)
+    Request.execute(:method => :patch, :url => url, :payload => payload, :headers => headers, &block)
+  end
+
   def self.put(url, payload, headers={}, &block)
     Request.execute(:method => :put, :url => url, :payload => payload, :headers => headers, &block)
   end
@@ -99,12 +105,6 @@ module RestClient
     @@log = create_log log
   end
 
-  def self.version
-    version_path = File.dirname(__FILE__) + "/../VERSION"
-    return File.read(version_path).chomp if File.file?(version_path)
-    "0.0.0"
-  end
-
   # Create a log that respond to << like a logger
   # param can be 'stdout', 'stderr', a string (then we will log to that file) or a logger (then we return it)
   def self.create_log param
@@ -158,6 +158,11 @@ module RestClient
     @@before_execution_procs << proc
   end
 
+  # Reset the procs to be called before each request is executed.
+  def self.reset_before_execution_procs
+    @@before_execution_procs = []
+  end
+
   def self.before_execution_procs # :nodoc:
     @@before_execution_procs
   end

data/lib/restclient/abstract_response.rb

@@ -31,7 +31,7 @@ module RestClient
 
     # Return the default behavior corresponding to the response code:
     # the response itself for code in 200..206, redirection for 301, 302 and 307 in get and head cases, redirection for 303 and an exception in other cases
-    def return! request  = nil, result = nil, & block
+    def return! request = nil, result = nil, & block
       if (200..207).include? code
         self
       elsif [301, 302, 307].include? code
@@ -47,7 +47,7 @@ module RestClient
       elsif Exceptions::EXCEPTIONS_MAP[code]
         raise Exceptions::EXCEPTIONS_MAP[code].new(self, code)
       else
-        raise RequestFailed self
+        raise RequestFailed.new(self, code)
       end
     end
 
@@ -67,9 +67,13 @@ module RestClient
       end
       args[:url] = url
       if request
+        if request.max_redirects == 0
+          raise MaxRedirectsReached
+        end
         args[:password] = request.password
         args[:user] = request.user
         args[:headers] = request.headers
+        args[:max_redirects] = request.max_redirects - 1
         # pass any cookie set in the result
         if result && result['set-cookie']
           args[:headers][:cookies] = (args[:headers][:cookies] || {}).merge(parse_cookie(result['set-cookie']))

data/lib/restclient/exceptions.rb

@@ -21,7 +21,7 @@ module RestClient
               305 => 'Use Proxy', # http/1.1
               306 => 'Switch Proxy', # no longer used
               307 => 'Temporary Redirect', # http/1.1
-              
+
               400 => 'Bad Request',
               401 => 'Unauthorized',
               402 => 'Payment Required',
@@ -46,7 +46,7 @@ module RestClient
               423 => 'Locked', #WebDAV
               424 => 'Failed Dependency', #WebDAV
               425 => 'Unordered Collection', #WebDAV
-              426 => 'Upgrade Required', 
+              426 => 'Upgrade Required',
               449 => 'Retry With', #Microsoft
               450 => 'Blocked By Windows Parental Controls', #Microsoft
 
@@ -80,10 +80,12 @@ module RestClient
   # For example, the entire result body (which is
   # probably an HTML error page) is e.response.
   class Exception < RuntimeError
-    attr_accessor :message, :response
+    attr_accessor :response
+    attr_writer   :message
 
     def initialize response = nil, initial_response_code = nil
       @response = response
+      @message = nil
       @initial_response_code = initial_response_code
 
       # compatibility: this make the exception behave like a Net::HTTPResponse
@@ -111,6 +113,10 @@ module RestClient
       inspect
     end
 
+    def message
+      @message || self.class.name
+    end
+
   end
 
   # Compatibility
@@ -149,7 +155,9 @@ module RestClient
   # A redirect was encountered; caught by execute to retry with the new url.
   class Redirect < Exception
 
-    message = 'Redirect'
+    def message
+      'Redirect'
+    end
 
     attr_accessor :url
 
@@ -158,11 +166,20 @@ module RestClient
     end
   end
 
+  class MaxRedirectsReached < Exception
+    def message
+      'Maximum number of redirect reached'
+    end
+  end
+
   # The server broke the connection prior to the request completing.  Usually
   # this means it crashed, or sometimes that your network connection was
   # severed before it could complete.
   class ServerBrokeConnection < Exception
-    message = 'Server broke connection'
+    def initialize(message = 'Server broke connection')
+      super nil, nil
+      self.message = message
+    end
   end
 
   class SSLCertificateNotVerified < Exception

data/lib/restclient/net_http_ext.rb

@@ -1,12 +1,46 @@
-#
-# Replace the request method in Net::HTTP to sniff the body type
-# and set the stream if appropriate
-#
-# Taken from:	
-# http://www.missiondata.com/blog/ruby/29/streaming-data-to-s3-with-ruby/
-
 module Net
   class HTTP
+
+  # Adding the patch method if it doesn't exist (rest-client issue: https://github.com/archiloque/rest-client/issues/79)
+  if !defined?(Net::HTTP::Patch)
+    # Code taken from this commit: https://github.com/ruby/ruby/commit/ab70e53ac3b5102d4ecbe8f38d4f76afad29d37d#lib/net/http.rb
+    class Protocol
+      # Sends a PATCH request to the +path+ and gets a response,
+      # as an HTTPResponse object.
+      def patch(path, data, initheader = nil, dest = nil, &block) # :yield: +body_segment+
+        send_entity(path, data, initheader, dest, Patch, &block)
+      end
+
+      # Executes a request which uses a representation
+      # and returns its body.
+      def send_entity(path, data, initheader, dest, type, &block)
+        res = nil
+        request(type.new(path, initheader), data) {|r|
+          r.read_body dest, &block
+          res = r
+        }
+        unless @newimpl
+          res.value
+          return res, res.body
+        end
+        res
+      end
+    end
+
+    class Patch < HTTPRequest
+      METHOD = 'PATCH'
+      REQUEST_HAS_BODY = true
+      RESPONSE_HAS_BODY = true
+    end
+  end
+
+    #
+    # Replace the request method in Net::HTTP to sniff the body type
+    # and set the stream if appropriate
+    #
+    # Taken from:
+    # http://www.missiondata.com/blog/ruby/29/streaming-data-to-s3-with-ruby/
+
     alias __request__ request
 
     def request(req, body=nil, &block)

data/lib/restclient/payload.rb

@@ -9,14 +9,14 @@ module RestClient
     def generate(params)
       if params.is_a?(String)
         Base.new(params)
-      elsif params.respond_to?(:read)
-        Streamed.new(params)
-      elsif params
+      elsif params.is_a?(Hash)
         if params.delete(:multipart) == true || has_file?(params)
           Multipart.new(params)
         else
           UrlEncoded.new(params)
         end
+      elsif params.respond_to?(:read)
+        Streamed.new(params)
       else
         nil
       end
@@ -27,6 +27,21 @@ module RestClient
         case v
           when Hash
             has_file?(v)
+          when Array
+            has_file_array?(v)
+          else
+            v.respond_to?(:path) && v.respond_to?(:read)
+        end
+      end
+    end
+
+    def has_file_array?(params)
+      params.any? do |v|
+        case v
+          when Hash
+            has_file?(v)
+          when Array
+            has_file_array?(v)
           else
             v.respond_to?(:path) && v.respond_to?(:read)
         end
@@ -70,7 +85,7 @@ module RestClient
         result = []
         value.each do |elem|
           if elem.is_a? Hash
-            result +=  flatten_params(elem, calculated_key)
+            result += flatten_params(elem, calculated_key)
           elsif elem.is_a? Array
             result += flatten_params_array(elem, calculated_key)
           else
@@ -91,7 +106,7 @@ module RestClient
       alias :length :size
 
       def close
-        @stream.close
+        @stream.close unless @stream.closed?
       end
 
       def inspect
@@ -132,12 +147,17 @@ module RestClient
 
       # for UrlEncoded escape the keys
       def handle_key key
-        URI.escape(key.to_s, Regexp.new("[^#{URI::PATTERN::UNRESERVED}]"))
+        parser.escape(key.to_s, Regexp.new("[^#{URI::PATTERN::UNRESERVED}]"))
       end
 
       def headers
         super.merge({'Content-Type' => 'application/x-www-form-urlencoded'})
       end
+
+      private
+        def parser
+          URI.const_defined?(:Parser) ? URI::Parser.new : URI
+        end
     end
 
     class Multipart < Base
@@ -190,7 +210,7 @@ module RestClient
             s.write(data)
           end
         ensure
-          v.close
+          v.close if v.respond_to?(:close)
         end
       end
 
@@ -213,7 +233,7 @@ module RestClient
       end
 
       def close
-        @stream.close
+        @stream.close!
       end
     end
   end

data/lib/restclient/platform.rb

@@ -0,0 +1,29 @@
+module RestClient
+  module Platform
+    # Return true if we are running on a darwin-based Ruby platform. This will
+    # be false for jruby even on OS X.
+    #
+    # @return [Boolean]
+    def self.mac?
+      RUBY_PLATFORM.include?('darwin')
+    end
+
+    # Return true if we are running on Windows.
+    #
+    # @return [Boolean]
+    #
+    def self.windows?
+      # Ruby only sets File::ALT_SEPARATOR on Windows, and the Ruby standard
+      # library uses that to test what platform it's on.
+      !!File::ALT_SEPARATOR
+    end
+
+    # Return true if we are running on jruby.
+    #
+    # @return [Boolean]
+    #
+    def self.jruby?
+      RUBY_PLATFORM == 'java'
+    end
+  end
+end

data/lib/restclient/request.rb

@@ -16,16 +16,20 @@ module RestClient
   # * :headers a hash containing the request headers
   # * :cookies will replace possible cookies in the :headers
   # * :user and :password for basic auth, will be replaced by a user/password available in the :url
+  # * :block_response call the provided block with the HTTPResponse as parameter
   # * :raw_response return a low-level RawResponse instead of a Response
+  # * :max_redirects maximum number of redirections (default to 10)
   # * :verify_ssl enable ssl verification, possible values are constants from OpenSSL::SSL
-  # * :timeout and :open_timeout
+  # * :timeout and :open_timeout passing in -1 will disable the timeout by setting the corresponding net timeout values to nil
   # * :ssl_client_cert, :ssl_client_key, :ssl_ca_file
+  # * :ssl_verify_callback, :ssl_verify_callback_warnings
   class Request
 
     attr_reader :method, :url, :headers, :cookies,
-                :payload, :user, :password, :timeout,
+                :payload, :user, :password, :timeout, :max_redirects,
                 :open_timeout, :raw_response, :verify_ssl, :ssl_client_cert,
-                :ssl_client_key, :ssl_ca_file, :processed_headers, :args
+                :ssl_client_key, :ssl_ca_file, :processed_headers, :args,
+                :ssl_verify_callback, :ssl_verify_callback_warnings
 
     def self.execute(args, & block)
       new(args).execute(& block)
@@ -35,7 +39,7 @@ module RestClient
       @method = args[:method] or raise ArgumentError, "must pass :method"
       @headers = args[:headers] || {}
       if args[:url]
-        @url = process_get_params(args[:url], headers)
+        @url = process_url_params(args[:url], headers)
       else
         raise ArgumentError, "must pass :url"
       end
@@ -45,12 +49,16 @@ module RestClient
       @password = args[:password]
       @timeout = args[:timeout]
       @open_timeout = args[:open_timeout]
+      @block_response = args[:block_response]
       @raw_response = args[:raw_response] || false
       @verify_ssl = args[:verify_ssl] || false
       @ssl_client_cert = args[:ssl_client_cert] || nil
       @ssl_client_key = args[:ssl_client_key] || nil
       @ssl_ca_file = args[:ssl_ca_file] || nil
+      @ssl_verify_callback = args[:ssl_verify_callback] || nil
+      @ssl_verify_callback_warnings = args.fetch(:ssl_verify_callback, true)
       @tf = nil # If you are a raw request, this is your tempfile
+      @max_redirects = args[:max_redirects] || 10
       @processed_headers = make_headers headers
       @args = args
     end
@@ -58,26 +66,24 @@ module RestClient
     def execute & block
       uri = parse_url_with_auth(url)
       transmit uri, net_http_request_class(method).new(uri.request_uri, processed_headers), payload, & block
+    ensure
+      payload.close if payload
     end
 
-    # Extract the query parameters for get request and append them to the url
-    def process_get_params url, headers
-      if [:get, :head].include? method
-        get_params = {}
-        headers.delete_if do |key, value|
-          if 'params' == key.to_s.downcase && value.is_a?(Hash)
-            get_params.merge! value
-            true
-          else
-            false
-          end
-        end
-        unless get_params.empty?
-          query_string = get_params.collect { |k, v| "#{k.to_s}=#{CGI::escape(v.to_s)}" }.join('&')
-          url + "?#{query_string}"
+    # Extract the query parameters and append them to the url
+    def process_url_params url, headers
+      url_params = {}
+      headers.delete_if do |key, value|
+        if 'params' == key.to_s.downcase && value.is_a?(Hash)
+          url_params.merge! value
+          true
         else
-          url
+          false
         end
+      end
+      unless url_params.empty?
+        query_string = url_params.collect { |k, v| "#{k.to_s}=#{CGI::escape(v.to_s)}" }.join('&')
+        url + "?#{query_string}"
       else
         url
       end
@@ -85,7 +91,7 @@ module RestClient
 
     def make_headers user_headers
       unless @cookies.empty?
-        user_headers[:cookie] = @cookies.map { |(key, val)| "#{key.to_s}=#{CGI::unescape(val)}" }.sort.join(';')
+        user_headers[:cookie] = @cookies.map { |(key, val)| "#{key.to_s}=#{CGI::unescape(val)}" }.sort.join('; ')
       end
       headers = stringify_headers(default_headers).merge(stringify_headers(user_headers))
       headers.merge!(@payload.headers) if @payload
@@ -127,29 +133,40 @@ module RestClient
           if p[k].is_a? Hash
             process_payload(p[k], key)
           else
-            value = URI.escape(p[k].to_s, Regexp.new("[^#{URI::PATTERN::UNRESERVED}]"))
+            value = parser.escape(p[k].to_s, Regexp.new("[^#{URI::PATTERN::UNRESERVED}]"))
             "#{key}=#{value}"
           end
         end.join("&")
       end
     end
 
+    def print_verify_callback_warnings
+      warned = false
+      if RestClient::Platform.mac?
+        warn('warning: ssl_verify_callback return code is ignored on OS X')
+        warned = true
+      end
+      if RestClient::Platform.jruby?
+        warn('warning: SSL verify_callback may not work correctly in jruby')
+        warn('see https://github.com/jruby/jruby/issues/597')
+        warned = true
+      end
+      warned
+    end
+
     def transmit uri, req, payload, & block
       setup_credentials req
 
       net = net_http_class.new(uri.host, uri.port)
       net.use_ssl = uri.is_a?(URI::HTTPS)
-      if @verify_ssl == false
-        net.verify_mode = OpenSSL::SSL::VERIFY_NONE
-      elsif @verify_ssl.is_a? Integer
-        net.verify_mode = @verify_ssl
-        net.verify_callback = lambda do |preverify_ok, ssl_context|
-          if (!preverify_ok) || ssl_context.error != 0
-            err_msg = "SSL Verification failed -- Preverify: #{preverify_ok}, Error: #{ssl_context.error_string} (#{ssl_context.error})"
-            raise SSLCertificateNotVerified.new(err_msg)
-          end
-          true
+      if @verify_ssl
+        if @verify_ssl.is_a? Integer
+          net.verify_mode = @verify_ssl
+        else
+          net.verify_mode = OpenSSL::SSL::VERIFY_PEER
         end
+      else
+        net.verify_mode = OpenSSL::SSL::VERIFY_NONE
       end
       net.cert = @ssl_client_cert if @ssl_client_cert
       net.key = @ssl_client_key if @ssl_client_key
@@ -157,6 +174,29 @@ module RestClient
       net.read_timeout = @timeout if @timeout
       net.open_timeout = @open_timeout if @open_timeout
 
+      # disable the timeout if the timeout value is -1
+      net.read_timeout = nil if @timeout == -1
+      net.open_timeout = nil if @open_timeout == -1
+
+      # verify_callback isn't well supported on all platforms, but do allow
+      # users to set one if they want.
+      if ssl_verify_callback
+        net.verify_callback = ssl_verify_callback
+
+        # Hilariously, jruby only calls the callback when cert_store is set to
+        # something, so make sure to set one.
+        # https://github.com/jruby/jruby/issues/597
+        if RestClient::Platform.jruby?
+          net.cert_store ||= OpenSSL::X509::Store.new
+        end
+
+        if ssl_verify_callback_warnings != false
+          if print_verify_callback_warnings
+            warn('pass :ssl_verify_callback_warnings => false to silence this')
+          end
+        end
+      end
+
       RestClient.before_execution_procs.each do |before_proc|
         before_proc.call(req, args)
       end
@@ -164,14 +204,23 @@ module RestClient
       log_request
 
       net.start do |http|
-        res = http.request(req, payload) { |http_response| fetch_body(http_response) }
-        log_response res
-        process_result res, & block
+        if @block_response
+          http.request(req, payload ? payload.to_s : nil, & @block_response)
+        else
+          res = http.request(req, payload ? payload.to_s : nil) { |http_response| fetch_body(http_response) }
+          log_response res
+          process_result res, & block
+        end
       end
     rescue EOFError
       raise RestClient::ServerBrokeConnection
     rescue Timeout::Error
       raise RestClient::RequestTimeout
+    rescue OpenSSL::SSL::SSLError => error
+      # UGH. Not sure if this is needed at all. SSLCertificateNotVerified is not being used internally.
+      # I think it would be better to leave SSLError processing to the client (they'd have to do that anyway...)
+      raise SSLCertificateNotVerified.new(error.message) if error.message.include?("certificate verify failed")
+      raise error
     end
 
     def setup_credentials(req)
@@ -228,7 +277,13 @@ module RestClient
       elsif content_encoding == 'gzip'
         Zlib::GzipReader.new(StringIO.new(body)).read
       elsif content_encoding == 'deflate'
-        Zlib::Inflate.new.inflate body
+        begin
+          Zlib::Inflate.new.inflate body
+        rescue Zlib::DataError
+          # No luck with Zlib decompression. Let's try with raw deflate,
+          # like some broken web servers do.
+          Zlib::Inflate.new(-Zlib::MAX_WBITS).inflate body
+        end
       else
         body
       end
@@ -279,6 +334,11 @@ module RestClient
       {:accept => '*/*; q=0.5, application/xml', :accept_encoding => 'gzip, deflate'}
     end
 
+    private
+      def parser
+        URI.const_defined?(:Parser) ? URI::Parser.new : URI
+      end
+
   end
 end