rest-client 1.7.2 → 2.1.0

data/Rakefile

@@ -34,6 +34,21 @@ RSpec::Core::RakeTask.new('rcov') do |t|
   t.rcov_opts = ['--exclude', 'examples']
 end
 
+desc 'Regenerate authors file'
+task :authors do
+  Dir.chdir(File.dirname(__FILE__)) do
+    File.open('AUTHORS', 'w') do |f|
+      f.write <<-EOM
+The Ruby REST Client would not be what it is today without the help of
+the following kind souls:
+
+      EOM
+    end
+
+    sh 'git shortlog -s | cut -f 2 >> AUTHORS'
+  end
+end
+
 task :default do
   sh 'rake -T'
 end
@@ -54,8 +69,8 @@ namespace :all do
   task :build => ['ruby:build'] + \
     WindowsPlatforms.map {|p| "windows:#{p}:build"}
 
-  desc "Create tag v#{RestClient::VERSION} and for all platforms build and push " \
-    "rest-client #{RestClient::VERSION} to Rubygems"
+  desc "Create tag v#{RestClient::VERSION} and for all platforms build and " \
+    "push rest-client #{RestClient::VERSION} to Rubygems"
   task :release => ['build', 'ruby:release'] + \
     WindowsPlatforms.map {|p| "windows:#{p}:push"}
 
@@ -111,6 +126,15 @@ Rake::RDocTask.new do |t|
   t.title    = "rest-client, fetch RESTful resources effortlessly"
   t.options << '--line-numbers' << '--inline-source' << '-A cattr_accessor=object'
   t.options << '--charset' << 'utf-8'
-  t.rdoc_files.include('README.rdoc')
+  t.rdoc_files.include('README.md')
   t.rdoc_files.include('lib/*.rb')
 end
+
+############################
+
+require 'rubocop/rake_task'
+
+RuboCop::RakeTask.new(:rubocop) do |t|
+  t.options = ['--display-cop-names']
+end
+alias_task(:lint, :rubocop)

data/bin/restclient

@@ -56,11 +56,9 @@ if @verb
 end
 
 POSSIBLE_VERBS.each do |m|
-  eval <<-end_eval
-def  #{m}(path, *args, &b)
-  r[path].#{m}(*args, &b)
-end
-  end_eval
+  define_method(m.to_sym) do |path, *args, &b|
+    r[path].public_send(m.to_sym, *args, &b)
+  end
 end
 
 def method_missing(s, * args, & b)

data/history.md

@@ -1,3 +1,186 @@
+# 2.1.0
+
+- Add a dependency on http-accept for parsing Content-Type charset headers.
+  This works around a bad memory leak introduced in MRI Ruby 2.4.0 and fixed in
+  Ruby 2.4.2. (#615)
+- Use mime/types/columnar from mime-types 2.6.1+, which is leaner in memory
+  usage than the older storage model of mime-types. (#393)
+- Add `:log` option to individual requests. This allows users to set a log on a
+  per-request / per-resource basis instead of the kludgy global log. (#538)
+- Log request duration by tracking request start and end times. Make
+  `log_response` a method on the Response object, and ensure the `size` method
+  works on RawResponse objects. (#126)
+  - `# => 200 OK | text/html 1270 bytes, 0.08s`
+  - Also add a new `:stream_log_percent` parameter, which is applicable only
+    when `:raw_response => true` is set. This causes progress logs to be
+    emitted only on every N% (default 10%) of the total download size rather
+    than on every chunk.
+- Drop custom handling of compression and use built-in Net::HTTP support for
+  supported Content-Encodings like gzip and deflate. Don't set any explicit
+  `Accept-Encoding` header, rely instead on Net::HTTP defaults. (#597)
+  - Note: this changes behavior for compressed responses when using
+    `:raw_response => true`. Previously the raw response would not have been
+    uncompressed by rest-client, but now Net::HTTP will uncompress it.
+- The previous fix to avoid having Netrc username/password override an
+  Authorization header was case-sensitive and incomplete. Fix this by
+  respecting existing Authorization headers, regardless of letter case. (#550)
+- Handle ParamsArray payloads. Previously, rest-client would silently drop a
+  ParamsArray passed as the payload. Instead, automatically use
+  Payload::Multipart if the ParamsArray contains a file handle, or use
+  Payload::UrlEncoded if it doesn't. (#508)
+- Gracefully handle Payload objects (Payload::Base or subclasses) that are
+  passed as a payload argument. Previously, `Payload.generate` would wrap a
+  Payload object in Payload::Streamed, creating a pointlessly nested payload.
+  Also add a `closed?` method to Payload objects, and don't error in
+  `short_inspect` if `size` returns nil. (#603)
+- Test with an image in the public domain to avoid licensing complexity. (#607)
+
+# 2.0.2
+
+- Suppress the header override warning introduced in 2.0.1 if the value is the
+  same. There's no conflict if the value is unchanged. (#578)
+
+# 2.0.1
+
+- Warn if auto-generated headers from the payload, such as Content-Type,
+  override headers set by the user. This is usually not what the user wants to
+  happen, and can be surprising. (#554)
+- Drop the old check for weak default TLS ciphers, and use the built-in Ruby
+  defaults. Ruby versions from Oct. 2014 onward use sane defaults, so this is
+  no longer needed. (#573)
+
+# 2.0.0
+
+This release is largely API compatible, but makes several breaking changes.
+
+- Drop support for Ruby 1.9
+- Allow mime-types as new as 3.x (requires ruby 2.0)
+- Respect Content-Type charset header provided by server. Previously,
+  rest-client would not override the string encoding chosen by Net::HTTP. Now
+  responses that specify a charset will yield a body string in that encoding.
+  For example, `Content-Type: text/plain; charset=EUC-JP` will return a String
+  encoded with `Encoding::EUC_JP`. (#361)
+- Change exceptions raised on request timeout. Instead of
+  `RestClient::RequestTimeout` (which is still used for HTTP 408), network
+  timeouts will now raise either `RestClient::Exceptions::ReadTimeout` or
+  `RestClient::Exceptions::OpenTimeout`, both of which inherit from
+  `RestClient::Exceptions::Timeout`. For backwards compatibility, this still
+  inherits from `RestClient::RequestTimeout` so existing uses will still work.
+  This may change in a future major release. These new timeout classes also
+  make the original wrapped exception available as `#original_exception`.
+- Unify request exceptions under `RestClient::RequestFailed`, which still
+  inherits from `ExceptionWithResponse`. Previously, HTTP 304, 401, and 404
+  inherited directly from `ExceptionWithResponse` rather than from
+  `RequestFailed`. Now _all_ HTTP status code exceptions inherit from both.
+- Rename the `:timeout` request option to `:read_timeout`. When `:timeout` is
+  passed, now set both `:read_timeout` and `:open_timeout`.
+- Change default HTTP Accept header to `*/*`
+- Use a more descriptive User-Agent header by default
+- Drop RC4-MD5 from default cipher list
+- Only prepend http:// to URIs without a scheme
+- Fix some support for using IPv6 addresses in URLs (still affected by Ruby
+  2.0+ bug https://bugs.ruby-lang.org/issues/9129, with the fix expected to be
+  backported to 2.0 and 2.1)
+- `Response` objects are now a subclass of `String` rather than a `String` that
+  mixes in the response functionality. Most of the methods remain unchanged,
+  but this makes it much easier to understand what is happening when you look
+  at a RestClient response object. There are a few additional changes:
+  - Response objects now implement `.inspect` to make this distinction clearer.
+  - `Response#to_i` will now behave like `String#to_i` instead of returning the
+    HTTP response code, which was very surprising behavior.
+  - `Response#body` and `#to_s` will now return a true `String` object rather
+    than self. Previously there was no easy way to get the true `String`
+    response instead of the Frankenstein response string object with
+    AbstractResponse mixed in.
+  - Response objects no longer accept an extra request args hash, but instead
+    access request args directly from the request object, which reduces
+    confusion and duplication.
+- Handle multiple HTTP response headers with the same name (except for
+  Set-Cookie, which is special) by joining the values with a comma space,
+  compliant with RFC 7230
+- Rewrite cookie support to be much smarter and to use cookie jars consistently
+  for requests, responses, and redirection in order to resolve long-standing
+  complaints about the previously broken behavior: (#498)
+  - The `:cookies` option may now be a Hash of Strings, an Array of
+    HTTP::Cookie objects, or a full HTTP::CookieJar.
+  - Add `RestClient::Request#cookie_jar` and reimplement `Request#cookies` to
+    be a wrapper around the cookie jar.
+  - Still support passing the `:cookies` option in the headers hash, but now
+    raise ArgumentError if that option is also passed to `Request#initialize`.
+  - Warn if both `:cookies` and a `Cookie` header are supplied.
+  - Use the `Request#cookie_jar` as the basis for `Response#cookie_jar`,
+    creating a copy of the jar and adding any newly received cookies.
+  - When following redirection, also use this same strategy so that cookies
+    from the original request are carried through in a standards-compliant way
+    by the cookie jar.
+- Don't set basic auth header if explicit `Authorization` header is specified
+- Add `:proxy` option to requests, which can be used for thread-safe
+  per-request proxy configuration, overriding `RestClient.proxy`
+- Allow overriding `ENV['http_proxy']` to disable proxies by setting
+  `RestClient.proxy` to a falsey value. Previously there was no way in Ruby 2.x
+  to turn off a proxy specified in the environment without changing `ENV`.
+- Add actual support for streaming request payloads. Previously rest-client
+  would call `.to_s` even on RestClient::Payload::Streamed objects. Instead,
+  treat any object that responds to `.read` as a streaming payload and pass it
+  through to `.body_stream=` on the Net:HTTP object. This massively reduces the
+  memory required for large file uploads.
+- Changes to redirection behavior: (#381, #484)
+  - Remove `RestClient::MaxRedirectsReached` in favor of the normal
+    `ExceptionWithResponse` subclasses. This makes the response accessible on
+    the exception object as `.response`, making it possible for callers to tell
+    what has actually happened when the redirect limit is reached.
+  - When following HTTP redirection, store a list of each previous response on
+    the response object as `.history`. This makes it possible to access the
+    original response headers and body before the redirection was followed.
+  - Follow redirection consistently, regardless of whether the HTTP method was
+    passed as a symbol or string. Under the hood rest-client now normalizes the
+    HTTP request method to a lowercase string.
+- Add `:before_execution_proc` option to `RestClient::Request`. This makes it
+  possible to add procs like `RestClient.add_before_execution_proc` to a single
+  request without global state.
+- Run tests on Travis's beta OS X support.
+- Make `Request#transmit` a private method, along with a few others.
+- Refactor URI parsing to happen earlier, in Request initialization.
+- Improve consistency and functionality of complex URL parameter handling:
+  - When adding URL params, handle URLs that already contain params.
+  - Add new convention for handling URL params containing deeply nested arrays
+    and hashes, unify handling of null/empty values, and use the same code for
+    GET and POST params. (#437)
+  - Add the RestClient::ParamsArray class, a simple array-like container that
+    can be used to pass multiple keys with same name or keys where the ordering
+    is significant.
+- Add a few more exception classes for obscure HTTP status codes.
+- Multipart: use a much more robust multipart boundary with greater entropy.
+- Make `RestClient::Payload::Base#inspect` stop pretending to be a String.
+- Add `Request#redacted_uri` and `Request#redacted_url` to display the URI
+  with any password redacted.
+
+# 2.0.0.rc1
+
+Changes in the release candidate that did not persist through the final 2.0.0
+release:
+- RestClient::Exceptions::Timeout was originally going to be a direct subclass
+  of RestClient::Exception in the release candidate. This exception tree was
+  made a subclass of RestClient::RequestTimeout prior to the final release.
+
+# 1.8.0
+
+- Security: implement standards compliant cookie handling by adding a
+  dependency on http-cookie. This breaks compatibility, but was necessary to
+  address a session fixation / cookie disclosure vulnerability.
+  (#369 / CVE-2015-1820)
+
+  Previously, any Set-Cookie headers found in an HTTP 30x response would be
+  sent to the redirection target, regardless of domain. Responses now expose a
+  cookie jar and respect standards compliant domain / path flags in Set-Cookie
+  headers.
+
+# 1.7.3
+
+- Security: redact password in URI from logs (#349 / OSVDB-117461)
+- Drop monkey patch on MIME::Types (added `type_for_extension` method, use
+  the public interface instead.
+
 # 1.7.2
 
 - Ignore duplicate certificates in CA store on Windows
@@ -33,6 +216,22 @@
   - Disable timeouts with :timeout => nil rather than :timeout => -1
   - Drop all Net::HTTP monkey patches
 
+# 1.6.14
+
+- This release is unchanged from 1.6.9. It was published in order to supersede
+  the malicious 1.6.10-13 versions, even for users who are still pinning to the
+  legacy 1.6.x series. All users are encouraged to upgrade to rest-client 2.x.
+
+# 1.6.10, 1.6.11, 1.6.12, 1.6.13 (CVE-2019-15224)
+
+- These versions were pushed by a malicious actor and included a backdoor permitting
+  remote code execution in Rails environments. (#713)
+- They were live for about five days before being yanked.
+
+# 1.6.9
+
+- Move rdoc to a development dependency
+
 # 1.6.8
 
 - The 1.6.x series will be the last to support Ruby 1.8.7

data/lib/restclient/abstract_response.rb

@@ -1,16 +1,40 @@
 require 'cgi'
+require 'http-cookie'
 
 module RestClient
 
   module AbstractResponse
 
-    attr_reader :net_http_res, :args
+    attr_reader :net_http_res, :request, :start_time, :end_time, :duration
+
+    def inspect
+      raise NotImplementedError.new('must override in subclass')
+    end
+
+    # Logger from the request, potentially nil.
+    def log
+      request.log
+    end
+
+    def log_response
+      return unless log
+
+      code = net_http_res.code
+      res_name = net_http_res.class.to_s.gsub(/\ANet::HTTP/, '')
+      content_type = (net_http_res['Content-type'] || '').gsub(/;.*\z/, '')
+
+      log << "# => #{code} #{res_name} | #{content_type} #{size} bytes, #{sprintf('%.2f', duration)}s\n"
+    end
 
     # HTTP status code
     def code
       @code ||= @net_http_res.code.to_i
     end
 
+    def history
+      @history ||= request.redirection_history || []
+    end
+
     # A hash of the headers, beautified with symbols and underscores.
     # e.g. "Content-type" will become :content_type.
     def headers
@@ -22,85 +46,207 @@ module RestClient
       @raw_headers ||= @net_http_res.to_hash
     end
 
-    # Hash of cookies extracted from response headers
+    # @param [Net::HTTPResponse] net_http_res
+    # @param [RestClient::Request] request
+    # @param [Time] start_time
+    def response_set_vars(net_http_res, request, start_time)
+      @net_http_res = net_http_res
+      @request = request
+      @start_time = start_time
+      @end_time = Time.now
+
+      if @start_time
+        @duration = @end_time - @start_time
+      else
+        @duration = nil
+      end
+
+      # prime redirection history
+      history
+    end
+
+    # Hash of cookies extracted from response headers.
+    #
+    # NB: This will return only cookies whose domain matches this request, and
+    # may not even return all of those cookies if there are duplicate names.
+    # Use the full cookie_jar for more nuanced access.
+    #
+    # @see #cookie_jar
+    #
+    # @return [Hash]
+    #
     def cookies
-      @cookies ||= (self.headers[:set_cookie] || {}).inject({}) do |out, cookie_content|
-        out.merge parse_cookie(cookie_content)
+      hash = {}
+
+      cookie_jar.cookies(@request.uri).each do |cookie|
+        hash[cookie.name] = cookie.value
+      end
+
+      hash
+    end
+
+    # Cookie jar extracted from response headers.
+    #
+    # @return [HTTP::CookieJar]
+    #
+    def cookie_jar
+      return @cookie_jar if defined?(@cookie_jar) && @cookie_jar
+
+      jar = @request.cookie_jar.dup
+      headers.fetch(:set_cookie, []).each do |cookie|
+        jar.parse(cookie, @request.uri)
       end
+
+      @cookie_jar = jar
     end
 
     # Return the default behavior corresponding to the response code:
-    # the response itself for code in 200..206, redirection for 301, 302 and 307 in get and head cases, redirection for 303 and an exception in other cases
-    def return! request = nil, result = nil, & block
-      if (200..207).include? code
+    #
+    # For 20x status codes: return the response itself
+    #
+    # For 30x status codes:
+    #   301, 302, 307: redirect GET / HEAD if there is a Location header
+    #   303: redirect, changing method to GET, if there is a Location header
+    #
+    # For all other responses, raise a response exception
+    #
+    def return!(&block)
+      case code
+      when 200..207
         self
-      elsif [301, 302, 307].include? code
-        unless [:get, :head].include? args[:method]
-          raise Exceptions::EXCEPTIONS_MAP[code].new(self, code)
+      when 301, 302, 307
+        case request.method
+        when 'get', 'head'
+          check_max_redirects
+          follow_redirection(&block)
         else
-          follow_redirection(request, result, & block)
+          raise exception_with_response
         end
-      elsif code == 303
-        args[:method] = :get
-        args.delete :payload
-        follow_redirection(request, result, & block)
-      elsif Exceptions::EXCEPTIONS_MAP[code]
-        raise Exceptions::EXCEPTIONS_MAP[code].new(self, code)
+      when 303
+        check_max_redirects
+        follow_get_redirection(&block)
       else
-        raise RequestFailed.new(self, code)
+        raise exception_with_response
       end
     end
 
     def to_i
-      code
+      warn('warning: calling Response#to_i is not recommended')
+      super
     end
 
     def description
       "#{code} #{STATUSES[code]} | #{(headers[:content_type] || '').gsub(/;.*$/, '')} #{size} bytes\n"
     end
 
-    # Follow a redirection
-    def follow_redirection request = nil, result = nil, & block
-      url = headers[:location]
-      if url !~ /^http/
-        url = URI.parse(args[:url]).merge(url).to_s
-      end
-      args[:url] = url
-      if request
-        if request.max_redirects == 0
-          raise MaxRedirectsReached
-        end
-        args[:password] = request.password
-        args[:user] = request.user
-        args[:headers] = request.headers
-        args[:max_redirects] = request.max_redirects - 1
-        # pass any cookie set in the result
-        if result && result['set-cookie']
-          args[:headers][:cookies] = (args[:headers][:cookies] || {}).merge(parse_cookie(result['set-cookie']))
-        end
-      end
-      Request.execute args, &block
+    # Follow a redirection response by making a new HTTP request to the
+    # redirection target.
+    def follow_redirection(&block)
+      _follow_redirection(request.args.dup, &block)
+    end
+
+    # Follow a redirection response, but change the HTTP method to GET and drop
+    # the payload from the original request.
+    def follow_get_redirection(&block)
+      new_args = request.args.dup
+      new_args[:method] = :get
+      new_args.delete(:payload)
+
+      _follow_redirection(new_args, &block)
     end
 
+    # Convert headers hash into canonical form.
+    #
+    # Header names will be converted to lowercase symbols with underscores
+    # instead of hyphens.
+    #
+    # Headers specified multiple times will be joined by comma and space,
+    # except for Set-Cookie, which will always be an array.
+    #
+    # Per RFC 2616, if a server sends multiple headers with the same key, they
+    # MUST be able to be joined into a single header by a comma. However,
+    # Set-Cookie (RFC 6265) cannot because commas are valid within cookie
+    # definitions. The newer RFC 7230 notes (3.2.2) that Set-Cookie should be
+    # handled as a special case.
+    #
+    # http://tools.ietf.org/html/rfc2616#section-4.2
+    # http://tools.ietf.org/html/rfc7230#section-3.2.2
+    # http://tools.ietf.org/html/rfc6265
+    #
+    # @param headers [Hash]
+    # @return [Hash]
+    #
     def self.beautify_headers(headers)
       headers.inject({}) do |out, (key, value)|
-        out[key.gsub(/-/, '_').downcase.to_sym] = %w{ set-cookie }.include?(key.downcase) ? value : value.first
+        key_sym = key.tr('-', '_').downcase.to_sym
+
+        # Handle Set-Cookie specially since it cannot be joined by comma.
+        if key.downcase == 'set-cookie'
+          out[key_sym] = value
+        else
+          out[key_sym] = value.join(', ')
+        end
+
         out
       end
     end
 
     private
 
-    # Parse a cookie value and return its content in an Hash
-    def parse_cookie cookie_content
-      out = {}
-      CGI::Cookie::parse(cookie_content).each do |key, cookie|
-        unless ['expires', 'path'].include? key
-          out[CGI::escape(key)] = cookie.value[0] ? (CGI::escape(cookie.value[0]) || '') : ''
-        end
+    # Follow a redirection
+    #
+    # @param new_args [Hash] Start with this hash of arguments for the
+    #   redirection request. The hash will be mutated, so be sure to dup any
+    #   existing hash that should not be modified.
+    #
+    def _follow_redirection(new_args, &block)
+
+      # parse location header and merge into existing URL
+      url = headers[:location]
+
+      # cannot follow redirection if there is no location header
+      unless url
+        raise exception_with_response
+      end
+
+      # handle relative redirects
+      unless url.start_with?('http')
+        url = URI.parse(request.url).merge(url).to_s
+      end
+      new_args[:url] = url
+
+      new_args[:password] = request.password
+      new_args[:user] = request.user
+      new_args[:headers] = request.headers
+      new_args[:max_redirects] = request.max_redirects - 1
+
+      # pass through our new cookie jar
+      new_args[:cookies] = cookie_jar
+
+      # prepare new request
+      new_req = Request.new(new_args)
+
+      # append self to redirection history
+      new_req.redirection_history = history + [self]
+
+      # execute redirected request
+      new_req.execute(&block)
+    end
+
+    def check_max_redirects
+      if request.max_redirects <= 0
+        raise exception_with_response
       end
-      out
     end
-  end
 
+    def exception_with_response
+      begin
+        klass = Exceptions::EXCEPTIONS_MAP.fetch(code)
+      rescue KeyError
+        raise RequestFailed.new(self, code)
+      end
+
+      raise klass.new(self, code)
+    end
+  end
 end