faraday_middleware 0.9.1 → 1.2.0

data/lib/faraday_middleware/response/caching.rb

@@ -1,40 +1,66 @@
+# frozen_string_literal: true
+
 require 'faraday'
 require 'forwardable'
-# fixes normalizing query strings:
-require 'faraday_middleware/addressable_patch' if defined? ::Addressable::URI
+require 'digest/sha1'
 
 module FaradayMiddleware
   # Public: Caches GET responses and pulls subsequent ones from the cache.
   class Caching < Faraday::Middleware
     attr_reader :cache
 
+    # Internal: List of status codes that can be cached:
+    # * 200 - 'OK'
+    # * 203 - 'Non-Authoritative Information'
+    # * 300 - 'Multiple Choices'
+    # * 301 - 'Moved Permanently'
+    # * 302 - 'Found'
+    # * 404 - 'Not Found'
+    # * 410 - 'Gone'
+    CACHEABLE_STATUS_CODES = [200, 203, 300, 301, 302, 404, 410].freeze
+
     extend Forwardable
     def_delegators :'Faraday::Utils', :parse_query, :build_query
 
     # Public: initialize the middleware.
     #
-    # cache   - An object that responds to read, write and fetch (default: nil).
+    # cache   - An object that responds to read and write (default: nil).
     # options - An options Hash (default: {}):
-    #           :ignore_params - String name or Array names of query params
-    #                            that should be ignored when forming the cache
-    #                            key (default: []).
+    #           :ignore_params - String name or Array names of query
+    #                                    params that should be ignored when forming
+    #                                    the cache key (default: []).
+    #           :write_options - Hash of settings that should be passed as the
+    #                                    third options parameter to the cache's #write
+    #                                    method. If not specified, no options parameter
+    #                                    will be passed.
+    #           :full_key      - Boolean - use full URL as cache key:
+    #                                    (url.host + url.request_uri)
+    #           :status_codes  - Array of http status code to be cache
+    #                                    (default: CACHEABLE_STATUS_CODE)
     #
     # Yields if no cache is given. The block should return a cache object.
     def initialize(app, cache = nil, options = {})
       super(app)
-      options, cache = cache, nil if cache.is_a? Hash and block_given?
+      if cache.is_a?(Hash) && block_given?
+        options = cache
+        cache = nil
+      end
       @cache = cache || yield
       @options = options
     end
 
     def call(env)
-      if :get == env[:method]
+      if env[:method] == :get
         if env[:parallel_manager]
           # callback mode
           cache_on_complete(env)
         else
           # synchronous mode
-          response = cache.fetch(cache_key(env)) { @app.call(env) }
+          key = cache_key(env)
+          unless (response = cache.read(key)) && response
+            response = @app.call(env)
+            store_response_in_cache(key, response)
+          end
           finalize_response(response, env)
         end
       else
@@ -46,24 +72,50 @@ module FaradayMiddleware
       url = env[:url].dup
       if url.query && params_to_ignore.any?
         params = parse_query url.query
-        params.reject! {|k,| params_to_ignore.include? k }
+        params.reject! { |k,| params_to_ignore.include? k }
         url.query = params.any? ? build_query(params) : nil
       end
       url.normalize!
-      url.request_uri
+      digest = full_key? ? url.host + url.request_uri : url.request_uri
+      Digest::SHA1.hexdigest(digest)
     end
 
     def params_to_ignore
-      @params_to_ignore ||= Array(@options[:ignore_params]).map { |p| p.to_s }
+      @params_to_ignore ||= Array(@options[:ignore_params]).map(&:to_s)
+    end
+
+    def full_key?
+      @full_key ||= @options[:full_key]
+    end
+
+    def custom_status_codes
+      @custom_status_codes ||= begin
+        codes = CACHEABLE_STATUS_CODES & Array(@options[:status_codes]).map(&:to_i)
+        codes.any? ? codes : CACHEABLE_STATUS_CODES
+      end
     end
 
     def cache_on_complete(env)
       key = cache_key(env)
-      if cached_response = cache.read(key)
+      if (cached_response = cache.read(key))
         finalize_response(cached_response, env)
       else
-        response = @app.call(env)
-        response.on_complete { cache.write(key, response) }
+        # response.status is nil at this point
+        # any checks need to be done inside on_complete block
+        @app.call(env).on_complete do |response_env|
+          store_response_in_cache(key, response_env.response)
+          response_env
+        end
+      end
+    end
+
+    def store_response_in_cache(key, response)
+      return unless custom_status_codes.include?(response.status)
+
+      if @options[:write_options]
+        cache.write(key, response, @options[:write_options])
+      else
+        cache.write(key, response)
       end
     end
 

data/lib/faraday_middleware/response/chunked.rb

@@ -1,16 +1,19 @@
+# frozen_string_literal: true
+
 require 'faraday_middleware/response_middleware'
 
 module FaradayMiddleware
-  # Public: Parse a Transfer-Encoding: Chunked response to just the original data
+  # Public: Parse a Transfer-Encoding. Chunks response to just the original data
   class Chunked < FaradayMiddleware::ResponseMiddleware
-    TRANSFER_ENCODING = 'transfer-encoding'.freeze
+    TRANSFER_ENCODING = 'transfer-encoding'
 
     define_parser do |raw_body|
       decoded_body = []
       until raw_body.empty?
         chunk_len, raw_body = raw_body.split("\r\n", 2)
-        chunk_len = chunk_len.split(';',2).first.hex
-        break if chunk_len == 0
+        chunk_len = chunk_len.split(';', 2).first.hex
+        break if chunk_len.zero?
+
         decoded_body << raw_body[0, chunk_len]
         # The 2 is to strip the extra CRLF at the end of the chunk
         raw_body = raw_body[chunk_len + 2, raw_body.length - chunk_len - 2]
@@ -19,11 +22,12 @@ module FaradayMiddleware
     end
 
     def parse_response?(env)
-      super and chunked_encoding?(env[:response_headers])
+      super && chunked_encoding?(env[:response_headers])
     end
 
     def chunked_encoding?(headers)
-      encoding = headers[TRANSFER_ENCODING] and encoding.split(',').include?('chunked')
+      (encoding = headers[TRANSFER_ENCODING]) &&
+        encoding.split(',').include?('chunked')
     end
   end
 end

data/lib/faraday_middleware/response/follow_redirects.rb

@@ -1,64 +1,66 @@
+# frozen_string_literal: true
+
 require 'faraday'
 require 'set'
 
 module FaradayMiddleware
-  # Public: Exception thrown when the maximum amount of requests is exceeded.
-  class RedirectLimitReached < Faraday::Error::ClientError
-    attr_reader :response
-
-    def initialize(response)
-      super "too many redirects; last one to: #{response['location']}"
-      @response = response
-    end
-  end
-
-  # Public: Follow HTTP 301, 302, 303, and 307 redirects for GET, PATCH, POST,
-  # PUT, and DELETE requests.
+  # Public: Follow HTTP 301, 302, 303, 307, and 308 redirects.
   #
-  # This middleware does not follow the HTTP specification for HTTP 302, by
-  # default, in that it follows the improper implementation used by most major
-  # web browsers which forces the redirected request to become a GET request
-  # regardless of the original request method.
+  # For HTTP 301, 302, and 303, the original GET, POST, PUT, DELETE, or PATCH
+  # request gets converted into a GET. With `:standards_compliant => true`,
+  # however, the HTTP method after 301/302 remains unchanged. This allows you
+  # to opt into HTTP/1.1 compliance and act unlike the major web browsers.
   #
-  # For HTTP 301, 302, and 303, the original request is transformed into a
-  # GET request to the response Location, by default. However, with standards
-  # compliance enabled, a 302 will instead act in accordance with the HTTP
-  # specification, which will replay the original request to the received
-  # Location, just as with a 307.
+  # This middleware currently only works with synchronous requests; i.e. it
+  # doesn't support parallelism.
   #
-  # For HTTP 307, the original request is replayed to the response Location,
-  # including original HTTP request method (GET, POST, PUT, DELETE, PATCH),
-  # original headers, and original body.
+  # If you wish to persist cookies across redirects, you could use
+  # the faraday-cookie_jar gem:
   #
-  # This middleware currently only works with synchronous requests; in other
-  # words, it doesn't support parallelism.
+  #   Faraday.new(:url => url) do |faraday|
+  #     faraday.use FaradayMiddleware::FollowRedirects
+  #     faraday.use :cookie_jar
+  #     faraday.adapter Faraday.default_adapter
+  #   end
   class FollowRedirects < Faraday::Middleware
     # HTTP methods for which 30x redirects can be followed
-    ALLOWED_METHODS = Set.new [:head, :options, :get, :post, :put, :patch, :delete]
+    ALLOWED_METHODS = Set.new %i[head options get post put patch delete]
     # HTTP redirect status codes that this middleware implements
-    REDIRECT_CODES  = Set.new [301, 302, 303, 307]
+    REDIRECT_CODES  = Set.new [301, 302, 303, 307, 308]
     # Keys in env hash which will get cleared between requests
-    ENV_TO_CLEAR    = Set.new [:status, :response, :response_headers]
+    ENV_TO_CLEAR    = Set.new %i[status response response_headers]
 
     # Default value for max redirects followed
     FOLLOW_LIMIT = 3
 
+    # Regex that matches characters that need to be escaped in URLs, sans
+    # the "%" character which we assume already represents an escaped sequence.
+    URI_UNSAFE = %r{[^\-_.!~*'()a-zA-Z\d;/?:@&=+$,\[\]%]}.freeze
+
+    AUTH_HEADER = 'Authorization'
+
     # Public: Initialize the middleware.
     #
     # options - An options Hash (default: {}):
-    #           limit - A Numeric redirect limit (default: 3)
-    #           standards_compliant - A Boolean indicating whether to respect
-    #                                 the HTTP spec when following 302
-    #                                 (default: false)
-    #          cookie - Use either an array of strings
-    #                  (e.g. ['cookie1', 'cookie2']) to choose kept cookies
-    #                  or :all to keep all cookies.
+    #     :limit                      - A Numeric redirect limit (default: 3)
+    #     :standards_compliant        - A Boolean indicating whether to respect
+    #                                  the HTTP spec when following 301/302
+    #                                  (default: false)
+    #     :callback                   - A callable used on redirects
+    #                                  with the old and new envs
+    #     :cookies                    - An Array of Strings (e.g.
+    #                                  ['cookie1', 'cookie2']) to choose
+    #                                  cookies to be kept, or :all to keep
+    #                                  all cookies (default: []).
+    #     :clear_authorization_header - A Boolean indicating whether the request
+    #                                  Authorization header should be cleared on
+    #                                  redirects (default: true)
     def initialize(app, options = {})
       super(app)
       @options = options
 
-      @replay_request_codes = Set.new [307]
-      @replay_request_codes << 302 if standards_compliant?
+      @convert_to_get = Set.new [303]
+      @convert_to_get << 301 << 302 unless standards_compliant?
     end
 
     def call(env)
@@ -67,76 +69,89 @@ module FaradayMiddleware
 
     private
 
-    def transform_into_get?(response)
-      return false if [:head, :options].include? response.env[:method]
-      # Never convert head or options to a get. That would just be silly.
-
-      !@replay_request_codes.include? response.status
+    def convert_to_get?(response)
+      !%i[head options].include?(response.env[:method]) &&
+        @convert_to_get.include?(response.status)
     end
 
     def perform_with_redirection(env, follows)
       request_body = env[:body]
       response = @app.call(env)
 
-      response.on_complete do |env|
-        if follow_redirect?(env, response)
+      response.on_complete do |response_env|
+        if follow_redirect?(response_env, response)
           raise RedirectLimitReached, response if follows.zero?
-          env = update_env(env, request_body, response)
-          response = perform_with_redirection(env, follows - 1)
+
+          new_request_env = update_env(response_env.dup, request_body, response)
+          callback&.call(response_env, new_request_env)
+          response = perform_with_redirection(new_request_env, follows - 1)
         end
       end
       response
     end
 
     def update_env(env, request_body, response)
-      env[:url] += response['location']
-      if @options[:cookies]
-        cookies = keep_cookies(env)
-        env[:request_headers][:cookies] = cookies unless cookies.nil?
-      end
+      redirect_from_url = env[:url].to_s
+      redirect_to_url = safe_escape(response['location'] || '')
+      env[:url] += redirect_to_url
+
+      ENV_TO_CLEAR.each { |key| env.delete key }
 
-      if transform_into_get?(response)
+      if convert_to_get?(response)
         env[:method] = :get
         env[:body] = nil
       else
         env[:body] = request_body
       end
 
-      ENV_TO_CLEAR.each {|key| env.delete key }
+      clear_authorization_header(env, redirect_from_url, redirect_to_url)
 
       env
     end
 
     def follow_redirect?(env, response)
-      ALLOWED_METHODS.include? env[:method] and
-        REDIRECT_CODES.include? response.status
+      ALLOWED_METHODS.include?(env[:method]) &&
+        REDIRECT_CODES.include?(response.status)
     end
 
     def follow_limit
       @options.fetch(:limit, FOLLOW_LIMIT)
     end
 
-    def keep_cookies(env)
-      cookies = @options.fetch(:cookies, [])
-      response_cookies = env[:response_headers][:cookies]
-      cookies == :all ? response_cookies : selected_request_cookies(response_cookies)
+    def standards_compliant?
+      @options.fetch(:standards_compliant, false)
     end
 
-    def selected_request_cookies(cookies)
-      selected_cookies(cookies)[0...-1]
+    def callback
+      @options[:callback]
     end
 
-    def selected_cookies(cookies)
-      "".tap do |cookie_string|
-        @options[:cookies].each do |cookie|
-          string = /#{cookie}=?[^;]*/.match(cookies)[0] + ';'
-          cookie_string << string
-        end
+    # Internal: escapes unsafe characters from an URL which might be a path
+    # component only or a fully qualified URI so that it can be joined onto an
+    # URI:HTTP using the `+` operator. Doesn't escape "%" characters so to not
+    # risk double-escaping.
+    def safe_escape(uri)
+      uri = uri.split('#')[0] # we want to remove the fragment if present
+      uri.to_s.gsub(URI_UNSAFE) do |match|
+        "%#{match.unpack('H2' * match.bytesize).join('%').upcase}"
       end
     end
 
-    def standards_compliant?
-      @options.fetch(:standards_compliant, false)
+    def clear_authorization_header(env, from_url, to_url)
+      return env if redirect_to_same_host?(from_url, to_url)
+      return env unless @options.fetch(:clear_authorization_header, true)
+
+      env[:request_headers].delete(AUTH_HEADER)
+    end
+
+    def redirect_to_same_host?(from_url, to_url)
+      return true if to_url.start_with?('/')
+
+      from_uri = URI.parse(from_url)
+      to_uri = URI.parse(to_url)
+
+      [from_uri.scheme, from_uri.host, from_uri.port] ==
+        [to_uri.scheme, to_uri.host, to_uri.port]
     end
   end
 end

data/lib/faraday_middleware/response/mashify.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'faraday'
 
 module FaradayMiddleware

data/lib/faraday_middleware/response/parse_dates.rb

@@ -1,10 +1,13 @@
-require "time"
-require "faraday"
+# frozen_string_literal: true
+
+require 'time'
+require 'faraday'
 
 module FaradayMiddleware
   # Parse dates from response body
   class ParseDates < ::Faraday::Response::Middleware
-    ISO_DATE_FORMAT = /\A\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(\.\d+)?Z\Z/m
+    ISO_DATE_FORMAT = /\A\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(\.\d+)?
+    (Z|((\+|-)\d{2}:?\d{2}))\Z/xm.freeze
 
     def initialize(app, options = {})
       @regexp = options[:match] || ISO_DATE_FORMAT

data/lib/faraday_middleware/response/parse_json.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'faraday_middleware/response_middleware'
 
 module FaradayMiddleware
@@ -7,8 +9,8 @@ module FaradayMiddleware
       require 'json' unless defined?(::JSON)
     end
 
-    define_parser do |body|
-      ::JSON.parse body unless body.strip.empty?
+    define_parser do |body, parser_options|
+      ::JSON.parse(body, parser_options || {}) unless body.strip.empty?
     end
 
     # Public: Override the content-type of the response with "application/json"
@@ -18,7 +20,7 @@ module FaradayMiddleware
     # This is to fix responses from certain API providers that insist on serving
     # JSON with wrong MIME-types such as "text/javascript".
     class MimeTypeFix < ResponseMiddleware
-      MIME_TYPE = 'application/json'.freeze
+      MIME_TYPE = 'application/json'
 
       def process_response(env)
         old_type = env[:response_headers][CONTENT_TYPE].to_s
@@ -27,19 +29,17 @@ module FaradayMiddleware
         env[:response_headers][CONTENT_TYPE] = new_type
       end
 
-      BRACKETS = %w- [ { -
-      WHITESPACE = [ " ", "\n", "\r", "\t" ]
+      BRACKETS = %w-[ {-.freeze
+      WHITESPACE = [' ', "\n", "\r", "\t"].freeze
 
       def parse_response?(env)
-        super and BRACKETS.include? first_char(env[:body])
+        super && BRACKETS.include?(first_char(env[:body]))
       end
 
       def first_char(body)
         idx = -1
-        begin
-          char = body[idx += 1]
-          char = char.chr if char
-        end while char and WHITESPACE.include? char
+        char = body[idx += 1]
+        char = body[idx += 1] while char && WHITESPACE.include?(char)
         char
       end
     end

data/lib/faraday_middleware/response/parse_marshal.rb

@@ -1,10 +1,12 @@
+# frozen_string_literal: true
+
 require 'faraday_middleware/response_middleware'
 
 module FaradayMiddleware
   # Public: Restore marshalled Ruby objects in response bodies.
   class ParseMarshal < ResponseMiddleware
     define_parser do |body|
-      ::Marshal.load body unless body.empty?
+      ::Marshal.load(body) unless body.empty?
     end
   end
 end

data/lib/faraday_middleware/response/parse_xml.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'faraday_middleware/response_middleware'
 
 module FaradayMiddleware
@@ -5,8 +7,8 @@ module FaradayMiddleware
   class ParseXml < ResponseMiddleware
     dependency 'multi_xml'
 
-    define_parser do |body|
-      ::MultiXml.parse(body)
+    define_parser do |body, parser_options|
+      ::MultiXml.parse(body, parser_options || {})
     end
   end
 end

data/lib/faraday_middleware/response/parse_yaml.rb

@@ -1,12 +1,36 @@
+# frozen_string_literal: true
+
 require 'faraday_middleware/response_middleware'
 
 module FaradayMiddleware
   # Public: Parse response bodies as YAML.
+  #
+  # Warning: This is not backwards compatible with versions of this middleware
+  # prior to faraday_middleware v0.12 - prior to this version, we used
+  # YAML.load rather than YAMl.safe_load, which exposes serious remote code
+  # execution risks - see https://github.com/ruby/psych/issues/119 for details.
+  # If you're sure you can trust YAML you're passing, you can set up an unsafe
+  # version of this middleware like this:
+  #
+  #     class UnsafelyParseYaml < FaradayMiddleware::ResponseMiddleware
+  #       dependency do
+  #         require 'yaml'
+  #       end
+  #
+  #       define_parser do |body|
+  #         YAML.load body
+  #       end
+  #     end
+  #
+  #     Faraday.new(..) do |config|
+  #       config.use UnsafelyParseYaml
+  #       ...
+  #     end
   class ParseYaml < ResponseMiddleware
-    dependency 'yaml'
+    dependency 'safe_yaml/load'
 
-    define_parser do |body|
-      ::YAML.load body
+    define_parser do |body, parser_options|
+      SafeYAML.load(body, nil, parser_options || {})
     end
   end
 end

data/lib/faraday_middleware/response/rashify.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'faraday_middleware/response/mashify'
 
 module FaradayMiddleware
@@ -6,7 +8,7 @@ module FaradayMiddleware
   class Rashify < Mashify
     dependency do
       require 'rash'
-      self.mash_class = ::Hashie::Rash
+      self.mash_class = ::Hashie::Mash::Rash
     end
   end
 end