typhoeus 1.6.0 → 1.7.0

data/lib/typhoeus/request.rb

@@ -1,16 +1,11 @@
+# frozen_string_literal: true
+
 require 'zlib'
 require 'digest/sha1'
 require 'typhoeus/request/actions'
-require 'typhoeus/request/before'
-require 'typhoeus/request/block_connection'
-require 'typhoeus/request/cacheable'
 require 'typhoeus/request/callbacks'
 require 'typhoeus/request/marshal'
-require 'typhoeus/request/memoizable'
-require 'typhoeus/request/operations'
-require 'typhoeus/request/responseable'
 require 'typhoeus/request/streamable'
-require 'typhoeus/request/stubbable'
 
 module Typhoeus
 
@@ -23,18 +18,11 @@ module Typhoeus
   #
   # @see (see #initialize)
   class Request
-    extend  Request::Actions
+    extend Request::Actions
     include Request::Callbacks::Types
     include Request::Callbacks
     include Request::Streamable
     include Request::Marshal
-    include Request::Operations
-    include Request::Responseable
-    include Request::Memoizable
-    include Request::Cacheable
-    include Request::BlockConnection
-    include Request::Stubbable
-    include Request::Before
 
     # Returns the provided base url.
     #
@@ -118,6 +106,184 @@ module Typhoeus
       set_defaults
     end
 
+    # Run the request.
+    #
+    # The before, stub, block connection, and cache checks run here explicitly
+    # instead of relying on the order of included modules.
+    #
+    # @example Make a request.
+    #   Typhoeus::Request.new("www.example.com").run
+    #
+    # @return [ Response ] The response.
+    def run
+      callbacks = Typhoeus.before_callbacks_for(self)
+      if callbacks
+        callbacks.each do |callback|
+          value = callback.call(self)
+          if value.is_a?(Response)
+            unless response
+              # A synthetic response is a short circuit like a stub. Mark it
+              # so success? works and it is not written to the cache.
+              value.mock = true
+              execute_headers_callbacks(value)
+              on_body.each { |body_callback| body_callback.call(value.body, value) }
+              finish(value)
+            end
+            return value
+          end
+          return response if value.nil? || value == false
+        end
+      end
+
+      unless Typhoeus.stubs_disabled?
+        if stub_response = Expectation.response_for(self)
+          # The stub block already ran this request for real.
+          return stub_response if stub_response.request.equal?(self)
+
+          execute_headers_callbacks(stub_response)
+          on_body.each { |callback| callback.call(stub_response.body, stub_response) }
+          return finish(stub_response)
+        end
+      end
+
+      raise Typhoeus::Errors::NoStub.new(self) if blocked?
+
+      if cached = cached_response
+        return finish_cached_response(cached)
+      end
+
+      perform
+    end
+
+    # Add a callback that runs before the request.
+    #
+    # @example Add a callback.
+    #   request.before { |request| request.options[:timeout] = 5 }
+    #
+    # @param [ Proc ] block The callback to run.
+    #
+    # @yield [ Typhoeus::Request ]
+    #
+    # @return [ Array<Proc> ] The callbacks.
+    def before(&block)
+      @before ||= []
+      @before << block if block_given?
+      @before
+    end
+
+    # Returns whether a request is blocked. Takes request.block_connection and
+    # the connection state of the current thread into consideration.
+    #
+    # @example Blocked?
+    #   request.blocked?
+    #
+    # @return [ Boolean ] True if blocked, false else.
+    def blocked?
+      if block_connection.nil?
+        Typhoeus.connection_blocked?
+      else
+        block_connection
+      end
+    end
+
+    # Return whether a request is memoizable.
+    #
+    # Streaming requests are not memoized because their body is not retained
+    # and every caller has to receive its own chunks.
+    #
+    # @example Is request memoizable?
+    #   request.memoizable?
+    #
+    # @return [ Boolean ] Return true if memoizable, false else.
+    def memoizable?
+      Typhoeus::Config.memoize &&
+        !streaming? &&
+        request_method == :get
+    end
+
+    # Return whether a request is cacheable.
+    #
+    # @return [ Boolean ]
+    def cacheable?
+      cache
+    end
+
+    # Return the cached response for the request, if there is one.
+    #
+    # @return [ Typhoeus::Response, nil ]
+    #
+    # @api private
+    def cached_response
+      cacheable? && cache.get(self)
+    end
+
+    # Finish a response that came from the cache and deliver it through the
+    # same callbacks a stubbed response uses.
+    #
+    # @param [ Typhoeus::Response ] response The cached response.
+    #
+    # @return [ Typhoeus::Response ] The response.
+    #
+    # @api private
+    def finish_cached_response(response)
+      copy = response.dup
+      copy.cached = true
+      copy.request = self
+      execute_headers_callbacks(copy)
+      on_body.each { |callback| callback.call(copy.body, copy) }
+      finish(copy)
+    end
+
+    # Return the cache TTL for the request.
+    #
+    # @return [ Integer, nil ]
+    def cache_ttl
+      options[:cache_ttl]
+    end
+
+    # Set the response and execute the callbacks.
+    #
+    # @param [ Typhoeus::Response ] response The response.
+    # @param [ Boolean ] bypass_memoization Whether to bypass memoization.
+    #
+    # @return [ Typhoeus::Response ] The response.
+    def finish(response, bypass_memoization = nil)
+      if bypass_memoization
+        @response = response
+      else
+        self.response = response
+      end
+      self.response.request = self
+      execute_callbacks
+      response
+    end
+
+    # Return the response.
+    #
+    # @return [ Typhoeus::Response, nil ]
+    def response
+      @response ||= nil
+    end
+
+    # Set the response. Cache and memoization hooks run explicitly before the
+    # response is assigned, instead of relying on the order modules were
+    # included in.
+    #
+    # @param [ Typhoeus::Response ] value The response to set.
+    def response=(value)
+      if cacheable? && cacheable_response?(value)
+        # The request exposes this response to its callbacks and the caller.
+        # Store a snapshot so later mutation cannot change what a cache hit
+        # returns.
+        cache.set(self, value.dup)
+      end
+      # Followers receive a copy of the stored response, and the leader's
+      # callbacks must not change what they see. Store a snapshot instead of
+      # the response the leader exposes to its own callbacks.
+      hydra.memory[self] = value.dup if memoizable? && hydra
+      @response = value
+    end
+
     # Return the url.
     # In contrast to base_url which returns the value you specified, url returns
     # the full url including the parameters.
@@ -127,10 +293,7 @@ module Typhoeus
     #
     # @since 0.5.5
     def url
-      easy = EasyFactory.new(self).get
-      url = easy.url
-      Typhoeus::Pool.release(easy)
-      url
+      EasyFactory.new(self).url
     end
 
     # Returns whether other is equal to self.
@@ -145,8 +308,8 @@ module Typhoeus
     # @api private
     def eql?(other)
       self.class == other.class &&
-        self.base_url == other.base_url &&
-        fuzzy_hash_eql?(self.options, other.options)
+        base_url.to_s == other.base_url.to_s &&
+        fuzzy_hash_eql?(normalized_options, other.normalized_options)
     end
 
     # Overrides Object#hash.
@@ -163,22 +326,118 @@ module Typhoeus
     #
     # @return [ String ] The cache key.
     def cache_key
-      Digest::SHA1.hexdigest "#{self.class.name}#{base_url}#{hashable_string_for(options)}"
+      # The cache store and the TTL only affect where and how long a response
+      # is stored, not what the response is. Leaving them in the key would
+      # silently miss the cache whenever the TTL changes.
+      hashable_options = normalized_options.tap do |dup_opts|
+        dup_opts.delete(:cache)
+        dup_opts.delete(:cache_ttl)
+      end
+
+      cache_key_components = [
+        self.class.name,
+        base_url,
+        hashable_string_for(hashable_options)
+      ]
+
+      Digest::SHA1.hexdigest(cache_key_components.join)
     end
 
-    # Mimics libcurls POST body generation. This is not accurate, but good
+    # Mimics libcurl's POST body generation. This is not accurate, but good
     # enough for VCR.
     #
     # @return [ String ] The encoded body.
-    #   otherwise.
     #
     # @api private
     def encoded_body
-      Ethon::Easy::Form.new(nil, options[:body]).to_s
+      easy = Typhoeus::Pool.get
+      begin
+        body_options = { :body => options[:body] }
+        [:params_encoding, :escape].each do |option|
+          body_options[option] = options[option] if options.key?(option)
+        end
+
+        # Let Ethon apply its escape rule for this method. The form must stay
+        # non-multipart because encoded_body returns a single string.
+        form = Ethon::Easy::Form.new(easy, options[:body])
+        form.define_singleton_method(:multipart?) { false }
+
+        action = if options.fetch(:method, :get).to_s.downcase == "put"
+          Ethon::Easy::Http::Put.new("http://localhost/", body_options)
+        else
+          Ethon::Easy::Http::Post.new("http://localhost/", body_options)
+        end
+        action.instance_variable_set(:@form, form)
+        action.setup(easy)
+        form.to_s
+      ensure
+        Typhoeus::Pool.release(easy) if easy
+      end
+    end
+
+    # Return the request method with the same defaults and spelling Ethon
+    # uses. Cache keys, memoization, and equality must not depend on whether
+    # the caller wrote :get, "GET", or omitted the default method.
+    #
+    # @return [ Symbol ] The normalized method.
+    #
+    # @api private
+    def request_method
+      (options[:method] || :get).to_s.downcase.to_sym
+    end
+
+    # Return the options with their implicit defaults filled in for comparison.
+    #
+    # @return [ Hash ] The normalized options.
+    #
+    # @api private
+    def normalized_options
+      normalized = options.merge(:method => request_method)
+      normalized.delete("method")
+      # GET is the default. Leaving it out keeps the cache key of an implicit
+      # GET stable across the normalization change.
+      normalized.delete(:method) if normalized[:method] == :get
+      normalized
     end
 
     private
 
+    # Perform the actual request.
+    #
+    # @return [ Typhoeus::Response ] The response.
+    def perform
+      easy_factory = EasyFactory.new(self)
+      easy = easy_factory.get
+      begin
+        easy.perform
+      rescue Exception
+        easy_factory.release_easy(easy) unless easy_factory.released?
+        raise
+      end
+      response
+    end
+
+    # Return the cache for the request, falling back to the global cache.
+    #
+    # @return [ Object, nil ]
+    def cache
+      return nil if options[:cache] === false
+      options[:cache] || Typhoeus::Config.cache
+    end
+
+    # A streaming response has no buffered body, and a transport failure or a
+    # stub is not a representation of the resource.
+    #
+    # @param [ Typhoeus::Response ] response The response to cache.
+    #
+    # @return [ Boolean ] Whether the response should be cached.
+    def cacheable_response?(response)
+      !response.cached? &&
+        !streaming? &&
+        !response.mock? &&
+        (response.return_code.nil? || response.return_code == :ok)
+    end
+
     # Checks if two hashes are equal or not, discarding
     # first-level hash order.
     #
@@ -199,11 +458,16 @@ module Typhoeus
     def hashable_string_for(obj)
       case obj
       when Hash
-        hashable_string_for(obj.sort_by {|sub_obj| sub_obj.first.to_s})
+        pairs = obj.map do |key, value|
+          "#{hashable_string_for(key)}#{hashable_string_for(value)}"
+        end
+        "hash:#{obj.size}:#{pairs.sort.join}"
       when Array
-        obj.map {|sub_obj| hashable_string_for(sub_obj)}.to_s
+        values = obj.map { |value| hashable_string_for(value) }
+        "array:#{obj.size}:#{values.join}"
       else
-        obj.to_s
+        value = obj.to_s
+        "#{obj.class}:#{value.bytesize}:#{value}"
       end
     end
 
@@ -211,13 +475,36 @@ module Typhoeus
     def set_defaults
       default_user_agent = Config.user_agent || Typhoeus::USER_AGENT
 
-      options[:headers] = {'User-Agent' => default_user_agent}.merge(options[:headers] || {})
-      options[:headers]['Expect'] ||= ''
+      headers = (options[:headers] || {}).dup
+      unless header_key?(headers, "User-Agent")
+        headers = {"User-Agent" => default_user_agent}.merge(headers)
+      end
+      # An empty value suppresses libcurl's 100-continue. A nil value opts out
+      # of the header entirely, which lets libcurl add it when it sees fit.
+      unless header_key?(headers, "Expect")
+        headers["Expect"] = ""
+      end
+      headers.reject! { |key, value| key.to_s.casecmp?("Expect") && value.nil? }
+      options[:headers] = headers
       options[:verbose] = Typhoeus::Config.verbose if options[:verbose].nil? && !Typhoeus::Config.verbose.nil?
       options[:timeout] = Typhoeus::Config.timeout if options[:timeout].nil? && !Typhoeus::Config.timeout.nil?
       options[:connecttimeout] = Typhoeus::Config.connecttimeout if options[:connecttimeout].nil? && !Typhoeus::Config.connecttimeout.nil?
       options[:maxredirs] ||= 50
       options[:proxy] = Typhoeus::Config.proxy unless options.has_key?(:proxy) || Typhoeus::Config.proxy.nil?
+      options[:cainfo] = ENV["SSL_CERT_FILE"] if options[:cainfo].nil? && ENV["SSL_CERT_FILE"]
+      options[:capath] = ENV["SSL_CERT_DIR"] if options[:capath].nil? && ENV["SSL_CERT_DIR"]
+    end
+
+    # Check the headers for a name without relying on its casing. Header names
+    # are case-insensitive and libcurl sends every spelling it is given, so a
+    # second default header would be added to the caller's value.
+    #
+    # @param [ Hash ] headers The request headers.
+    # @param [ String ] name The header name to look for.
+    #
+    # @return [ Boolean ] Whether the header is already present.
+    def header_key?(headers, name)
+      headers.keys.any? { |key| key.to_s.casecmp?(name) }
     end
   end
 end

data/lib/typhoeus/response/cacheable.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Typhoeus
   class Response
     module Cacheable

data/lib/typhoeus/response/header.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'delegate'
 
 module Typhoeus
@@ -23,8 +25,173 @@ module Typhoeus
         parse
       end
 
+      def initialize_dup(other)
+        super
+        copy_delegate(other)
+      end
+      private :initialize_dup
+
+      def initialize_clone(other, freeze: nil)
+        if freeze.nil?
+          super(other)
+        else
+          super(other, freeze: freeze)
+        end
+        copy_delegate(other)
+      end
+      private :initialize_clone
+
       def [](key)
-        fetch(key) { @sanitized[key.to_s.downcase] }
+        @sanitized[convert_key(key)]
+      end
+
+      def []=(key, value)
+        # Store string keys so the delegated Hash and the case-insensitive
+        # index describe the same header set even when a caller used a symbol.
+        key = key.to_s
+        delete_underlying_key(key)
+        __getobj__[key] = value
+        @sanitized[convert_key(key)] = value
+      end
+      alias_method :store, :[]=
+
+      def merge(other)
+        merged = dup
+        merged.update(other) do |key, old, new|
+          block_given? ? yield(key, old, new) : new
+        end
+        merged
+      end
+
+      def delete(key)
+        value = self[key]
+        delete_underlying_key(key)
+        @sanitized.delete(convert_key(key))
+        value
+      end
+
+      def clear
+        __getobj__.clear
+        @sanitized.clear
+        self
+      end
+
+      def update(other)
+        other.each_pair do |key, value|
+          value = yield(key, self[key], value) if block_given? && include?(key)
+          self[key] = value
+        end
+        self
+      end
+      alias_method :merge!, :update
+
+      def replace(other)
+        clear
+        update(other)
+      end
+
+      def delete_if(&block)
+        return enum_for(:delete_if) unless block
+
+        super
+        rebuild_sanitized
+        self
+      end
+
+      def keep_if(&block)
+        return enum_for(:keep_if) unless block
+
+        super
+        rebuild_sanitized
+        self
+      end
+
+      def select!(&block)
+        return enum_for(:select!) unless block
+
+        result = super
+        rebuild_sanitized
+        result ? self : nil
+      end
+      alias_method :filter!, :select!
+
+      def reject!(&block)
+        return enum_for(:reject!) unless block
+
+        result = super
+        rebuild_sanitized
+        result ? self : nil
+      end
+
+      def shift
+        result = super
+        rebuild_sanitized
+        result
+      end
+
+      def compact!
+        result = super
+        rebuild_sanitized
+        result ? self : nil
+      end
+
+      def transform_keys!(*args, &block)
+        return enum_for(:transform_keys!, *args) if !block && args.empty?
+
+        super
+        rebuild_sanitized
+        self
+      end
+
+      def transform_values!(&block)
+        return enum_for(:transform_values!) unless block
+
+        super
+        rebuild_sanitized
+        self
+      end
+
+      def include?(key)
+        @sanitized.include?(convert_key(key))
+      end
+      alias_method :has_key?, :include?
+      alias_method :key?, :include?
+      alias_method :member?, :include?
+
+      def fetch(key, *extras, &block)
+        if block
+          # Hash#fetch yields the key the caller asked for, not a normalized
+          # one. Keep that behavior while looking up case-insensitively.
+          @sanitized.fetch(convert_key(key), *extras) { block.call(key) }
+        else
+          @sanitized.fetch(convert_key(key), *extras)
+        end
+      end
+
+      def dig(key, *identifiers)
+        @sanitized.dig(convert_key(key), *identifiers)
+      end
+
+      def assoc(key)
+        @sanitized.assoc(convert_key(key))
+      end
+
+      def values_at(*keys)
+        keys.map! { |key| convert_key(key) }
+        @sanitized.values_at(*keys)
+      end
+
+      def fetch_values(*indices, &block)
+        indices.map do |key|
+          converted = convert_key(key)
+          if @sanitized.key?(converted)
+            @sanitized[converted]
+          elsif block
+            block.call(key)
+          else
+            @sanitized.fetch(converted)
+          end
+        end
       end
 
       # Parses the raw header.
@@ -32,12 +199,11 @@ module Typhoeus
       # @example Parse header.
       #   header.parse
       def parse
-        case @raw
-        when Hash
-          raw.each do |k, v|
+        if raw.respond_to?(:each_pair)
+          raw.each_pair do |k, v|
             process_pair(k, v)
           end
-        when String
+        elsif raw.is_a?(String)
           raw.split(/\r?\n(?!\s)/).each do |header|
             header.strip!
             next if header.empty? || header.start_with?( 'HTTP/' )
@@ -60,8 +226,9 @@ module Typhoeus
       #
       # @return [ void ]
       def process_pair(key, value)
-        set_value(key, value, self)
-        @sanitized[key.downcase] = self[key]
+        sanitized_key = convert_key(key)
+        set_value(sanitized_key, duplicate_value(value), @sanitized)
+        self[key] = @sanitized[sanitized_key]
       end
 
       # Sets value for key in specified hash
@@ -90,14 +257,67 @@ module Typhoeus
         @raw || ''
       end
 
-      # Sets the default proc for the specified hash independent of the Ruby version.
-      #
-      # @return [ void ]
-      def set_default_proc_on(hash, default_proc)
-        if hash.respond_to?(:default_proc=)
-          hash.default_proc = default_proc
+      def convert_key(key)
+        key.to_s.downcase
+      end
+
+      def duplicate_value(value)
+        case value
+        when Array
+          value.map { |nested| duplicate_value(nested) }
+        when String
+          value.dup
         else
-          hash.replace(Hash.new(&default_proc).merge(hash))
+          value
+        end
+      end
+
+      # DelegateClass copies only the delegated hash itself. Rebuild both
+      # hashes so later mutations cannot reach the original header. The
+      # sanitized index is built from the copied hash so both stores keep
+      # referring to the same values, as they do on the original.
+      def copy_delegate(other)
+        copy = {}
+        other.__getobj__.each do |key, value|
+          copy[key] = duplicate_value(value)
+        end
+        __setobj__(copy)
+
+        @sanitized = {}
+        copy.each do |key, value|
+          converted = convert_key(key)
+          if @sanitized.key?(converted)
+            existing = @sanitized[converted]
+            existing = [existing] unless existing.is_a?(Array)
+            existing.concat(value.is_a?(Array) ? value : [value])
+            @sanitized[converted] = existing
+          else
+            @sanitized[converted] = value
+          end
+        end
+      end
+
+      def delete_underlying_key(key)
+        converted = convert_key(key)
+        __getobj__.keys.each do |existing_key|
+          __getobj__.delete(existing_key) if convert_key(existing_key) == converted
+        end
+      end
+
+      # Rebuild the case-insensitive index after a Hash mutator that cannot be
+      # expressed through #[]=.
+      def rebuild_sanitized
+        @sanitized = {}
+        __getobj__.each do |key, value|
+          converted = convert_key(key)
+          if @sanitized.key?(converted)
+            existing = @sanitized[converted]
+            existing = [existing] unless existing.is_a?(Array)
+            existing.concat(value.is_a?(Array) ? value : [value])
+            @sanitized[converted] = existing
+          else
+            @sanitized[converted] = value
+          end
         end
       end
     end