typhoeus 1.6.0 → 1.7.0

data/lib/typhoeus/easy_factory.rb

@@ -1,4 +1,7 @@
+# frozen_string_literal: true
+
 require 'set'
+require 'typhoeus/errors/invalid_option'
 
 module Typhoeus
 
@@ -35,8 +38,9 @@ module Typhoeus
 
     REMOVED_OPTIONS =  Set.new([:cache_key_basis, :cache_timeout, :user_agent])
 
-    SANITIZE_IGNORE  = Set.new([:method, :cache_ttl, :cache])
+    SANITIZE_IGNORE  = Set.new([:method, :cache_ttl, :cache, :set_read_callback])
     SANITIZE_TIMEOUT = Set.new([:timeout_ms, :connecttimeout_ms])
+    URL_OPTIONS      = Set.new([:params, :params_encoding, :escape])
 
     # Returns the request provided.
     #
@@ -58,6 +62,16 @@ module Typhoeus
     def initialize(request, hydra = nil)
       @request = request
       @hydra = hydra
+      @released = false
+    end
+
+    # Returns whether the easy was already returned to the pool.
+    #
+    # @return [ Boolean ]
+    #
+    # @api private
+    def released?
+      @released
     end
 
     # Return the easy in question.
@@ -77,26 +91,53 @@ module Typhoeus
     #
     # @return [ Ethon::Easy ] The easy.
     def get
+      @released = false
       begin
-        read_callback_body = request.options.delete(:set_read_callback)
+        read_callback_body = request.options[:set_read_callback]
+        easy = self.easy
+        url = request.base_url.to_s
+        options = sanitize(request.options)
+        options.delete(:params) if url.empty?
 
         easy.http_request(
-          request.base_url.to_s,
+          url,
           request.options.fetch(:method, :get),
-          sanitize(request.options)
+          options
         )
 
         # this needs to happen after http_request because
         # ethon will set infilesize to zero if form.empty?
         set_read_callback(read_callback_body) if !read_callback_body.nil?
-      rescue Ethon::Errors::InvalidOption => e
-        help = provide_help(e.message.match(/:\s(\w+)/)[1])
-        raise $!, "#{$!}#{help}", $!.backtrace
+        set_callback
+      rescue Exception
+        release_easy(@easy) if @easy
+        raise
       end
-      set_callback
       easy
     end
 
+    # Returns the full request URL without applying unrelated request options.
+    # This keeps introspection such as Request#url and the NoStub error working
+    # even when another option would be rejected by Ethon.
+    #
+    # @return [ String ] The URL including params.
+    def url
+      @released = false
+      easy = self.easy
+      begin
+        url = request.base_url.to_s
+        options = sanitize(
+          request.options.select { |key, _| URL_OPTIONS.include?(key.to_sym) }
+        )
+        options.delete(:params) if url.empty?
+
+        easy.http_request(url, :get, options)
+        easy.url
+      ensure
+        release_easy(easy)
+      end
+    end
+
     private
 
     def sanitize(options)
@@ -109,8 +150,21 @@ module Typhoeus
         if new_option = RENAMED_OPTIONS[k.to_sym]
           warn("Deprecated option #{k}. Please use #{new_option} instead.")
           sanitized[new_option] = v
+        elsif new_option = CHANGED_OPTIONS[s]
+          raise Errors::InvalidOption.new(
+            s,
+            "\nPlease try #{new_option} instead of #{s}."
+          )
+        elsif REMOVED_OPTIONS.include?(s)
+          raise Errors::InvalidOption.new(s, "\nThe option #{s} was removed.")
         # sanitize timeouts
         elsif SANITIZE_TIMEOUT.include?(s)
+          # An unset value means "no timeout" and never reaches Ethon.
+          next if v.nil?
+
+          unless v.is_a?(Numeric)
+            raise ArgumentError, "Value '#{v}' for option '#{k}' must be a number."
+          end
           if !v.integer?
             warn("Value '#{v}' for option '#{k}' must be integer.")
           end
@@ -128,15 +182,52 @@ module Typhoeus
 
     def sanitize_timeout!(options, timeout)
       timeout_ms = :"#{timeout}_ms"
-      if options[timeout] && options[timeout].round != options[timeout]
+      value = options[timeout]
+      return options if value.nil?
+
+      unless value.is_a?(Numeric)
+        raise ArgumentError, "Value '#{value}' for option '#{timeout}' must be a number."
+      end
+
+      if value.round != value
         if !options[timeout_ms]
-          options[timeout_ms] = (options[timeout]*1000).ceil
+          options[timeout_ms] = (value*1000).ceil
         end
-        options[timeout] = options[timeout].ceil
+        options[timeout] = value.ceil
+      end
+
+      # CURLOPT_TIMEOUT and CURLOPT_TIMEOUT_MS set the same value, so libcurl
+      # uses the one that is applied last. Keep the millisecond option last:
+      # an explicit timeout_ms must also win over a timeout added from Config.
+      if options.key?(timeout_ms)
+        options[timeout_ms] = options.delete(timeout_ms)
       end
       options
     end
 
+    public
+
+    # Releases the easy into the pool once. Easies that used a custom
+    # resolve list are cleaned up instead because libcurl keeps the
+    # resolved addresses in its DNS cache across resets.
+    #
+    # @param [ Ethon::Easy ] easy The easy to release.
+    #
+    # @return [ void ]
+    #
+    # @api private
+    def release_easy(easy)
+      return if released?
+
+      begin
+        Typhoeus::Pool.release(easy, :cleanup => !request.options[:resolve].nil?)
+      ensure
+        @released = true
+      end
+    end
+
+    private
+
     # Sets on_complete callback on easy in order to be able to
     # track progress.
     #
@@ -161,14 +252,16 @@ module Typhoeus
           request.execute_headers_callbacks(Response.new(Ethon::Easy::Mirror.from_easy(easy).options))
         end
       end
-      request.on_progress.each do |callback|
-        easy.on_progress do |dltotal, dlnow, ultotal, ulnow, easy|
-          callback.call(dltotal, dlnow, ultotal, ulnow, response)
-        end
+      # Ethon calls progress callbacks with the four transfer counters.
+      (Typhoeus.on_progress + request.on_progress).each do |callback|
+        easy.on_progress(&callback)
       end
       easy.on_complete do |easy|
-        request.finish(Response.new(easy.mirror.options))
-        Typhoeus::Pool.release(easy)
+        begin
+          request.finish(Response.new(easy.mirror.options))
+        ensure
+          release_easy(easy)
+        end
         if hydra && !hydra.queued_requests.empty?
           hydra.dequeue_many
         end
@@ -182,6 +275,11 @@ module Typhoeus
     #
     # @return [ Ethon::Easy ] The easy.
     def set_read_callback(body)
+      # Uploading a file means sending it from the beginning. An IO that was
+      # already read from would otherwise be shorter than the advertised
+      # infilesize and libcurl would fail with a read error.
+      rewind_body(body)
+
       easy.infilesize_large =
         if body.respond_to?(:bytesize)
           body.bytesize
@@ -195,12 +293,20 @@ module Typhoeus
       easy
     end
 
-    def provide_help(option)
-      if new_option = CHANGED_OPTIONS[option.to_sym]
-        "\nPlease try #{new_option} instead of #{option}." if new_option
-      elsif REMOVED_OPTIONS.include?(option.to_sym)
-        "\nThe option #{option} was removed."
-      end
+    # Rewind a seekable body so the read callback starts at its first byte.
+    # Pipes and other non-seekable bodies do not respond to rewind.
+    #
+    # @param [ Object ] body The body to rewind.
+    #
+    # @return [ void ]
+    def rewind_body(body)
+      return unless body.respond_to?(:rewind)
+
+      body.rewind
+    rescue IOError, SystemCallError
+      # A body that cannot be rewound is still usable from its current
+      # position. Do not turn that into a request setup error.
+      nil
     end
   end
 end

data/lib/typhoeus/errors/invalid_option.rb

@@ -0,0 +1,23 @@
+# frozen_string_literal: true
+
+module Typhoeus
+  module Errors
+
+    # Raised for options Typhoeus knows about but Ethon does not accept.
+    #
+    # It stays compatible with Ethon's InvalidOption while carrying the
+    # replacement or removal hint that Typhoeus adds.
+    class InvalidOption < Ethon::Errors::InvalidOption
+      def initialize(option, help = nil)
+        @help = help
+        super(option)
+      end
+
+      # Exception#message calls #to_s, so the hint is included everywhere the
+      # error is rendered and not only when #message is called explicitly.
+      def to_s
+        "#{super}#{@help}"
+      end
+    end
+  end
+end

data/lib/typhoeus/errors/no_response.rb

@@ -0,0 +1,18 @@
+# frozen_string_literal: true
+
+require 'typhoeus/errors/typhoeus_error'
+
+module Typhoeus
+  module Errors
+
+    # Raised when a request matches a stub that does not provide a usable
+    # response.
+    class NoResponse < TyphoeusError
+      def initialize(base_url, reason = nil)
+        message = "No response defined for stub: #{base_url.inspect}"
+        message = "#{message} (#{reason})" if reason
+        super(message)
+      end
+    end
+  end
+end

data/lib/typhoeus/errors/no_stub.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Typhoeus
   module Errors
 
@@ -5,7 +7,12 @@ module Typhoeus
     # and making a real request.
     class NoStub < TyphoeusError
       def initialize(request)
-        super("The connection is blocked and no stub defined: #{request.url}")
+        url = begin
+          request.url
+        rescue StandardError
+          request.base_url
+        end
+        super("The connection is blocked and no stub defined: #{url}")
       end
     end
   end

data/lib/typhoeus/errors/typhoeus_error.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Typhoeus
   module Errors
 

data/lib/typhoeus/errors.rb

@@ -1,5 +1,9 @@
+# frozen_string_literal: true
+
 require 'typhoeus/errors/typhoeus_error'
 require 'typhoeus/errors/no_stub'
+require 'typhoeus/errors/no_response'
+require 'typhoeus/errors/invalid_option'
 
 module Typhoeus
 

data/lib/typhoeus/expectation.rb

@@ -1,3 +1,7 @@
+# frozen_string_literal: true
+
+require 'typhoeus/errors/no_response'
+
 module Typhoeus
 
   # This class represents an expectation. It is part
@@ -49,6 +53,16 @@ module Typhoeus
     # @api private
     attr_reader :from
 
+    # Whenever an expectation is triggered this counter increases which can be used in testing to ensure that a defined stub is actually used.
+    #
+    # @example Stub a request and check if it was used
+    #   expectation = Typhoeus.stub("www.example.com").and_return(Typhoeus::Response.new)
+    #
+    #   Typhoeus.get("www.example.com")
+    #   expectation.response_counter
+    #   #=> 1
+    attr_reader :response_counter
+
     class << self
 
       # Returns all expectations.
@@ -179,11 +193,29 @@ module Typhoeus
     # @api private
     def response(request)
       response = responses.fetch(@response_counter, responses.last)
+      raise Errors::NoResponse.new(base_url) if response.nil?
+
       if response.respond_to?(:call)
         response = response.call(request)
+        if response.nil?
+          raise Errors::NoResponse.new(
+            base_url,
+            "the response block returned nil"
+          )
+        end
       end
+
+      unless response.is_a?(Response)
+        raise Errors::NoResponse.new(
+          base_url,
+          "expected a Typhoeus::Response, got #{response.class}"
+        )
+      end
+
       @response_counter += 1
-      response.mock = @from || true
+      # A stub block can return the response of a real request, which must keep
+      # its real mock state and must not be finished a second time.
+      response.mock = @from || true unless response.request
       response
     end
 

data/lib/typhoeus/hydra/queueable.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Typhoeus
   class Hydra
 
@@ -67,17 +69,40 @@ module Typhoeus
       # adds them to the hydra until max_concurrency
       # is reached.
       #
+      # Short-circuited requests call this method while it is already
+      # draining the queue. The guard keeps that from recursing.
+      #
       # @example Dequeue requests.
       #   hydra.dequeue_many
       #
       # @since 0.6.8
       def dequeue_many
-        number = multi.easy_handles.count
-        until number == max_concurrency || queued_requests.empty?
-          add(queued_requests.shift)
-          number += 1
+        return if @dequeuing_many
+
+        begin
+          @dequeuing_many = true
+          until queued_requests.empty?
+            break if easy_handle_count >= max_concurrency
+
+            add(queued_requests.shift)
+          end
+        ensure
+          @dequeuing_many = false
         end
       end
+
+      private
+
+      # Count the attached easies without creating the native multi handle.
+      # Requests that a before callback, a stub, or the cache answers never
+      # need a multi, and Ethon 0.19 rejects a handle created before a fork.
+      #
+      # @return [ Integer ] The number of running easy handles.
+      def easy_handle_count
+        return 0 unless defined?(@multi) && @multi
+
+        multi.easy_handles.count
+      end
     end
   end
 end