typhoeus 1.6.0 → 1.7.0

data/lib/typhoeus/hydra.rb

@@ -1,11 +1,6 @@
-require 'typhoeus/hydra/addable'
-require 'typhoeus/hydra/before'
-require 'typhoeus/hydra/cacheable'
-require 'typhoeus/hydra/block_connection'
-require 'typhoeus/hydra/memoizable'
+# frozen_string_literal: true
+
 require 'typhoeus/hydra/queueable'
-require 'typhoeus/hydra/runnable'
-require 'typhoeus/hydra/stubbable'
 
 module Typhoeus
 
@@ -39,21 +34,13 @@ module Typhoeus
   # @note Callbacks are going to delay the request
   #   execution.
   class Hydra
-    include Hydra::Addable
-    include Hydra::Runnable
-    include Hydra::Memoizable
-    include Hydra::Cacheable
-    include Hydra::BlockConnection
-    include Hydra::Stubbable
-    include Hydra::Before
     include Hydra::Queueable
 
     # @example Set max_concurrency.
     #   Typhoeus::Hydra.new(max_concurrency: 20)
-    attr_accessor :max_concurrency
-
-    # @api private
-    attr_reader :multi
+    #
+    # @return [ Integer ] The maximum number of concurrent requests.
+    attr_reader :max_concurrency
 
     class << self
 
@@ -87,9 +74,350 @@ module Typhoeus
     # @see http://rubydoc.info/github/typhoeus/ethon/Ethon/Multi#initialize-instance_method
     #   Ethon::Multi#initialize
     def initialize(options = {})
-      @options = options
-      @max_concurrency = Integer(@options.fetch(:max_concurrency, 200))
-      @multi = Ethon::Multi.new(options.reject{|k,_| k==:max_concurrency})
+      @options = options.dup
+      @multi_creation_failed = false
+      self.max_concurrency = @options.fetch(:max_concurrency, 200)
+    end
+
+    # Return the Ethon multi, creating it on first use. Constructing a Multi
+    # initializes libcurl, which Ethon 0.19 refuses to do in a process that
+    # forked after the parent initialized it. Creating or memoizing a Hydra
+    # during boot must therefore not create the native handle yet.
+    #
+    # @return [ Ethon::Multi ] The multi handle.
+    #
+    # @api private
+    def multi
+      @multi ||= build_multi
+    end
+
+    # Add a request to the hydra.
+    #
+    # This is the hot path for every request. The checks are unrolled instead
+    # of spread over a chain of included modules so a request that uses no
+    # optional features takes the shortest route to the multi.
+    #
+    # @example Add request.
+    #   hydra.add(request)
+    #
+    # @param [ Typhoeus::Request ] request The request to add.
+    #
+    # @return [ Object ] The result of the handling callback or the multi add.
+    def add(request)
+      request.hydra = self
+
+      before_callbacks = Typhoeus.before_callbacks_for(request)
+      if before_callbacks
+        before_callbacks.each do |callback|
+          value = callback.call(request)
+          if value.is_a?(Response)
+            unless request.response
+              # A synthetic response is a short circuit like a stub. Mark it
+              # so success? works and it is not written to the cache.
+              value.mock = true
+              request.execute_headers_callbacks(value)
+              request.on_body.each { |body_callback| body_callback.call(value.body, value) }
+              request.finish(value)
+            end
+            dequeue_many
+            return value
+          elsif value.nil? || value == false
+            dequeue_many
+            return value
+          end
+        end
+      end
+
+      unless Typhoeus.stubs_disabled?
+        if response = Expectation.response_for(request)
+          # The stub block already ran this request for real.
+          return if response.request.equal?(request)
+
+          request.execute_headers_callbacks(response)
+          request.on_body.each { |callback| callback.call(response.body, response) }
+          return request.finish(response)
+        end
+      end
+
+      raise Typhoeus::Errors::NoStub.new(request) if request.blocked?
+
+      if request.cacheable? && response = request.cached_response
+        request.finish_cached_response(response)
+        return dequeue_many
+      end
+
+      if request.memoizable?
+        add_memoized(request)
+      else
+        add_to_multi(request)
+      end
+    end
+
+    # Return the memory.
+    #
+    # @example Get memory.
+    #   hydra.memory
+    #
+    # @return [ Hash ] The memory.
+    def memory
+      @memory ||= {}
+    end
+
+    # Requests that are waiting for an identical request to finish.
+    #
+    # @return [ Hash ]
+    #
+    # @api private
+    def pending
+      @pending ||= {}
+    end
+
+    # Start the hydra run.
+    #
+    # @example Start hydra run.
+    #   hydra.run
+    #
+    # @return [ Symbol ] Return value from multi.perform.
+    def run
+      memory.clear
+      dequeue_many
+      # Nothing reached the multi (all requests were answered by a before
+      # callback, a stub, or the cache), so do not initialize libcurl.
+      return if !defined?(@multi) || @multi.nil?
+
+      multi.perform
+    rescue Exception => error
+      begin
+        cleanup_after_error
+        reset_multi if multi_handle_unusable?(error)
+      rescue StandardError
+        # Preserve the original error.
+      end
+      raise
+    ensure
+      finish_pending_after_error
+      memory.clear
+      pending.clear
+    end
+
+    # Set the maximum number of concurrent requests.
+    #
+    # @example Set max_concurrency.
+    #   hydra.max_concurrency = 20
+    #
+    # @param [ Integer, String ] value The maximum number of requests to run
+    #   at the same time.
+    #
+    # @raise [ ArgumentError ] When the value is not greater than zero.
+    #
+    # @return [ Integer ] The maximum number of concurrent requests.
+    def max_concurrency=(value)
+      concurrency = Integer(value)
+      if concurrency <= 0
+        raise ArgumentError, "max_concurrency must be greater than zero"
+      end
+      @max_concurrency = concurrency
+    end
+
+    private
+
+    # Add the easy handle for a request to the multi.
+    #
+    # @param [ Typhoeus::Request ] request The request to add.
+    #
+    # @return [ Object ] The result of the multi add.
+    def add_to_multi(request)
+      easy_factory = EasyFactory.new(request, self)
+      easy = easy_factory.get
+      begin
+        multi.add(easy)
+      rescue Exception
+        easy_factory.release_easy(easy) unless easy_factory.released?
+        raise
+      end
+    end
+
+    # Answer a memoized request from memory or attach it to the request that
+    # is already in flight.
+    #
+    # @param [ Typhoeus::Request ] request The request to add.
+    #
+    # @return [ Object ]
+    def add_memoized(request)
+      if memory.key?(request)
+        finish_request(request, memory[request])
+        dequeue_many
+      elsif pending.key?(request)
+        pending[request] << request
+      else
+        pending[request] = []
+        request.on_complete do |response|
+          # Finish from the snapshot taken before the leader's callbacks
+          # ran. A callback that changes the leader's response must not
+          # change the copy its followers receive. Keep the leader's own
+          # callback result as this callback's return value.
+          snapshot = memoized_response_for(request) || response
+          finish_pending(request, snapshot)
+          response.handled_response
+        end
+        begin
+          add_to_multi(request)
+        rescue Exception
+          pending.delete(request)
+          raise
+        end
+      end
+    end
+
+    # Finish all requests that waited for the given request.
+    #
+    # @param [ Typhoeus::Request ] request
+    # @param [ Typhoeus::Response ] response
+    #
+    # @return [ Object ] The response's handled response.
+    def finish_pending(request, response)
+      requests = delete_pending(request)
+      return response.handled_response unless requests
+
+      error = nil
+      requests.each do |pending_request|
+        begin
+          finish_request(pending_request, response)
+        rescue StandardError => e
+          error ||= e
+        end
+      end
+      raise error if error
+
+      response.handled_response
+    end
+
+    # Finish a request whose response is already available. Every request gets
+    # its own response object and the same header callbacks it would see for
+    # a stub or a cache hit.
+    #
+    # @param [ Typhoeus::Request ] request The waiting request.
+    # @param [ Typhoeus::Response ] response The response to copy.
+    #
+    # @return [ Object ] The response's handled response.
+    def finish_request(request, response)
+      copy = response.dup
+      copy.request = request
+      request.execute_headers_callbacks(copy)
+      request.finish(copy, true)
+    end
+
+    # Finish followers whose leader produced a response but whose callbacks
+    # raised before finish_pending could run.
+    #
+    # @return [ void ]
+    def finish_pending_after_error
+      pending.to_a.each do |request, requests|
+        response = memoized_response_for(request)
+        next unless response
+
+        requests.each do |pending_request|
+          next if pending_request.response
+
+          begin
+            finish_request(pending_request, response)
+          rescue StandardError
+            # Preserve the original run error.
+          end
+        end
+      end
+    end
+
+    # Remove the waiting requests without hashing the leader again.
+    # Request#hash includes the request options, which a callback may have
+    # changed after the entry was added.
+    #
+    # @param [ Typhoeus::Request ] request The leader request.
+    #
+    # @return [ Array<Typhoeus::Request>, nil ] The waiting requests.
+    def delete_pending(request)
+      requests = nil
+      pending.delete_if do |pending_request, waiting_requests|
+        next false unless pending_request.equal?(request)
+
+        requests = waiting_requests
+        true
+      end
+      requests
+    end
+
+    # Find the leader's response by identity for the same reason. A changed
+    # option makes the request unusable as a hash key.
+    #
+    # @param [ Typhoeus::Request ] request The leader request.
+    #
+    # @return [ Typhoeus::Response, nil ] The memoized response.
+    def memoized_response_for(request)
+      memory.each_pair do |memoized_request, response|
+        return response if memoized_request.equal?(request)
+      end
+      nil
+    end
+
+    # Clean up the easies a failed run left behind. Requests that never reached
+    # the multi stay queued, so a caller can retry after handling the error.
+    #
+    # @return [ void ]
+    def cleanup_after_error
+      if defined?(@multi) && @multi
+        multi.easy_handles.dup.each do |easy|
+          begin
+            multi.delete(easy)
+          rescue StandardError
+            # The handle may already be unusable.
+          end
+          begin
+            Typhoeus::Pool.release(easy, :cleanup => true)
+          rescue StandardError
+            # Keep cleaning up the remaining handles.
+          end
+        end
+      end
+    ensure
+      # Invalid multi options make every queued request fail the same way.
+      queued_requests.clear if @multi_creation_failed
+    end
+
+    # A fatal multi_perform error leaves the native multi handle in an
+    # unknown state. Replace it so the hydra can run requests again.
+    #
+    # @return [ void ]
+    def reset_multi
+      begin
+        multi.cleanup if multi.respond_to?(:cleanup)
+      rescue StandardError
+        # Freeing the old handle is best effort; it is unusable anyway.
+      end
+      @multi = build_multi
+    end
+
+    # Ethon 0.19 raises MultiPerform when libcurl reports a fatal error from
+    # curl_multi_perform. Earlier versions do not have the class.
+    #
+    # @param [ Exception ] error The error raised by the run.
+    #
+    # @return [ Boolean ] Whether the multi handle has to be replaced.
+    def multi_handle_unusable?(error)
+      defined?(Ethon::Errors::MultiPerform) &&
+        error.is_a?(Ethon::Errors::MultiPerform)
+    end
+
+    # Build the multi handle from the options the hydra was created with.
+    #
+    # @return [ Ethon::Multi ] The new multi handle.
+    #
+    # @api private
+    def build_multi
+      @multi_creation_failed = false
+      Ethon::Multi.new(@options.reject { |key, _| key == :max_concurrency })
+    rescue StandardError
+      @multi_creation_failed = true
+      raise
     end
   end
 end

data/lib/typhoeus/pool.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require 'thread'
 
 module Typhoeus
@@ -16,11 +18,32 @@ module Typhoeus
     #
     # @example Release easy.
     #   Typhoeus::Pool.release(easy)
-    def self.release(easy)
-      easy.cookielist = "flush" # dump all known cookies to 'cookiejar'
-      easy.cookielist = "all" # remove all cookies from memory for this handle
+    #
+    # @example Clean up instead of pooling.
+    #   Typhoeus::Pool.release(easy, :cleanup => true)
+    def self.release(easy, cleanup: false)
+      reset_after_fork
+
+      if cleanup
+        easy.cleanup
+        return
+      end
+
+      clear_cookies(easy)
       easy.reset
-      @mutex.synchronize { easies << easy }
+
+      discarded = nil
+      @mutex.synchronize do
+        if max_size && easies.length >= max_size
+          discarded = easy
+        else
+          easies << easy
+        end
+      end
+      discarded.cleanup if discarded
+    rescue StandardError => e
+      # Ethon 0.19+ refuses to reset handles created before a fork.
+      raise unless defined?(Ethon::Errors::ForkSafety) && e.is_a?(Ethon::Errors::ForkSafety)
     end
 
     # Return an easy from the pool.
@@ -30,22 +53,34 @@ module Typhoeus
     #
     # @return [ Ethon::Easy ] The easy.
     def self.get
-      @mutex.synchronize do
-        if @pid == Process.pid
-          easies.pop
-        else
-          # Process has forked. Clear all easies to avoid sockets being
-          # shared between processes.
-          @pid = Process.pid
-          easies.clear
-          nil
-        end
-      end || Ethon::Easy.new
+      reset_after_fork
+      @mutex.synchronize { easies.pop } || Ethon::Easy.new
     end
 
-    # Clear the pool
+    # Clear the pool and release the native easy handles it owns.
+    #
+    # @example Clear the pool.
+    #   Typhoeus::Pool.clear
+    #
+    # @return [ Array<Ethon::Easy> ] The now empty pool.
     def self.clear
-      @mutex.synchronize { easies.clear }
+      reset_after_fork
+      cleared = @mutex.synchronize do
+        current = easies.dup
+        easies.clear
+        current
+      end
+
+      cleared.each do |easy|
+        begin
+          easy.cleanup
+        rescue StandardError
+          # Clearing the pool is best effort. A handle that cannot be freed
+          # must not keep the other pooled handles alive.
+        end
+      end
+
+      easies
     end
 
     # Use yielded easy, will be released automatically afterwards.
@@ -63,8 +98,36 @@ module Typhoeus
 
     private
 
+    # Drop state inherited from a parent process. The previous mutex may have
+    # been locked by a thread that does not exist after the fork, so it is
+    # replaced without locking it to avoid a deadlock.
+    def self.reset_after_fork
+      return if @pid == Process.pid
+
+      # Replace the lock before publishing the new pid so a thread that sees
+      # the new pid cannot pick up the inherited lock.
+      @mutex = Mutex.new
+      @easies = []
+      @pid = Process.pid
+    end
+
+    def self.max_size
+      Typhoeus::Config.max_idle_easies if defined?(Typhoeus::Config)
+    end
+
     def self.easies
       @easies ||= []
     end
+
+    # Flush cookies to the configured jar and clear them from the handle.
+    # libcurl builds without cookie support reject CURLOPT_COOKIELIST; that
+    # must not prevent the handle from being recycled.
+    def self.clear_cookies(easy)
+      easy.cookielist = "flush"
+      easy.cookielist = "all"
+    rescue StandardError => e
+      raise unless defined?(Ethon::Errors::UnsupportedOption) &&
+        e.is_a?(Ethon::Errors::UnsupportedOption)
+    end
   end
 end

data/lib/typhoeus/railtie.rb

@@ -1,11 +1,13 @@
+# frozen_string_literal: true
+
 require "typhoeus"
 
 module Rails
   module Typhoeus
     class Railtie < Rails::Railtie
       # Need to include the Typhoeus middleware.
-      initializer "include the identity map" do |app|
-        app.config.middleware.use "Rack::Typhoeus::Middleware::ParamsDecoder"
+      initializer "typhoeus.params_decoder" do |app|
+        app.config.middleware.use Rack::Typhoeus::Middleware::ParamsDecoder
       end
     end
   end

data/lib/typhoeus/request/actions.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Typhoeus
   class Request
 

data/lib/typhoeus/request/callbacks.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Typhoeus
   class Request
 
@@ -92,6 +94,9 @@ module Typhoeus
 
         # Set on_progress callback.
         #
+        # The callback is invoked by libcurl while the request transfers data.
+        # It is not called when the request completes.
+        #
         # @example Set on_progress.
         #   request.on_progress do |dltotal, dlnow, ultotal, ulnow|
         #     puts "dltotal (#{dltotal}), dlnow (#{dlnow}), ultotal (#{ultotal}), ulnow (#{ulnow})"
@@ -99,7 +104,8 @@ module Typhoeus
         #
         # @param [ Block ] block The block to execute.
         #
-        # @yield [ Typhoeus::Response ]
+        # @yield [ Integer, Integer, Integer, Integer ] The download total,
+        #   downloaded bytes, upload total, and uploaded bytes.
         #
         # @return [ Array<Block> ] All on_progress blocks.
         def on_progress(&block)
@@ -124,8 +130,9 @@ module Typhoeus
       end
 
       # Execute necessary callback and yields response. This
-      # include in every case on_complete and on_progress, on_success
-      # if successful and on_failure if not.
+      # includes on_complete in every case, on_success if successful
+      # and on_failure if not. Progress callbacks are invoked by libcurl
+      # while the transfer is running and are not called here.
       #
       # @example Execute callbacks.
       #   request.execute_callbacks
@@ -134,7 +141,7 @@ module Typhoeus
       #
       # @api private
       def execute_callbacks
-        callbacks = Typhoeus.on_complete + Typhoeus.on_progress + on_complete + on_progress
+        callbacks = Typhoeus.on_complete + on_complete
 
         if response && response.success?
           callbacks += Typhoeus.on_success + on_success

data/lib/typhoeus/request/marshal.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Typhoeus
   class Request
 
@@ -5,9 +7,10 @@ module Typhoeus
     module Marshal
 
       # Return the important data needed to serialize this Request, except the
-      # request callbacks and `hydra`, since they cannot be marshalled.
+      # request callbacks, `hydra`, and the response. The response may contain
+      # arbitrary callback results and can be serialized separately.
       def marshal_dump
-        unmarshallable = %w(@on_complete @on_success @on_failure @on_progress @on_headers @on_body @hydra)
+        unmarshallable = %w(@on_complete @on_success @on_failure @on_progress @on_headers @on_body @before @hydra @response)
         (instance_variables - unmarshallable - unmarshallable.map(&:to_sym)).map do |name|
           [name, instance_variable_get(name)]
         end

data/lib/typhoeus/request/streamable.rb

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 module Typhoeus
   class Request