typhoeus 0.4.2 → 0.5.0.rc

data/lib/typhoeus/hydra.rb

@@ -1,247 +1,94 @@
-require 'typhoeus/hydra/callbacks'
-require 'typhoeus/hydra/connect_options'
-require 'typhoeus/hydra/stubbing'
+require 'typhoeus/hydra/before'
+require 'typhoeus/hydra/block_connection'
+require 'typhoeus/hydra/easy_factory'
+require 'typhoeus/hydra/easy_pool'
+require 'typhoeus/hydra/memoizable'
+require 'typhoeus/hydra/queueable'
+require 'typhoeus/hydra/runnable'
+require 'typhoeus/hydra/stubbable'
 
 module Typhoeus
-  class Hydra
-    include ConnectOptions
-    include Stubbing
-    extend Callbacks
-
-    def initialize(options = {})
-      @memoize_requests = true
-      @multi       = Multi.new
-      @easy_pool   = []
-      initial_pool_size = options[:initial_pool_size] || 10
-      @max_concurrency = options[:max_concurrency] || 200
-      initial_pool_size.times { @easy_pool << Easy.new }
-      @memoized_requests = {}
-      @retrieved_from_cache = {}
-      @queued_requests = []
-      @running_requests = 0
-
-      self.stubs = []
-      @active_stubs = []
-    end
-
-    def self.hydra
-      @hydra ||= new
-    end
 
-    def self.hydra=(val)
-      @hydra = val
+  # Hydra manages making parallel HTTP requests. This
+  # is achieved by using libcurls multi interface:
+  # http://curl.haxx.se/libcurl/c/libcurl-multi.html
+  # The benefits are that you don't have to worry running
+  # the requests by yourself.
+  #
+  # Hydra will also handle how many requests you can
+  # make in parallel. Things will get flakey if you
+  # try to make too many requests at the same time.
+  # The built in limit is 200. When more requests than
+  # that are queued up, hydra will save them for later
+  # and start the requests as others are finished. You
+  # can raise or lower the concurrency limit through
+  # the Hydra constructor.
+  #
+  # Regarding the asynchronous behavior of the hydra,
+  # it is important to know that this is completely hidden
+  # from the developer and you are free to apply
+  # whatever technique you want to your code. That should not
+  # conflict with libcurls internal concurrency mechanism.
+  #
+  # @example Use the hydra to do multiple requests.
+  #   hydra = Typhoeus::Hydra.new
+  #   requests = (0..9).map{ Typhoeus::Request.new("www.example.com") }
+  #   requests.each{ |request| hydra.queue(request) }
+  #   hydra.run
+  class Hydra
+    include Hydra::EasyPool
+    include Hydra::Queueable
+    include Hydra::Runnable
+    include Hydra::Memoizable
+    include Hydra::BlockConnection
+    include Hydra::Stubbable
+    include Hydra::Before
+
+    # @example Set max_concurrency.
+    #   Typhoeus::Hydra.new(:max_concurrency => 20)
+    attr_reader :max_concurrency
+
+    # @api private
+    attr_reader :multi
+
+    class << self
+
+      # Returns a memoized hydra instance.
+      #
+      # @example Get a hydra.
+      #   Typhoeus::Hydra.hydra
+      #
+      # @return [Typhoeus::Hydra] A new hydra.
+      #
+      # @deprecated This is only for convenience because so
+      #   much external code relies on it.
+      def hydra
+        @hydra ||= new
+      end
     end
 
+    # Create a new hydra. All
+    # {http://rubydoc.info/github/typhoeus/ethon/Ethon/Multi#initialize-instance_method Ethon::Multi#initialize}
+    # options are also available.
     #
-    # Abort the run on a best-effort basis.
+    # @example Create a hydra.
+    #   Typhoeus::Hydra.new
     #
-    # It won't abort the current burst of @max_concurrency requests,
-    # however it won't fire the rest of the queued requests so the run
-    # will be aborted as soon as possible...
+    # @example Create a hydra with max_concurrency.
+    #   Typhoeus::Hydra.new(:max_concurrency => 20)
     #
-    def abort
-      @queued_requests.clear
-    end
-
-    def clear_cache_callbacks
-      @cache_setter = nil
-      @cache_getter = nil
-    end
-
-    def fire_and_forget
-      @queued_requests.each {|r| queue(r, false)}
-      @multi.fire_and_forget
-    end
-
-    def queue(request, obey_concurrency_limit = true)
-      return if assign_to_stub(request)
-
-      # At this point, we are running over live HTTP. Make sure we haven't
-      # disabled live requests.
-      check_allow_net_connect!(request)
-
-      if @running_requests >= @max_concurrency && obey_concurrency_limit
-        @queued_requests << request
-      else
-        if request.method == :get
-          if @memoize_requests && @memoized_requests.key?(request.url)
-            if response = @retrieved_from_cache[request.url]
-              request.response = response
-              request.call_handlers
-            else
-              @memoized_requests[request.url] << request
-            end
-          else
-            @memoized_requests[request.url] = [] if @memoize_requests
-            get_from_cache_or_queue(request)
-          end
-        else
-          get_from_cache_or_queue(request)
-        end
-      end
-    end
-
-    def run
-      while !@active_stubs.empty?
-        m = @active_stubs.first
-        while request = m.requests.shift
-          response = m.response
-          response.request = request
-          handle_request(request, response)
-        end
-        @active_stubs.delete(m)
-      end
-
-      @multi.perform
-    ensure
-      @multi.reset_easy_handles{|easy| release_easy_object(easy)}
-      @memoized_requests = {}
-      @retrieved_from_cache = {}
-      @running_requests = 0
-    end
-
-    def disable_memoization
-      @memoize_requests = false
-    end
-
-    def cache_getter(&block)
-      @cache_getter = block
-    end
-
-    def cache_setter(&block)
-      @cache_setter = block
-    end
-
-    def on_complete(&block)
-      @on_complete = block
-    end
-
-    def on_complete=(proc)
-      @on_complete = proc
-    end
-
-    private
-
-    def get_from_cache_or_queue(request)
-      if @cache_getter
-        val = @cache_getter.call(request)
-        if val
-          @retrieved_from_cache[request.url] = val
-          queue_next
-          handle_request(request, val, false)
-        else
-          @multi.add(get_easy_object(request))
-        end
-      else
-        @multi.add(get_easy_object(request))
-      end
-    end
-
-    def get_easy_object(request)
-      @running_requests += 1
-
-      easy = @easy_pool.pop || Easy.new
-      easy.verbose          = request.verbose
-      if request.username || request.password
-        auth = { :username => request.username, :password => request.password }
-        auth[:method] = request.auth_method if request.auth_method
-        easy.auth = auth
-      end
-
-      if request.proxy
-        proxy = { :server => request.proxy }
-        proxy[:type] = request.proxy_type if request.proxy_type
-        easy.proxy = proxy if request.proxy
-      end
-
-      if request.proxy_username || request.proxy_password
-        auth = { :username => request.proxy_username, :password => request.proxy_password }
-        auth[:method] = request.proxy_auth_method if request.proxy_auth_method
-        easy.proxy_auth = auth
-      end
-
-      easy.url          = request.url
-      easy.method       = request.method
-      easy.params       = request.params  if [:post, :put].include?(request.method) && !request.params.nil?
-      easy.headers      = request.headers if request.headers
-      easy.request_body = request.body    if [:post, :put].include?(request.method) && !request.body.nil?
-      easy.timeout      = request.timeout if request.timeout
-      easy.connect_timeout = request.connect_timeout if request.connect_timeout
-      easy.interface       = request.interface if request.interface
-      easy.follow_location = request.follow_location if request.follow_location
-      easy.max_redirects = request.max_redirects if request.max_redirects
-      easy.disable_ssl_peer_verification if request.disable_ssl_peer_verification
-      easy.disable_ssl_host_verification if request.disable_ssl_host_verification
-      easy.ssl_cert         = request.ssl_cert
-      easy.ssl_cert_type    = request.ssl_cert_type
-      easy.ssl_key          = request.ssl_key
-      easy.ssl_key_type     = request.ssl_key_type
-      easy.ssl_key_password = request.ssl_key_password
-      easy.ssl_cacert       = request.ssl_cacert
-      easy.ssl_capath       = request.ssl_capath
-      easy.ssl_version      = request.ssl_version || :default
-      easy.verbose          = request.verbose
-
-      easy.on_success do |easy|
-        queue_next
-        handle_request(request, response_from_easy(easy, request))
-        release_easy_object(easy)
-      end
-      easy.on_failure do |easy|
-        queue_next
-        handle_request(request, response_from_easy(easy, request))
-        release_easy_object(easy)
-      end
-      easy.set_headers
-      easy
-    end
-
-    def queue_next
-      @running_requests -= 1
-      queue(@queued_requests.shift) unless @queued_requests.empty?
-    end
-
-    def release_easy_object(easy)
-      easy.reset
-      @easy_pool.push easy
-    end
-
-    def handle_request(request, response, live_request = true)
-      request.response = response
-
-      self.class.run_global_hooks_for(:after_request_before_on_complete,
-                                      request)
-
-      if live_request && request.cache_timeout && @cache_setter
-        @cache_setter.call(request)
-      end
-      @on_complete.call(response) if @on_complete
-
-      request.call_handlers
-      if requests = @memoized_requests[request.url]
-        requests.each do |r|
-          r.response = response
-          r.call_handlers
-        end
-      end
-    end
-
-    def response_from_easy(easy, request)
-      Response.new(:code                => easy.response_code,
-                   :headers             => easy.response_header,
-                   :body                => easy.response_body,
-                   :time                => easy.total_time_taken,
-                   :start_transfer_time => easy.start_transfer_time,
-                   :app_connect_time    => easy.app_connect_time,
-                   :pretransfer_time    => easy.pretransfer_time,
-                   :connect_time        => easy.connect_time,
-                   :name_lookup_time    => easy.name_lookup_time,
-                   :effective_url       => easy.effective_url,
-                   :primary_ip          => easy.primary_ip,
-                   :curl_return_code => easy.curl_return_code,
-                   :curl_error_message => easy.curl_error_message,
-                   :redirect_count      => easy.redirect_count,
-                   :request             => request)
+    # @param [ Hash ] options The options hash.
+    #
+    # @option options :max_concurrency [ Integer ] Number
+    #  of max concurrent connections to create. Default is
+    #  200.
+    #
+    # @see http://rubydoc.info/github/typhoeus/ethon/Ethon/Multi#initialize-instance_method
+    #   Ethon::Multi#initialize
+    def initialize(options = {})
+      @options = options
+      @max_concurrency = @options.fetch(:max_concurrency, 200)
+      @multi = Ethon::Multi.new(options.reject{|k,_| k==:max_concurrency})
     end
   end
 end

data/lib/typhoeus/request/actions.rb

@@ -0,0 +1,125 @@
+module Typhoeus
+  class Request
+
+    # Module containing logic about shortcuts to
+    # http methods. Like
+    #   Typhoeus.get("www.example.com")
+    module Actions
+
+      # Make a get request.
+      #
+      # @example Make get request.
+      #   Typhoeus.get("www.example.com")
+      #
+      # @param (see Typhoeus::Request#initialize)
+      #
+      # @option (see Typhoeus::Request#initialize)
+      #
+      # @return (see Typhoeus::Request#initialize)
+      #
+      # @note (see Typhoeus::Request#initialize)
+      def get(url, options = {})
+        Request.run(url, options.merge(:method => :get))
+      end
+
+      # Make a post request.
+      #
+      # @example Make post request.
+      #   Typhoeus.post("www.example.com")
+      #
+      # @param (see Typhoeus::Request#initialize)
+      #
+      # @option (see Typhoeus::Request#initialize)
+      #
+      # @return (see Typhoeus::Request#initialize)
+      #
+      # @note (see Typhoeus::Request#initialize)
+      def post(url, options = {})
+        Request.run(url, options.merge(:method => :post))
+      end
+
+      # Make a put request.
+      #
+      # @example Make put request.
+      #   Typhoeus.put("www.example.com")
+      #
+      # @param (see Typhoeus::Request#initialize)
+      #
+      # @option options :params [ Hash ] Params hash which
+      #   is attached to the url.
+      # @option options :body [ Hash ] Body hash which
+      #   becomes a PUT request body.
+      #
+      # @return (see Typhoeus::Request#initialize)
+      #
+      # @note (see Typhoeus::Request#initialize)
+      def put(url, options = {})
+        Request.run(url, options.merge(:method => :put))
+      end
+
+      # Make a delete request.
+      #
+      # @example Make delete request.
+      #   Typhoeus.delete("www.example.com")
+      #
+      # @param (see Typhoeus::Request#initialize)
+      #
+      # @option (see Typhoeus::Request#initialize)
+      #
+      # @return (see Typhoeus::Request#initialize)
+      #
+      # @note (see Typhoeus::Request#initialize)
+      def delete(url, options = {})
+        Request.run(url, options.merge(:method => :delete))
+      end
+
+      # Make a head request.
+      #
+      # @example Make head request.
+      #   Typhoeus.head("www.example.com")
+      #
+      # @param (see Typhoeus::Request#initialize)
+      #
+      # @option (see Typhoeus::Request#initialize)
+      #
+      # @return (see Typhoeus::Request#initialize)
+      #
+      # @note (see Typhoeus::Request#initialize)
+      def head(url, options = {})
+        Request.run(url, options.merge(:method => :head))
+      end
+
+      # Make a patch request.
+      #
+      # @example Make patch request.
+      #   Typhoeus.patch("www.example.com")
+      #
+      # @param (see Typhoeus::Request#initialize)
+      #
+      # @option (see Typhoeus::Request#initialize)
+      #
+      # @return (see Typhoeus::Request#initialize)
+      #
+      # @note (see Typhoeus::Request#initialize)
+      def patch(url, options = {})
+        Request.run(url, options.merge(:method => :patch))
+      end
+
+      # Make a options request.
+      #
+      # @example Make options request.
+      #   Typhoeus.options("www.example.com")
+      #
+      # @param (see Typhoeus::Request#initialize)
+      #
+      # @option (see Typhoeus::Request#initialize)
+      #
+      # @return (see Typhoeus::Request#initialize)
+      #
+      # @note (see Typhoeus::Request#initialize)
+      def options(url, options = {})
+        Request.run(url, options.merge(:method => :options))
+      end
+    end
+  end
+end

data/lib/typhoeus/request/before.rb

@@ -0,0 +1,30 @@
+module Typhoeus
+  class Request
+
+    # This module provides a way to hook into before
+    # a request runs. This is very powerful
+    # and you should be careful because when you accidently
+    # return a falsy value the request won't be executed.
+    #
+    # @api private
+    module Before
+
+      # Overrride run in order to execute callbacks in
+      # Typhoeus.before. Will break and return when a
+      # callback returns nil or false. Calls super
+      # otherwise.
+      #
+      # @example Run the request.
+      #   request.run
+      def run
+        Typhoeus.before.each do |callback|
+          value = callback.call(self)
+          if value.nil? || value == false || value.is_a?(Response)
+            return value
+          end
+        end
+        super
+      end
+    end
+  end
+end

data/lib/typhoeus/request/block_connection.rb

@@ -0,0 +1,52 @@
+module Typhoeus
+  class Request
+
+    # This module handles the blocked connection request mode on
+    # the request side, where only stubbed requests
+    # are allowed.
+    # Connection blocking needs to be turned on:
+    #   Typhoeus.configure do |config|
+    #     config.block_connection = true
+    #   end
+    #
+    # When trying to do real requests a NoStub error
+    # is raised.
+    #
+    # @api private
+    module BlockConnection
+
+      # Overrides run in order to check before if block connection
+      # is turned on. If thats the case a NoStub error is
+      # raised.
+      #
+      # @example Run request.
+      #   request.run
+      #
+      # @raise [Typhoeus::Errors::NoStub] If connection is blocked
+      #   and no stub defined.
+      def run
+        if blocked?
+          raise Typhoeus::Errors::NoStub.new(self)
+        else
+          super
+        end
+      end
+
+      # Returns wether a request is blocked or not. Takes
+      # request.block_connection and Typhoeus::Config.block_connection
+      # into consideration.
+      #
+      # @example Blocked?
+      #   request.blocked?
+      #
+      # @return [ Boolean ] True if blocked, false else.
+      def blocked?
+        if block_connection.nil?
+         Typhoeus::Config.block_connection
+        else
+         block_connection
+        end
+      end
+    end
+  end
+end

data/lib/typhoeus/request/callbacks.rb

@@ -0,0 +1,98 @@
+module Typhoeus
+  class Request
+
+    # This module contains the logic for the response callbacks.
+    # The on_complete callback is the only one at the moment.
+    #
+    # You can set multiple callbacks, which are then executed
+    # in the same order.
+    #
+    #   request.on_complete { |response| p 1 }
+    #   request.on_complete { |response| p 2 }
+    #   request.execute_callbacks
+    #   #=> 1
+    #   #=> 2
+    #
+    # You can clear the callbacks:
+    #
+    #   request.on_complete { |response| p 1 }
+    #   request.on_complete { |response| p 2 }
+    #   request.on_complete.clear
+    #   request.execute_callbacks
+    #   #=> []
+    module Callbacks
+
+      module Types # :nodoc:
+        # Set on_complete callback.
+        #
+        # @example Set on_complete.
+        #   request.on_complete { |response| p "yay" }
+        #
+        # @param [ Block ] block The block to execute.
+        #
+        # @yield [ Typhoeus::Response ]
+        #
+        # @return [ Array<Block> ] All on_complete blocks.
+        def on_complete(&block)
+          @on_complete ||= []
+          @on_complete << block if block_given?
+          @on_complete
+        end
+
+        # Set on_success callback.
+        #
+        # @example Set on_success.
+        #   request.on_success { |response| p "yay" }
+        #
+        # @param [ Block ] block The block to execute.
+        #
+        # @yield [ Typhoeus::Response ]
+        #
+        # @return [ Array<Block> ] All on_success blocks.
+        def on_success(&block)
+          @on_success ||= []
+          @on_success << block if block_given?
+          @on_success
+        end
+
+        # Set on_failure callback.
+        #
+        # @example Set on_failure.
+        #   request.on_failure { |response| p "yay" }
+        #
+        # @param [ Block ] block The block to execute.
+        #
+        # @yield [ Typhoeus::Response ]
+        #
+        # @return [ Array<Block> ] All on_failure blocks.
+        def on_failure(&block)
+          @on_failure ||= []
+          @on_failure << block if block_given?
+          @on_failure
+        end
+      end
+
+      # Execute nessecary callback and yields response. This
+      # include in every case on_complete, on_success if
+      # successful and on_failure if not.
+      #
+      # @example Execute callbacks.
+      #   request.execute_callbacks
+      #
+      # @return [ void ]
+      #
+      # @api private
+      def execute_callbacks
+        callbacks = Typhoeus.on_complete + on_complete
+
+        if response && response.success?
+          callbacks += Typhoeus.on_success + on_success
+        elsif response
+          callbacks += Typhoeus.on_failure + on_failure
+        end
+
+        callbacks.map{ |callback| callback.call(self.response) }
+      end
+    end
+  end
+end

data/lib/typhoeus/request/marshal.rb

@@ -0,0 +1,21 @@
+module Typhoeus
+  class Request
+
+    # This module contains custom serializer.
+    module Marshal
+
+      # Return the important data needed to serialize this Request, except the
+      # `on_complete` handler, since they cannot be marshalled.
+      def marshal_dump
+        (instance_variables - ['@on_complete', :@on_complete]).map do |name|
+          [name, instance_variable_get(name)]
+        end
+      end
+
+      # Load.
+      def marshal_load(attributes)
+        attributes.each { |name, value| instance_variable_set(name, value) }
+      end
+    end
+  end
+end

data/lib/typhoeus/request/memoizable.rb

@@ -0,0 +1,38 @@
+module Typhoeus
+  class Request
+
+    # This module handles the GET request memoization
+    # on the request side. Memoization needs to be turned
+    # on:
+    #   Typhoeus.configure do |config|
+    #     config.memoize = true
+    #   end
+    #
+    # @api private
+    module Memoizable
+
+      # Override response setter and memoizes response
+      # if the request is memoizable.
+      #
+      # @param [ Response ] response The response to set.
+      #
+      # @example Set response.
+      #   request.response = response
+      def response=(response)
+        hydra.memory[self] = response if memoizable?
+        super
+      end
+
+      # Return whether a request is memoizable.
+      #
+      # @example Is request memoizable?
+      #   request.memoizable?
+      #
+      # @return [ Boolean ] Return true if memoizable, false else.
+      def memoizable?
+        Typhoeus::Config.memoize &&
+          (options[:method].nil? || options[:method] == :get)
+      end
+    end
+  end
+end