typhoeus 0.4.2 → 1.4.0

data/lib/typhoeus/hydra/stubbable.rb

@@ -0,0 +1,28 @@
+module Typhoeus
+  class Hydra
+
+    # This module handles stubbing on the hydra side.
+    # It plays well with the block_connection configuration,
+    # which raises when you make a request which is not stubbed.
+    #
+    # @api private
+    module Stubbable
+
+      # Override add in order to check for matching expecations.
+      # When an expecation is found, super is not called. Instead a
+      # canned response is assigned to the request.
+      #
+      # @example Add the request.
+      #   hydra.add(request)
+      def add(request)
+        if response = Expectation.response_for(request)
+          request.execute_headers_callbacks(response)
+          request.on_body.each{ |callback| callback.call(response.body, response) }
+          request.finish(response)
+        else
+          super
+        end
+      end
+    end
+  end
+end

data/lib/typhoeus/hydra.rb

@@ -1,247 +1,95 @@
-require 'typhoeus/hydra/callbacks'
-require 'typhoeus/hydra/connect_options'
-require 'typhoeus/hydra/stubbing'
+require 'typhoeus/hydra/addable'
+require 'typhoeus/hydra/before'
+require 'typhoeus/hydra/cacheable'
+require 'typhoeus/hydra/block_connection'
+require 'typhoeus/hydra/memoizable'
+require 'typhoeus/hydra/queueable'
+require 'typhoeus/hydra/runnable'
+require 'typhoeus/hydra/stubbable'
 
 module Typhoeus
-  class Hydra
-    include ConnectOptions
-    include Stubbing
-    extend Callbacks
-
-    def initialize(options = {})
-      @memoize_requests = true
-      @multi       = Multi.new
-      @easy_pool   = []
-      initial_pool_size = options[:initial_pool_size] || 10
-      @max_concurrency = options[:max_concurrency] || 200
-      initial_pool_size.times { @easy_pool << Easy.new }
-      @memoized_requests = {}
-      @retrieved_from_cache = {}
-      @queued_requests = []
-      @running_requests = 0
-
-      self.stubs = []
-      @active_stubs = []
-    end
-
-    def self.hydra
-      @hydra ||= new
-    end
 
-    def self.hydra=(val)
-      @hydra = val
+  # Hydra manages making parallel HTTP requests. This
+  # is achieved by using libcurls multi interface:
+  # http://curl.haxx.se/libcurl/c/libcurl-multi.html
+  # The benefits are that you don't have to worry running
+  # the requests by yourself.
+  #
+  # Hydra will also handle how many requests you can
+  # make in parallel. Things will get flakey if you
+  # try to make too many requests at the same time.
+  # The built in limit is 200. When more requests than
+  # that are queued up, hydra will save them for later
+  # and start the requests as others are finished. You
+  # can raise or lower the concurrency limit through
+  # the Hydra constructor.
+  #
+  # Regarding the asynchronous behavior of the hydra,
+  # it is important to know that this is completely hidden
+  # from the developer and you are free to apply
+  # whatever technique you want to your code. That should not
+  # conflict with libcurls internal concurrency mechanism.
+  #
+  # @example Use the hydra to do multiple requests.
+  #   hydra = Typhoeus::Hydra.new
+  #   requests = (0..9).map{ Typhoeus::Request.new("www.example.com") }
+  #   requests.each{ |request| hydra.queue(request) }
+  #   hydra.run
+  #
+  # @note Callbacks are going to delay the request
+  #   execution.
+  class Hydra
+    include Hydra::Addable
+    include Hydra::Runnable
+    include Hydra::Memoizable
+    include Hydra::Cacheable
+    include Hydra::BlockConnection
+    include Hydra::Stubbable
+    include Hydra::Before
+    include Hydra::Queueable
+
+    # @example Set max_concurrency.
+    #   Typhoeus::Hydra.new(max_concurrency: 20)
+    attr_accessor :max_concurrency
+
+    # @api private
+    attr_reader :multi
+
+    class << self
+
+      # Returns a memoized hydra instance.
+      #
+      # @example Get a hydra.
+      #   Typhoeus::Hydra.hydra
+      #
+      # @return [Typhoeus::Hydra] A new hydra.
+      def hydra
+        Thread.current[:typhoeus_hydra] ||= new
+      end
     end
 
+    # Create a new hydra. All
+    # {http://rubydoc.info/github/typhoeus/ethon/Ethon/Multi#initialize-instance_method Ethon::Multi#initialize}
+    # options are also available.
     #
-    # Abort the run on a best-effort basis.
+    # @example Create a hydra.
+    #   Typhoeus::Hydra.new
     #
-    # It won't abort the current burst of @max_concurrency requests,
-    # however it won't fire the rest of the queued requests so the run
-    # will be aborted as soon as possible...
+    # @example Create a hydra with max_concurrency.
+    #   Typhoeus::Hydra.new(max_concurrency: 20)
     #
-    def abort
-      @queued_requests.clear
-    end
-
-    def clear_cache_callbacks
-      @cache_setter = nil
-      @cache_getter = nil
-    end
-
-    def fire_and_forget
-      @queued_requests.each {|r| queue(r, false)}
-      @multi.fire_and_forget
-    end
-
-    def queue(request, obey_concurrency_limit = true)
-      return if assign_to_stub(request)
-
-      # At this point, we are running over live HTTP. Make sure we haven't
-      # disabled live requests.
-      check_allow_net_connect!(request)
-
-      if @running_requests >= @max_concurrency && obey_concurrency_limit
-        @queued_requests << request
-      else
-        if request.method == :get
-          if @memoize_requests && @memoized_requests.key?(request.url)
-            if response = @retrieved_from_cache[request.url]
-              request.response = response
-              request.call_handlers
-            else
-              @memoized_requests[request.url] << request
-            end
-          else
-            @memoized_requests[request.url] = [] if @memoize_requests
-            get_from_cache_or_queue(request)
-          end
-        else
-          get_from_cache_or_queue(request)
-        end
-      end
-    end
-
-    def run
-      while !@active_stubs.empty?
-        m = @active_stubs.first
-        while request = m.requests.shift
-          response = m.response
-          response.request = request
-          handle_request(request, response)
-        end
-        @active_stubs.delete(m)
-      end
-
-      @multi.perform
-    ensure
-      @multi.reset_easy_handles{|easy| release_easy_object(easy)}
-      @memoized_requests = {}
-      @retrieved_from_cache = {}
-      @running_requests = 0
-    end
-
-    def disable_memoization
-      @memoize_requests = false
-    end
-
-    def cache_getter(&block)
-      @cache_getter = block
-    end
-
-    def cache_setter(&block)
-      @cache_setter = block
-    end
-
-    def on_complete(&block)
-      @on_complete = block
-    end
-
-    def on_complete=(proc)
-      @on_complete = proc
-    end
-
-    private
-
-    def get_from_cache_or_queue(request)
-      if @cache_getter
-        val = @cache_getter.call(request)
-        if val
-          @retrieved_from_cache[request.url] = val
-          queue_next
-          handle_request(request, val, false)
-        else
-          @multi.add(get_easy_object(request))
-        end
-      else
-        @multi.add(get_easy_object(request))
-      end
-    end
-
-    def get_easy_object(request)
-      @running_requests += 1
-
-      easy = @easy_pool.pop || Easy.new
-      easy.verbose          = request.verbose
-      if request.username || request.password
-        auth = { :username => request.username, :password => request.password }
-        auth[:method] = request.auth_method if request.auth_method
-        easy.auth = auth
-      end
-
-      if request.proxy
-        proxy = { :server => request.proxy }
-        proxy[:type] = request.proxy_type if request.proxy_type
-        easy.proxy = proxy if request.proxy
-      end
-
-      if request.proxy_username || request.proxy_password
-        auth = { :username => request.proxy_username, :password => request.proxy_password }
-        auth[:method] = request.proxy_auth_method if request.proxy_auth_method
-        easy.proxy_auth = auth
-      end
-
-      easy.url          = request.url
-      easy.method       = request.method
-      easy.params       = request.params  if [:post, :put].include?(request.method) && !request.params.nil?
-      easy.headers      = request.headers if request.headers
-      easy.request_body = request.body    if [:post, :put].include?(request.method) && !request.body.nil?
-      easy.timeout      = request.timeout if request.timeout
-      easy.connect_timeout = request.connect_timeout if request.connect_timeout
-      easy.interface       = request.interface if request.interface
-      easy.follow_location = request.follow_location if request.follow_location
-      easy.max_redirects = request.max_redirects if request.max_redirects
-      easy.disable_ssl_peer_verification if request.disable_ssl_peer_verification
-      easy.disable_ssl_host_verification if request.disable_ssl_host_verification
-      easy.ssl_cert         = request.ssl_cert
-      easy.ssl_cert_type    = request.ssl_cert_type
-      easy.ssl_key          = request.ssl_key
-      easy.ssl_key_type     = request.ssl_key_type
-      easy.ssl_key_password = request.ssl_key_password
-      easy.ssl_cacert       = request.ssl_cacert
-      easy.ssl_capath       = request.ssl_capath
-      easy.ssl_version      = request.ssl_version || :default
-      easy.verbose          = request.verbose
-
-      easy.on_success do |easy|
-        queue_next
-        handle_request(request, response_from_easy(easy, request))
-        release_easy_object(easy)
-      end
-      easy.on_failure do |easy|
-        queue_next
-        handle_request(request, response_from_easy(easy, request))
-        release_easy_object(easy)
-      end
-      easy.set_headers
-      easy
-    end
-
-    def queue_next
-      @running_requests -= 1
-      queue(@queued_requests.shift) unless @queued_requests.empty?
-    end
-
-    def release_easy_object(easy)
-      easy.reset
-      @easy_pool.push easy
-    end
-
-    def handle_request(request, response, live_request = true)
-      request.response = response
-
-      self.class.run_global_hooks_for(:after_request_before_on_complete,
-                                      request)
-
-      if live_request && request.cache_timeout && @cache_setter
-        @cache_setter.call(request)
-      end
-      @on_complete.call(response) if @on_complete
-
-      request.call_handlers
-      if requests = @memoized_requests[request.url]
-        requests.each do |r|
-          r.response = response
-          r.call_handlers
-        end
-      end
-    end
-
-    def response_from_easy(easy, request)
-      Response.new(:code                => easy.response_code,
-                   :headers             => easy.response_header,
-                   :body                => easy.response_body,
-                   :time                => easy.total_time_taken,
-                   :start_transfer_time => easy.start_transfer_time,
-                   :app_connect_time    => easy.app_connect_time,
-                   :pretransfer_time    => easy.pretransfer_time,
-                   :connect_time        => easy.connect_time,
-                   :name_lookup_time    => easy.name_lookup_time,
-                   :effective_url       => easy.effective_url,
-                   :primary_ip          => easy.primary_ip,
-                   :curl_return_code => easy.curl_return_code,
-                   :curl_error_message => easy.curl_error_message,
-                   :redirect_count      => easy.redirect_count,
-                   :request             => request)
+    # @param [ Hash ] options The options hash.
+    #
+    # @option options :max_concurrency [ Integer ] Number
+    #  of max concurrent connections to create. Default is
+    #  200.
+    #
+    # @see http://rubydoc.info/github/typhoeus/ethon/Ethon/Multi#initialize-instance_method
+    #   Ethon::Multi#initialize
+    def initialize(options = {})
+      @options = options
+      @max_concurrency = Integer(@options.fetch(:max_concurrency, 200))
+      @multi = Ethon::Multi.new(options.reject{|k,_| k==:max_concurrency})
     end
   end
 end

data/lib/typhoeus/pool.rb

@@ -0,0 +1,70 @@
+require 'thread'
+
+module Typhoeus
+
+  # The easy pool stores already initialized
+  # easy handles for future use. This is useful
+  # because creating them is expensive.
+  #
+  # @api private
+  module Pool
+    @mutex = Mutex.new
+    @pid = Process.pid
+
+    # Releases easy into the pool. The easy handle is
+    # reset before it gets back in.
+    #
+    # @example Release easy.
+    #   Typhoeus::Pool.release(easy)
+    def self.release(easy)
+      easy.cookielist = "flush" # dump all known cookies to 'cookiejar'
+      easy.cookielist = "all" # remove all cookies from memory for this handle
+      easy.reset
+      @mutex.synchronize { easies << easy }
+    end
+
+    # Return an easy from the pool.
+    #
+    # @example Return easy.
+    #   Typhoeus::Pool.get
+    #
+    # @return [ Ethon::Easy ] The easy.
+    def self.get
+      @mutex.synchronize do
+        if @pid == Process.pid
+          easies.pop
+        else
+          # Process has forked. Clear all easies to avoid sockets being
+          # shared between processes.
+          @pid = Process.pid
+          easies.clear
+          nil
+        end
+      end || Ethon::Easy.new
+    end
+
+    # Clear the pool
+    def self.clear
+      @mutex.synchronize { easies.clear }
+    end
+
+    # Use yielded easy, will be released automatically afterwards.
+    #
+    # @example Use easy.
+    #   Typhoeus::Pool.with_easy do |easy|
+    #     # use easy
+    #   end
+    def self.with_easy(&block)
+      easy = get
+      yield easy
+    ensure
+      release(easy) if easy
+    end
+
+    private
+
+    def self.easies
+      @easies ||= []
+    end
+  end
+end

data/lib/typhoeus/railtie.rb

@@ -0,0 +1,12 @@
+require "typhoeus"
+
+module Rails
+  module Typhoeus
+    class Railtie < Rails::Railtie
+      # Need to include the Typhoeus middleware.
+      initializer "include the identity map" do |app|
+        app.config.middleware.use "Rack::Typhoeus::Middleware::ParamsDecoder"
+      end
+    end
+  end
+end

data/lib/typhoeus/request/actions.rb

@@ -0,0 +1,125 @@
+module Typhoeus
+  class Request
+
+    # Module containing logic about shortcuts to
+    # http methods. Like
+    #   Typhoeus.get("www.example.com")
+    module Actions
+
+      # Make a get request.
+      #
+      # @example Make get request.
+      #   Typhoeus.get("www.example.com")
+      #
+      # @param (see Typhoeus::Request#initialize)
+      #
+      # @option (see Typhoeus::Request#initialize)
+      #
+      # @return (see Typhoeus::Response#initialize)
+      #
+      # @note (see Typhoeus::Request#initialize)
+      def get(base_url, options = {})
+        Request.new(base_url, options.merge(:method => :get)).run
+      end
+
+      # Make a post request.
+      #
+      # @example Make post request.
+      #   Typhoeus.post("www.example.com")
+      #
+      # @param (see Typhoeus::Request#initialize)
+      #
+      # @option (see Typhoeus::Request#initialize)
+      #
+      # @return (see Typhoeus::Response#initialize)
+      #
+      # @note (see Typhoeus::Request#initialize)
+      def post(base_url, options = {})
+        Request.new(base_url, options.merge(:method => :post)).run
+      end
+
+      # Make a put request.
+      #
+      # @example Make put request.
+      #   Typhoeus.put("www.example.com")
+      #
+      # @param (see Typhoeus::Request#initialize)
+      #
+      # @option options :params [ Hash ] Params hash which
+      #   is attached to the base_url.
+      # @option options :body [ Hash ] Body hash which
+      #   becomes a PUT request body.
+      #
+      # @return (see Typhoeus::Response#initialize)
+      #
+      # @note (see Typhoeus::Request#initialize)
+      def put(base_url, options = {})
+        Request.new(base_url, options.merge(:method => :put)).run
+      end
+
+      # Make a delete request.
+      #
+      # @example Make delete request.
+      #   Typhoeus.delete("www.example.com")
+      #
+      # @param (see Typhoeus::Request#initialize)
+      #
+      # @option (see Typhoeus::Request#initialize)
+      #
+      # @return (see Typhoeus::Response#initialize)
+      #
+      # @note (see Typhoeus::Request#initialize)
+      def delete(base_url, options = {})
+        Request.new(base_url, options.merge(:method => :delete)).run
+      end
+
+      # Make a head request.
+      #
+      # @example Make head request.
+      #   Typhoeus.head("www.example.com")
+      #
+      # @param (see Typhoeus::Request#initialize)
+      #
+      # @option (see Typhoeus::Request#initialize)
+      #
+      # @return (see Typhoeus::Response#initialize)
+      #
+      # @note (see Typhoeus::Request#initialize)
+      def head(base_url, options = {})
+        Request.new(base_url, options.merge(:method => :head)).run
+      end
+
+      # Make a patch request.
+      #
+      # @example Make patch request.
+      #   Typhoeus.patch("www.example.com")
+      #
+      # @param (see Typhoeus::Request#initialize)
+      #
+      # @option (see Typhoeus::Request#initialize)
+      #
+      # @return (see Typhoeus::Response#initialize)
+      #
+      # @note (see Typhoeus::Request#initialize)
+      def patch(base_url, options = {})
+        Request.new(base_url, options.merge(:method => :patch)).run
+      end
+
+      # Make a options request.
+      #
+      # @example Make options request.
+      #   Typhoeus.options("www.example.com")
+      #
+      # @param (see Typhoeus::Request#initialize)
+      #
+      # @option (see Typhoeus::Request#initialize)
+      #
+      # @return (see Typhoeus::Response#initialize)
+      #
+      # @note (see Typhoeus::Request#initialize)
+      def options(base_url, options = {})
+        Request.new(base_url, options.merge(:method => :options)).run
+      end
+    end
+  end
+end

data/lib/typhoeus/request/before.rb

@@ -0,0 +1,30 @@
+module Typhoeus
+  class Request
+
+    # This module provides a way to hook into before
+    # a request runs. This is very powerful
+    # and you should be careful because when you accidently
+    # return a falsy value the request won't be executed.
+    #
+    # @api private
+    module Before
+
+      # Overrride run in order to execute callbacks in
+      # Typhoeus.before. Will break and return when a
+      # callback returns nil or false. Calls super
+      # otherwise.
+      #
+      # @example Run the request.
+      #   request.run
+      def run
+        Typhoeus.before.each do |callback|
+          value = callback.call(self)
+          if value.nil? || value == false || value.is_a?(Response)
+            return response
+          end
+        end
+        super
+      end
+    end
+  end
+end

data/lib/typhoeus/request/block_connection.rb

@@ -0,0 +1,52 @@
+module Typhoeus
+  class Request
+
+    # This module handles the blocked connection request mode on
+    # the request side, where only stubbed requests
+    # are allowed.
+    # Connection blocking needs to be turned on:
+    #   Typhoeus.configure do |config|
+    #     config.block_connection = true
+    #   end
+    #
+    # When trying to do real requests a NoStub error
+    # is raised.
+    #
+    # @api private
+    module BlockConnection
+
+      # Overrides run in order to check before if block connection
+      # is turned on. If thats the case a NoStub error is
+      # raised.
+      #
+      # @example Run request.
+      #   request.run
+      #
+      # @raise [Typhoeus::Errors::NoStub] If connection is blocked
+      #   and no stub defined.
+      def run
+        if blocked?
+          raise Typhoeus::Errors::NoStub.new(self)
+        else
+          super
+        end
+      end
+
+      # Returns wether a request is blocked or not. Takes
+      # request.block_connection and Typhoeus::Config.block_connection
+      # into consideration.
+      #
+      # @example Blocked?
+      #   request.blocked?
+      #
+      # @return [ Boolean ] True if blocked, false else.
+      def blocked?
+        if block_connection.nil?
+          Typhoeus::Config.block_connection
+        else
+          block_connection
+        end
+      end
+    end
+  end
+end

data/lib/typhoeus/request/cacheable.rb

@@ -0,0 +1,38 @@
+module Typhoeus
+  class Request
+    module Cacheable
+      def response=(response)
+        cache.set(self, response) if cacheable? && !response.cached?
+        super
+      end
+
+      def cacheable?
+        cache
+      end
+
+      def run
+        if response = cached_response
+          response.cached = true
+          finish(response)
+        else
+          super
+        end
+      end
+
+      def cached_response
+        cacheable? && cache.get(self)
+      end
+
+      def cache_ttl
+        options[:cache_ttl]
+      end
+
+      private
+
+      def cache
+        return nil if options[:cache] === false
+        options[:cache] || Typhoeus::Config.cache
+      end
+    end
+  end
+end