typhoeus 0.1.6 → 0.2.0

data/lib/typhoeus/hydra.rb

@@ -1,63 +1,91 @@
+require 'typhoeus/hydra/callbacks'
+require 'typhoeus/hydra/connect_options'
+require 'typhoeus/hydra/stubbing'
+
 module Typhoeus
   class Hydra
-    def initialize(initial_pool_size = 10)
+    include ConnectOptions
+    include Stubbing
+    extend Callbacks
+
+    def initialize(options = {})
       @memoize_requests = true
       @multi       = Multi.new
       @easy_pool   = []
+      initial_pool_size = options[:initial_pool_size] || 10
+      @max_concurrency = options[:max_concurrency] || 200
       initial_pool_size.times { @easy_pool << Easy.new }
-      @stubs       = []
       @memoized_requests = {}
       @retrieved_from_cache = {}
+      @queued_requests = []
+      @running_requests = 0
+
+      self.stubs = []
+      @active_stubs = []
     end
 
     def self.hydra
       @hydra ||= new
     end
-    
+
     def self.hydra=(val)
       @hydra = val
     end
-    
-    def clear_stubs
-      @stubs = []
+
+    def clear_cache_callbacks
+      @cache_setter = nil
+      @cache_getter = nil
     end
-    
+
     def fire_and_forget
+      @queued_requests.each {|r| queue(r, false)}
       @multi.fire_and_forget
     end
 
-    def queue(request)
+    def queue(request, obey_concurrency_limit = true)
       return if assign_to_stub(request)
 
-      if request.method == :get
-        if @memoize_requests && @memoized_requests.has_key?(request.url)
-          if response = @retrieved_from_cache[request.url]
-            request.response = response
-            request.call_handlers
+      # At this point, we are running over live HTTP. Make sure we haven't
+      # disabled live requests.
+      check_allow_net_connect!(request)
+
+      if @running_requests >= @max_concurrency && obey_concurrency_limit
+        @queued_requests << request
+      else
+        if request.method == :get
+          if @memoize_requests && @memoized_requests.has_key?(request.url)
+            if response = @retrieved_from_cache[request.url]
+              request.response = response
+              request.call_handlers
+            else
+              @memoized_requests[request.url] << request
+            end
           else
-            @memoized_requests[request.url] << request
+            @memoized_requests[request.url] = [] if @memoize_requests
+            get_from_cache_or_queue(request)
           end
         else
-          @memoized_requests[request.url] = [] if @memoize_requests
           get_from_cache_or_queue(request)
         end
-      else
-        get_from_cache_or_queue(request)
       end
     end
 
     def run
-      @stubs.each do |m|
-        m.requests.each do |request|
-          m.response.request = request
-          handle_request(request, m.response)
+      while !@active_stubs.empty?
+        m = @active_stubs.first
+        while request = m.requests.shift
+          response = m.response
+          response.request = request
+          handle_request(request, response)
         end
+        @active_stubs.delete(m)
       end
+
       @multi.perform
       @memoized_requests = {}
       @retrieved_from_cache = {}
     end
-    
+
     def disable_memoization
       @memoize_requests = false
     end
@@ -78,17 +106,6 @@ module Typhoeus
       @on_complete = proc
     end
 
-    def stub(method, url)
-      @stubs << HydraMock.new(url, method)
-      @stubs.last
-    end
-
-    def assign_to_stub(request)
-      m = @stubs.detect {|stub| stub.matches? request}
-      m && m.add_request(request)
-    end
-    private :assign_to_stub
-
     def get_from_cache_or_queue(request)
       if @cache_getter
         val = @cache_getter.call(request)
@@ -105,17 +122,42 @@ module Typhoeus
     private :get_from_cache_or_queue
 
     def get_easy_object(request)
+      @running_requests += 1
+
       easy = @easy_pool.pop || Easy.new
+      easy.verbose          = request.verbose
+      if request.username || request.password
+        auth = { :username => request.username, :password => request.password }
+        auth[:method] = Typhoeus::Easy::AUTH_TYPES["CURLAUTH_#{request.auth_method.to_s.upcase}".to_sym] if request.auth_method
+        easy.auth = auth
+      end
       easy.url          = request.url
       easy.method       = request.method
+      easy.params       = request.params  if request.method == :post && !request.params.nil?
       easy.headers      = request.headers if request.headers
       easy.request_body = request.body    if request.body
       easy.timeout      = request.timeout if request.timeout
+      easy.connect_timeout = request.connect_timeout if request.connect_timeout
+      easy.follow_location = request.follow_location if request.follow_location
+      easy.max_redirects = request.max_redirects if request.max_redirects
+      easy.proxy = request.proxy if request.proxy
+      easy.disable_ssl_peer_verification if request.disable_ssl_peer_verification
+      easy.ssl_cert         = request.ssl_cert
+      easy.ssl_cert_type    = request.ssl_cert_type
+      easy.ssl_key          = request.ssl_key
+      easy.ssl_key_type     = request.ssl_key_type
+      easy.ssl_key_password = request.ssl_key_password
+      easy.ssl_cacert       = request.ssl_cacert
+      easy.ssl_capath       = request.ssl_capath
+      easy.verbose          = request.verbose
+
       easy.on_success do |easy|
+        queue_next
         handle_request(request, response_from_easy(easy, request))
         release_easy_object(easy)
       end
       easy.on_failure do |easy|
+        queue_next
         handle_request(request, response_from_easy(easy, request))
         release_easy_object(easy)
       end
@@ -124,6 +166,12 @@ module Typhoeus
     end
     private :get_easy_object
 
+    def queue_next
+      @running_requests -= 1
+      queue(@queued_requests.pop) unless @queued_requests.empty?
+    end
+    private :queue_next
+
     def release_easy_object(easy)
       easy.reset
       @easy_pool.push easy
@@ -133,6 +181,9 @@ module Typhoeus
     def handle_request(request, response, live_request = true)
       request.response = response
 
+      self.class.run_global_hooks_for(:after_request_before_on_complete,
+                                      request)
+
       if live_request && request.cache_timeout && @cache_setter
         @cache_setter.call(request)
       end
@@ -153,34 +204,9 @@ module Typhoeus
                    :headers => easy.response_header,
                    :body    => easy.response_body,
                    :time    => easy.total_time_taken,
+                   :effective_url => easy.effective_url,
                    :request => request)
     end
     private :response_from_easy
   end
-
-  class HydraMock
-    attr_reader :url, :method, :response, :requests
-
-    def initialize(url, method)
-      @url      = url
-      @method   = method
-      @requests = []
-    end
-
-    def add_request(request)
-      @requests << request
-    end
-
-    def and_return(val)
-      @response = val
-    end
-    
-    def matches?(request)
-      if url.kind_of?(String)
-        request.method == method && request.url == url
-      else
-        request.method == method && url =~ request.url
-      end
-    end
-  end
 end

data/lib/typhoeus/hydra_mock.rb

@@ -0,0 +1,131 @@
+module Typhoeus
+  class HydraMock
+    attr_reader :url, :method, :requests, :uri
+
+    def initialize(url, method, options = {})
+      @url      = url
+      @uri      = URI.parse(url) if url.kind_of?(String)
+      @method   = method
+      @requests = []
+      @options = options
+      if @options[:headers]
+        @options[:headers] = Typhoeus::NormalizedHeaderHash.new(@options[:headers])
+      end
+
+      @current_response_index = 0
+    end
+
+    def body
+      @options[:body]
+    end
+
+    def body?
+      @options.has_key?(:body)
+    end
+
+    def headers
+      @options[:headers]
+    end
+
+    def headers?
+      @options.has_key?(:headers)
+    end
+
+    def add_request(request)
+      @requests << request
+    end
+
+    def and_return(val)
+      if val.respond_to?(:each)
+        @responses = val
+      else
+        @responses = [val]
+      end
+
+      # make sure to mark them as a mock.
+      @responses.each { |r| r.mock = true }
+
+      val
+    end
+
+    def response
+      if @current_response_index == (@responses.length - 1)
+        @responses.last
+      else
+        value = @responses[@current_response_index]
+        @current_response_index += 1
+        value
+      end
+    end
+
+    def matches?(request)
+      if !method_matches?(request) or !url_matches?(request)
+        return false
+      end
+
+      if body?
+        return false unless body_matches?(request)
+      end
+
+      if headers?
+        return false unless headers_match?(request)
+      end
+
+      true
+    end
+
+  private
+    def method_matches?(request)
+      self.method == :any or self.method == request.method
+    end
+
+    def url_matches?(request)
+      if url.kind_of?(String)
+        request_uri = URI.parse(request.url)
+        request_uri == self.uri
+      else
+        self.url =~ request.url
+      end
+    end
+
+    def body_matches?(request)
+      !request.body.nil? && !request.body.empty? && request.body == self.body
+    end
+
+    def headers_match?(request)
+      request_headers = NormalizedHeaderHash.new(request.headers)
+
+      if empty_headers?(self.headers)
+        empty_headers?(request_headers)
+      else
+        return false if empty_headers?(request_headers)
+
+        headers.each do |key, value|
+          return false unless header_value_matches?(value, request_headers[key])
+        end
+
+        true
+      end
+    end
+
+    def header_value_matches?(mock_value, request_value)
+      mock_arr = mock_value.is_a?(Array) ? mock_value : [mock_value]
+      request_arr = request_value.is_a?(Array) ? request_value : [request_value]
+
+      return false unless mock_arr.size == request_arr.size
+      mock_arr.all? do |value|
+        request_arr.any? { |a| value === a }
+      end
+    end
+
+    def empty_headers?(headers)
+      # We consider the default User-Agent header to be empty since
+      # Typhoeus always adds that.
+      headers.nil? || headers.empty? || default_typhoeus_headers?(headers)
+    end
+
+    def default_typhoeus_headers?(headers)
+      headers.size == 1 && headers['User-Agent'] == Typhoeus::USER_AGENT
+    end
+  end
+end

data/lib/typhoeus/multi.rb

@@ -9,19 +9,20 @@ module Typhoeus
     def remove(easy)
       multi_remove_handle(easy)
     end
-    
+
     def add(easy)
+      easy.set_headers() if easy.headers.empty?
       @easy_handles << easy
       multi_add_handle(easy)
     end
-    
+
     def perform()
       while active_handle_count > 0 do
         multi_perform
       end
       reset_easy_handles
     end
-    
+
     def cleanup()
       multi_cleanup
     end

data/lib/typhoeus/normalized_header_hash.rb

@@ -0,0 +1,58 @@
+module Typhoeus
+  class NormalizedHeaderHash < ::Hash
+    def initialize(constructor = {})
+      if constructor.is_a?(Hash)
+        super
+        update(constructor)
+      else
+        super(constructor)
+      end
+    end
+
+    def fetch(key, *extras)
+      super(convert_key(key), *extras)
+    end
+
+    def key?(key)
+      super(convert_key(key))
+    end
+
+    [:include?, :has_key?, :member?].each do |method|
+      alias_method method, :key?
+    end
+
+    def [](key)
+      super(convert_key(key))
+    end
+
+    def []=(key, value)
+      super(convert_key(key), value)
+    end
+
+    def update(other_hash)
+      other_hash.each_pair do |key, value|
+        self[convert_key(key)] = value
+      end
+      self
+    end
+
+    alias_method :merge!, :update
+
+    def dup
+      self.class.new(self)
+    end
+
+    def merge(hash)
+      self.dup.update(hash)
+    end
+
+    def delete(key)
+      super(convert_key(key))
+    end
+
+  private
+    def convert_key(key)
+      key.to_s.split(/_|-/).map { |segment| segment.capitalize }.join("-")
+    end
+  end
+end

data/lib/typhoeus/remote_proxy_object.rb

@@ -1,7 +1,7 @@
 module Typhoeus
   class RemoteProxyObject
-    instance_methods.each { |m| undef_method m unless m =~ /^__/ }
-    
+    instance_methods.each { |m| undef_method m unless m =~ /^__|object_id/ }
+
     def initialize(clear_memoized_store_proc, easy, options = {})
       @clear_memoized_store_proc = clear_memoized_store_proc
       @easy      = easy
@@ -12,13 +12,13 @@ module Typhoeus
       @timeout   = options.delete(:cache_timeout)
       Typhoeus.add_easy_request(@easy)
     end
-    
+
     def method_missing(sym, *args, &block)
       unless @proxied_object
         if @cache && @cache_key
           @proxied_object = @cache.get(@cache_key) rescue nil
         end
-        
+
         unless @proxied_object
           Typhoeus.perform_easy_requests
           response = Response.new(:code => @easy.response_code,
@@ -31,7 +31,7 @@ module Typhoeus
           if @easy.response_code >= 200 && @easy.response_code < 300
             Typhoeus.release_easy_object(@easy)
             @proxied_object = @success.nil? ? response : @success.call(response)
-            
+
             if @cache && @cache_key
               @cache.set(@cache_key, @proxied_object, @timeout)
             end
@@ -41,7 +41,7 @@ module Typhoeus
          @clear_memoized_store_proc.call
        end
       end
-      
+
       @proxied_object.__send__(sym, *args, &block)
     end
   end