typhoeus 0.1.6 → 0.1.28

data/lib/typhoeus/easy.rb

@@ -1,8 +1,9 @@
 module Typhoeus
   class Easy
-    attr_reader :response_body, :response_header, :method, :headers, :url
+    attr_reader :response_body, :response_header, :method, :headers, :url, :params
     attr_accessor :start_time
 
+    # These integer codes are available in curl/curl.h
     CURLINFO_STRING = 1048576
     OPTION_VALUES = {
       :CURLOPT_URL            => 10002,
@@ -16,31 +17,78 @@ module Typhoeus
       :CURLOPT_TIMEOUT_MS     => 155,
       :CURLOPT_NOSIGNAL       => 99,
       :CURLOPT_HTTPHEADER     => 10023,
-      :CURLOPT_FOLLOWLOCATION => 52
+      :CURLOPT_FOLLOWLOCATION => 52,
+      :CURLOPT_MAXREDIRS      => 68,
+      :CURLOPT_HTTPAUTH       => 107,
+      :CURLOPT_USERPWD        => 10000 + 5,
+      :CURLOPT_VERBOSE        => 41,
+      :CURLOPT_PROXY          => 10004,
+      :CURLOPT_VERIFYPEER     => 64,
+      :CURLOPT_NOBODY         => 44,
+      :CURLOPT_ENCODING       => 102,
+      :CURLOPT_SSLCERT        => 10025,
+      :CURLOPT_SSLCERTTYPE    => 10086,
+      :CURLOPT_SSLKEY         => 10087,
+      :CURLOPT_SSLKEYTYPE     => 10088,
+      :CURLOPT_KEYPASSWD      => 10026,
+      :CURLOPT_CAINFO         => 10065,
+      :CURLOPT_CAPATH         => 10097
     }
     INFO_VALUES = {
       :CURLINFO_RESPONSE_CODE => 2097154,
-      :CURLINFO_TOTAL_TIME    => 3145731
+      :CURLINFO_TOTAL_TIME    => 3145731,
+      :CURLINFO_HTTPAUTH_AVAIL => 0x200000 + 23,
+      :CURLINFO_EFFECTIVE_URL => 0x100000 + 1
     }
-    
+    AUTH_TYPES = {
+      :CURLAUTH_BASIC         => 1,
+      :CURLAUTH_DIGEST        => 2,
+      :CURLAUTH_GSSNEGOTIATE  => 4,
+      :CURLAUTH_NTLM          => 8,
+      :CURLAUTH_DIGEST_IE     => 16
+    }
+
     def initialize
       @method = :get
       @post_dat_set = nil
       @headers = {}
+
+      set_option(OPTION_VALUES[:CURLOPT_ENCODING], 'zlib') if supports_zlib?
     end
-    
+
     def headers=(hash)
       @headers = hash
     end
-    
+
+    def proxy=(proxy)
+      set_option(OPTION_VALUES[:CURLOPT_PROXY], proxy)
+    end
+
+    def auth=(authinfo)
+      set_option(OPTION_VALUES[:CURLOPT_USERPWD], "#{authinfo[:username]}:#{authinfo[:password]}")
+      set_option(OPTION_VALUES[:CURLOPT_HTTPAUTH], authinfo[:method]) if authinfo[:method]
+    end
+
+    def auth_methods
+      get_info_long(INFO_VALUES[:CURLINFO_HTTPAUTH_AVAIL])
+    end
+
+    def verbose=(boolean)
+      set_option(OPTION_VALUES[:CURLOPT_VERBOSE], !!boolean ? 1 : 0)
+    end
+
     def total_time_taken
       get_info_double(INFO_VALUES[:CURLINFO_TOTAL_TIME])
     end
-    
+
+    def effective_url
+      get_info_string(INFO_VALUES[:CURLINFO_EFFECTIVE_URL])
+    end
+
     def response_code
       get_info_long(INFO_VALUES[:CURLINFO_RESPONSE_CODE])
     end
-    
+
     def follow_location=(boolean)
       if boolean
         set_option(OPTION_VALUES[:CURLOPT_FOLLOWLOCATION], 1)
@@ -48,17 +96,25 @@ module Typhoeus
         set_option(OPTION_VALUES[:CURLOPT_FOLLOWLOCATION], 0)
       end
     end
-    
+
+    def max_redirects=(redirects)
+      set_option(OPTION_VALUES[:CURLOPT_MAXREDIRS], redirects)
+    end
+
     def timeout=(milliseconds)
       @timeout = milliseconds
       set_option(OPTION_VALUES[:CURLOPT_NOSIGNAL], 1)
       set_option(OPTION_VALUES[:CURLOPT_TIMEOUT_MS], milliseconds)
     end
-    
+
     def timed_out?
       @timeout && total_time_taken > @timeout && response_code == 0
     end
-    
+
+    def supports_zlib?
+      !!(curl_version.match(/zlib/))
+    end
+
     def request_body=(request_body)
       @request_body = request_body
       if @method == :put
@@ -69,16 +125,20 @@ module Typhoeus
         self.post_data = request_body
       end
     end
-    
+
     def user_agent=(user_agent)
       set_option(OPTION_VALUES[:CURLOPT_USERAGENT], user_agent)
     end
-          
+
     def url=(url)
       @url = url
       set_option(OPTION_VALUES[:CURLOPT_URL], url)
     end
-    
+
+    def disable_ssl_peer_verification
+      set_option(OPTION_VALUES[:CURLOPT_VERIFYPEER], 0)
+    end
+
     def method=(method)
       @method = method
       if method == :get
@@ -89,18 +149,21 @@ module Typhoeus
       elsif method == :put
         set_option(OPTION_VALUES[:CURLOPT_UPLOAD], 1)
         self.request_body = "" unless @request_body
+      elsif method == :head
+        set_option(OPTION_VALUES[:CURLOPT_NOBODY], 1)
       else
-        set_option(OPTION_VALUES[:CURLOPT_CUSTOMREQUEST], "DELETE")
+        set_option(OPTION_VALUES[:CURLOPT_CUSTOMREQUEST], method.to_s.upcase)
       end
     end
-    
+
     def post_data=(data)
       @post_data_set = true
       set_option(OPTION_VALUES[:CURLOPT_POSTFIELDS], data)
       set_option(OPTION_VALUES[:CURLOPT_POSTFIELDSIZE], data.length)
     end
-    
+
     def params=(params)
+      @params = params
       params_string = params.keys.collect do |k|
         value = params[k]
         if value.is_a? Hash
@@ -112,57 +175,111 @@ module Typhoeus
           "#{Rack::Utils.escape(k.to_s)}=#{Rack::Utils.escape(params[k].to_s)}"
         end
       end.flatten.join("&")
-      
+
       if method == :post
         self.post_data = params_string
       else
         self.url = "#{url}?#{params_string}"
       end
     end
-    
+
+    # Set SSL certificate
+    # " The string should be the file name of your certificate. "
+    # The default format is "PEM" and can be changed with ssl_cert_type=
+    def ssl_cert=(cert)
+      set_option(OPTION_VALUES[:CURLOPT_SSLCERT], cert)
+    end
+
+    # Set SSL certificate type
+    # " The string should be the format of your certificate. Supported formats are "PEM" and "DER" "
+    def ssl_cert_type=(cert_type)
+      raise "Invalid ssl cert type : '#{cert_type}'..." if cert_type and !%w(PEM DER).include?(cert_type) 
+      set_option(OPTION_VALUES[:CURLOPT_SSLCERTTYPE], cert_type)
+    end
+
+    # Set SSL Key file
+    # " The string should be the file name of your private key. "
+    # The default format is "PEM" and can be changed with ssl_key_type=
+    #
+    def ssl_key=(key)
+      set_option(OPTION_VALUES[:CURLOPT_SSLKEY], key)
+    end
+
+    # Set SSL Key type
+    # " The string should be the format of your private key. Supported formats are "PEM", "DER" and "ENG". "
+    #
+    def ssl_key_type=(key_type)
+      raise "Invalid ssl key type : '#{key_type}'..." if key_type and !%w(PEM DER ENG).include?(key_type)
+      set_option(OPTION_VALUES[:CURLOPT_SSLKEYTYPE], key_type)
+    end
+
+    def ssl_key_password=(key_password)
+      set_option(OPTION_VALUES[:CURLOPT_KEYPASSWD], key_password)
+    end
+
+    # Set SSL CACERT
+    # " File holding one or more certificates to verify the peer with. "
+    #
+    def ssl_cacert=(cacert)
+      set_option(OPTION_VALUES[:CURLOPT_CAINFO], cacert)
+    end
+
+    # Set CAPATH
+    # " directory holding multiple CA certificates to verify the peer with. The certificate directory must be prepared using the openssl c_rehash utility. "
+    #
+    def ssl_capath=(capath)
+      set_option(OPTION_VALUES[:CURLOPT_CAPATH], capath)
+    end
+
     def set_option(option, value)
       if value.class == String
         easy_setopt_string(option, value)
-      else
+      elsif value
         easy_setopt_long(option, value)
       end
     end
-    
+
     def perform
       set_headers()
       easy_perform()
-      response_code()
+      resp_code = response_code() 
+      if resp_code >= 200 && resp_code <= 299
+        success
+      else
+        failure
+      end
+      resp_code
     end
-    
+
     def set_headers
       headers.each_pair do |key, value|
         easy_add_header("#{key}: #{value}")
       end
       easy_set_headers() unless headers.empty?
     end
-    
+
     # gets called when finished and response code is 200-299
     def success
       @success.call(self) if @success
     end
-    
+
     def on_success(&block)
       @success = block
     end
-    
+
     def on_success=(block)
       @success = block
     end
-    
+
     # gets called when finished and response code is 300-599
     def failure
       @failure.call(self) if @failure
     end
-    
+
     def on_failure(&block)
       @failure = block
     end
-    
+
     def on_failure=(block)
       @failure = block
     end
@@ -170,20 +287,20 @@ module Typhoeus
     def retries
       @retries ||= 0
     end
-    
+
     def increment_retries
       @retries ||= 0
       @retries += 1
     end
-    
+
     def max_retries
       @max_retries ||= 40
     end
-    
+
     def max_retries?
       retries >= max_retries
     end
-    
+
     def reset
       @retries = 0
       @response_code = 0
@@ -191,21 +308,22 @@ module Typhoeus
       @response_body = ""
       easy_reset()
     end
-    
+
     def get_info_string(option)
       easy_getinfo_string(option)
     end
-    
+
     def get_info_long(option)
       easy_getinfo_long(option)
     end
-    
+
     def get_info_double(option)
       easy_getinfo_double(option)
     end
-    
+
     def curl_version
       version
     end
+
   end
 end

data/lib/typhoeus/hydra.rb

@@ -1,63 +1,82 @@
 module Typhoeus
   class Hydra
-    def initialize(initial_pool_size = 10)
+    def initialize(options = {})
       @memoize_requests = true
       @multi       = Multi.new
       @easy_pool   = []
+      initial_pool_size = options[:initial_pool_size] || 10
+      @max_concurrency = options[:max_concurrency] || 200
       initial_pool_size.times { @easy_pool << Easy.new }
       @stubs       = []
       @memoized_requests = {}
       @retrieved_from_cache = {}
+      @queued_requests = []
+      @running_requests = 0
+      @stubbed_request_count = 0
     end
 
     def self.hydra
       @hydra ||= new
     end
-    
+
     def self.hydra=(val)
       @hydra = val
     end
-    
+
+    def clear_cache_callbacks
+      @cache_setter = nil
+      @cache_getter = nil
+    end
+
     def clear_stubs
       @stubs = []
     end
-    
+
     def fire_and_forget
+      @queued_requests.each {|r| queue(r, false)}
       @multi.fire_and_forget
     end
 
-    def queue(request)
+    def queue(request, obey_concurrency_limit = true)
       return if assign_to_stub(request)
 
-      if request.method == :get
-        if @memoize_requests && @memoized_requests.has_key?(request.url)
-          if response = @retrieved_from_cache[request.url]
-            request.response = response
-            request.call_handlers
+      if @running_requests >= @max_concurrency && obey_concurrency_limit
+        @queued_requests << request
+      else
+        if request.method == :get
+          if @memoize_requests && @memoized_requests.has_key?(request.url)
+            if response = @retrieved_from_cache[request.url]
+              request.response = response
+              request.call_handlers
+            else
+              @memoized_requests[request.url] << request
+            end
           else
-            @memoized_requests[request.url] << request
+            @memoized_requests[request.url] = [] if @memoize_requests
+            get_from_cache_or_queue(request)
           end
         else
-          @memoized_requests[request.url] = [] if @memoize_requests
           get_from_cache_or_queue(request)
         end
-      else
-        get_from_cache_or_queue(request)
       end
     end
 
     def run
-      @stubs.each do |m|
-        m.requests.each do |request|
-          m.response.request = request
-          handle_request(request, m.response)
+      while @stubbed_request_count > 0
+        @stubs.each do |m|
+          while request = m.requests.shift
+            @stubbed_request_count -= 1
+            m.response.request = request
+            handle_request(request, m.response)
+          end
         end
       end
+
       @multi.perform
       @memoized_requests = {}
       @retrieved_from_cache = {}
     end
-    
+
     def disable_memoization
       @memoize_requests = false
     end
@@ -85,7 +104,12 @@ module Typhoeus
 
     def assign_to_stub(request)
       m = @stubs.detect {|stub| stub.matches? request}
-      m && m.add_request(request)
+      if m
+        m.add_request(request)
+        @stubbed_request_count += 1
+      else
+        nil
+      end
     end
     private :assign_to_stub
 
@@ -105,17 +129,34 @@ module Typhoeus
     private :get_from_cache_or_queue
 
     def get_easy_object(request)
+      @running_requests += 1
+
       easy = @easy_pool.pop || Easy.new
       easy.url          = request.url
       easy.method       = request.method
+      easy.params       = request.params  if request.method == :post && !request.params.nil?
       easy.headers      = request.headers if request.headers
       easy.request_body = request.body    if request.body
       easy.timeout      = request.timeout if request.timeout
+      easy.follow_location = request.follow_location if request.follow_location
+      easy.max_redirects = request.max_redirects if request.max_redirects
+      easy.proxy = request.proxy if request.proxy
+      easy.disable_ssl_peer_verification if request.disable_ssl_peer_verification
+      easy.ssl_cert         = request.ssl_cert
+      easy.ssl_cert_type    = request.ssl_cert_type
+      easy.ssl_key          = request.ssl_key
+      easy.ssl_key_type     = request.ssl_key_type
+      easy.ssl_key_password = request.ssl_key_password
+      easy.ssl_cacert       = request.ssl_cacert
+      easy.ssl_capath       = request.ssl_capath
+
       easy.on_success do |easy|
+        queue_next
         handle_request(request, response_from_easy(easy, request))
         release_easy_object(easy)
       end
       easy.on_failure do |easy|
+        queue_next
         handle_request(request, response_from_easy(easy, request))
         release_easy_object(easy)
       end
@@ -124,6 +165,12 @@ module Typhoeus
     end
     private :get_easy_object
 
+    def queue_next
+      @running_requests -= 1
+      queue(@queued_requests.pop) unless @queued_requests.empty?
+    end
+    private :queue_next
+
     def release_easy_object(easy)
       easy.reset
       @easy_pool.push easy
@@ -153,6 +200,7 @@ module Typhoeus
                    :headers => easy.response_header,
                    :body    => easy.response_body,
                    :time    => easy.total_time_taken,
+                   :effective_url => easy.effective_url,
                    :request => request)
     end
     private :response_from_easy
@@ -174,7 +222,7 @@ module Typhoeus
     def and_return(val)
       @response = val
     end
-    
+
     def matches?(request)
       if url.kind_of?(String)
         request.method == method && request.url == url

data/lib/typhoeus/multi.rb

@@ -9,19 +9,20 @@ module Typhoeus
     def remove(easy)
       multi_remove_handle(easy)
     end
-    
+
     def add(easy)
+      easy.set_headers()
       @easy_handles << easy
       multi_add_handle(easy)
     end
-    
+
     def perform()
       while active_handle_count > 0 do
         multi_perform
       end
       reset_easy_handles
     end
-    
+
     def cleanup()
       multi_cleanup
     end

data/lib/typhoeus/remote_proxy_object.rb

@@ -1,7 +1,7 @@
 module Typhoeus
   class RemoteProxyObject
-    instance_methods.each { |m| undef_method m unless m =~ /^__/ }
-    
+    instance_methods.each { |m| undef_method m unless m =~ /^__|object_id/ }
+
     def initialize(clear_memoized_store_proc, easy, options = {})
       @clear_memoized_store_proc = clear_memoized_store_proc
       @easy      = easy
@@ -12,13 +12,13 @@ module Typhoeus
       @timeout   = options.delete(:cache_timeout)
       Typhoeus.add_easy_request(@easy)
     end
-    
+
     def method_missing(sym, *args, &block)
       unless @proxied_object
         if @cache && @cache_key
           @proxied_object = @cache.get(@cache_key) rescue nil
         end
-        
+
         unless @proxied_object
           Typhoeus.perform_easy_requests
           response = Response.new(:code => @easy.response_code,
@@ -31,7 +31,7 @@ module Typhoeus
           if @easy.response_code >= 200 && @easy.response_code < 300
             Typhoeus.release_easy_object(@easy)
             @proxied_object = @success.nil? ? response : @success.call(response)
-            
+
             if @cache && @cache_key
               @cache.set(@cache_key, @proxied_object, @timeout)
             end
@@ -41,7 +41,7 @@ module Typhoeus
          @clear_memoized_store_proc.call
        end
       end
-      
+
       @proxied_object.__send__(sym, *args, &block)
     end
   end