fblee-typhoeus 0.1.31

data/lib/typhoeus/easy.rb

@@ -0,0 +1,348 @@
+module Typhoeus
+  class Easy
+    attr_reader :response_body, :response_header, :method, :headers, :url, :params
+    attr_accessor :start_time
+
+    # These integer codes are available in curl/curl.h
+    CURLINFO_STRING = 1048576
+    OPTION_VALUES = {
+      :CURLOPT_URL            => 10002,
+      :CURLOPT_HTTPGET        => 80,
+      :CURLOPT_HTTPPOST       => 10024,
+      :CURLOPT_UPLOAD         => 46,
+      :CURLOPT_CUSTOMREQUEST  => 10036,
+      :CURLOPT_POSTFIELDS     => 10015,
+      :CURLOPT_COPYPOSTFIELDS     => 10165,
+      :CURLOPT_POSTFIELDSIZE  => 60,
+      :CURLOPT_USERAGENT      => 10018,
+      :CURLOPT_TIMEOUT_MS     => 155,
+      # Time-out connect operations after this amount of milliseconds.
+      # [Only works on unix-style/SIGALRM operating systems. IOW, does
+      # not work on Windows.
+      :CURLOPT_CONNECTTIMEOUT_MS  => 156,
+      :CURLOPT_NOSIGNAL       => 99,
+      :CURLOPT_HTTPHEADER     => 10023,
+      :CURLOPT_FOLLOWLOCATION => 52,
+      :CURLOPT_MAXREDIRS      => 68,
+      :CURLOPT_HTTPAUTH       => 107,
+      :CURLOPT_USERPWD        => 10000 + 5,
+      :CURLOPT_VERBOSE        => 41,
+      :CURLOPT_COOKIEFILE     => 10000 + 31,
+      :CURLOPT_PROXY          => 10004,
+      :CURLOPT_VERIFYPEER     => 64,
+      :CURLOPT_NOBODY         => 44,
+      :CURLOPT_ENCODING       => 10000 + 102,
+      :CURLOPT_SSLCERT        => 10025,
+      :CURLOPT_SSLCERTTYPE    => 10086,
+      :CURLOPT_SSLKEY         => 10087,
+      :CURLOPT_SSLKEYTYPE     => 10088,
+      :CURLOPT_KEYPASSWD      => 10026,
+      :CURLOPT_CAINFO         => 10065,
+      :CURLOPT_CAPATH         => 10097
+    }
+    INFO_VALUES = {
+      :CURLINFO_RESPONSE_CODE => 2097154,
+      :CURLINFO_TOTAL_TIME    => 3145731,
+      :CURLINFO_HTTPAUTH_AVAIL => 0x200000 + 23,
+      :CURLINFO_EFFECTIVE_URL => 0x100000 + 1
+    }
+    AUTH_TYPES = {
+      :CURLAUTH_BASIC         => 1,
+      :CURLAUTH_DIGEST        => 2,
+      :CURLAUTH_GSSNEGOTIATE  => 4,
+      :CURLAUTH_NTLM          => 8,
+      :CURLAUTH_DIGEST_IE     => 16,
+      :CURLAUTH_AUTO          => 16 | 8 | 4 | 2 | 1
+    }
+
+    def initialize
+      @method = :get
+      @headers = {}
+
+      # Enable encoding/compression support
+      set_option(OPTION_VALUES[:CURLOPT_ENCODING], '')
+    end
+
+    def headers=(hash)
+      @headers = hash
+    end
+
+    def proxy=(proxy)
+      set_option(OPTION_VALUES[:CURLOPT_PROXY], proxy)
+    end
+
+    def auth=(authinfo)
+      set_option(OPTION_VALUES[:CURLOPT_USERPWD], "#{authinfo[:username]}:#{authinfo[:password]}")
+      set_option(OPTION_VALUES[:CURLOPT_HTTPAUTH], authinfo[:method]) if authinfo[:method]
+    end
+
+    def auth_methods
+      get_info_long(INFO_VALUES[:CURLINFO_HTTPAUTH_AVAIL])
+    end
+
+    def verbose=(boolean)
+      set_option(OPTION_VALUES[:CURLOPT_VERBOSE], !!boolean ? 1 : 0)
+    end
+
+    def total_time_taken
+      get_info_double(INFO_VALUES[:CURLINFO_TOTAL_TIME])
+    end
+
+    def effective_url
+      get_info_string(INFO_VALUES[:CURLINFO_EFFECTIVE_URL])
+    end
+
+    def response_code
+      get_info_long(INFO_VALUES[:CURLINFO_RESPONSE_CODE])
+    end
+
+    def follow_location=(boolean)
+      if boolean
+        set_option(OPTION_VALUES[:CURLOPT_FOLLOWLOCATION], 1)
+      else
+        set_option(OPTION_VALUES[:CURLOPT_FOLLOWLOCATION], 0)
+      end
+    end
+    
+    def enable_cookies=(boolean)
+      if boolean
+        set_option(OPTION_VALUES[:CURLOPT_COOKIEFILE], '')            
+      end
+    end
+
+    def max_redirects=(redirects)
+      set_option(OPTION_VALUES[:CURLOPT_MAXREDIRS], redirects)
+    end
+    
+    def connect_timeout=(milliseconds)
+      @connect_timeout = milliseconds
+      set_option(OPTION_VALUES[:CURLOPT_NOSIGNAL], 1)
+      set_option(OPTION_VALUES[:CURLOPT_CONNECTTIMEOUT_MS], milliseconds)
+    end
+
+    def timeout=(milliseconds)
+      @timeout = milliseconds
+      set_option(OPTION_VALUES[:CURLOPT_NOSIGNAL], 1)
+      set_option(OPTION_VALUES[:CURLOPT_TIMEOUT_MS], milliseconds)
+    end
+
+    def timed_out?
+      @timeout && total_time_taken > @timeout && response_code == 0
+    end
+
+    def supports_zlib?
+      !!(curl_version.match(/zlib/))
+    end
+
+    def request_body=(request_body)
+      @request_body = request_body
+      if @method == :put
+        easy_set_request_body(@request_body)
+        headers["Transfer-Encoding"] = ""
+        headers["Expect"] = ""
+      else
+        self.post_data = request_body
+      end
+    end
+
+    def user_agent=(user_agent)
+      set_option(OPTION_VALUES[:CURLOPT_USERAGENT], user_agent)
+    end
+
+    def url=(url)
+      @url = url
+      set_option(OPTION_VALUES[:CURLOPT_URL], url)
+    end
+
+    def disable_ssl_peer_verification
+      set_option(OPTION_VALUES[:CURLOPT_VERIFYPEER], 0)
+    end
+
+    def method=(method)
+      @method = method
+      if method == :get
+        set_option(OPTION_VALUES[:CURLOPT_HTTPGET], 1)
+      elsif method == :post
+        set_option(OPTION_VALUES[:CURLOPT_HTTPPOST], 1)
+        self.post_data = ""
+      elsif method == :put
+        set_option(OPTION_VALUES[:CURLOPT_UPLOAD], 1)
+        self.request_body = "" unless @request_body
+      elsif method == :head
+        set_option(OPTION_VALUES[:CURLOPT_NOBODY], 1)
+      else
+        set_option(OPTION_VALUES[:CURLOPT_CUSTOMREQUEST], method.to_s.upcase)
+      end
+    end
+
+    def post_data=(data)
+      @post_data_set = true
+      set_option(OPTION_VALUES[:CURLOPT_POSTFIELDSIZE], data.length)
+      set_option(OPTION_VALUES[:CURLOPT_COPYPOSTFIELDS], data)
+    end
+
+    def params=(params)
+      @params = params
+      params_string = params.keys.collect do |k|
+        value = params[k]
+        if value.is_a? Hash
+          value.keys.collect {|sk| Rack::Utils.escape("#{k}[#{sk}]") + "=" + Rack::Utils.escape(value[sk].to_s)}
+        elsif value.is_a? Array
+          key = Rack::Utils.escape(k.to_s)
+          value.collect { |v| "#{key}=#{Rack::Utils.escape(v.to_s)}" }.join('&')
+        else
+          "#{Rack::Utils.escape(k.to_s)}=#{Rack::Utils.escape(params[k].to_s)}"
+        end
+      end.flatten.join("&")
+
+      if method == :post
+        self.post_data = params_string
+      else
+        self.url = "#{url}?#{params_string}"
+      end
+    end
+
+    # Set SSL certificate
+    # " The string should be the file name of your certificate. "
+    # The default format is "PEM" and can be changed with ssl_cert_type=
+    def ssl_cert=(cert)
+      set_option(OPTION_VALUES[:CURLOPT_SSLCERT], cert)
+    end
+
+    # Set SSL certificate type
+    # " The string should be the format of your certificate. Supported formats are "PEM" and "DER" "
+    def ssl_cert_type=(cert_type)
+      raise "Invalid ssl cert type : '#{cert_type}'..." if cert_type and !%w(PEM DER).include?(cert_type) 
+      set_option(OPTION_VALUES[:CURLOPT_SSLCERTTYPE], cert_type)
+    end
+
+    # Set SSL Key file
+    # " The string should be the file name of your private key. "
+    # The default format is "PEM" and can be changed with ssl_key_type=
+    #
+    def ssl_key=(key)
+      set_option(OPTION_VALUES[:CURLOPT_SSLKEY], key)
+    end
+
+    # Set SSL Key type
+    # " The string should be the format of your private key. Supported formats are "PEM", "DER" and "ENG". "
+    #
+    def ssl_key_type=(key_type)
+      raise "Invalid ssl key type : '#{key_type}'..." if key_type and !%w(PEM DER ENG).include?(key_type)
+      set_option(OPTION_VALUES[:CURLOPT_SSLKEYTYPE], key_type)
+    end
+
+    def ssl_key_password=(key_password)
+      set_option(OPTION_VALUES[:CURLOPT_KEYPASSWD], key_password)
+    end
+
+    # Set SSL CACERT
+    # " File holding one or more certificates to verify the peer with. "
+    #
+    def ssl_cacert=(cacert)
+      set_option(OPTION_VALUES[:CURLOPT_CAINFO], cacert)
+    end
+
+    # Set CAPATH
+    # " directory holding multiple CA certificates to verify the peer with. The certificate directory must be prepared using the openssl c_rehash utility. "
+    #
+    def ssl_capath=(capath)
+      set_option(OPTION_VALUES[:CURLOPT_CAPATH], capath)
+    end
+
+    def set_option(option, value)
+      if value.class == String
+        easy_setopt_string(option, value)
+      elsif value
+        easy_setopt_long(option, value)
+      end
+    end
+
+    def perform
+      set_headers()
+      easy_perform()
+      resp_code = response_code() 
+      if resp_code >= 200 && resp_code <= 299
+        success
+      else
+        failure
+      end
+      resp_code
+    end
+
+    def set_headers
+      headers.each_pair do |key, value|
+        easy_add_header("#{key}: #{value}")
+      end
+      easy_set_headers() unless headers.empty?
+    end
+
+    # gets called when finished and response code is 200-299
+    def success
+      @success.call(self) if @success
+    end
+
+    def on_success(&block)
+      @success = block
+    end
+
+    def on_success=(block)
+      @success = block
+    end
+
+    # gets called when finished and response code is 300-599
+    def failure
+      @failure.call(self) if @failure
+    end
+
+    def on_failure(&block)
+      @failure = block
+    end
+
+    def on_failure=(block)
+      @failure = block
+    end
+
+    def retries
+      @retries ||= 0
+    end
+
+    def increment_retries
+      @retries ||= 0
+      @retries += 1
+    end
+
+    def max_retries
+      @max_retries ||= 40
+    end
+
+    def max_retries?
+      retries >= max_retries
+    end
+
+    def reset
+      @retries = 0
+      @response_code = 0
+      @response_header = ""
+      @response_body = ""
+      easy_reset()
+    end
+
+    def get_info_string(option)
+      easy_getinfo_string(option)
+    end
+
+    def get_info_long(option)
+      easy_getinfo_long(option)
+    end
+
+    def get_info_double(option)
+      easy_getinfo_double(option)
+    end
+
+    def curl_version
+      version
+    end
+
+  end
+end

data/lib/typhoeus/filter.rb

@@ -0,0 +1,28 @@
+module Typhoeus
+  class Filter
+    attr_reader :method_name
+    
+    def initialize(method_name, options = {})
+      @method_name = method_name
+      @options = options
+    end
+    
+    def apply_filter?(method_name)
+      if @options[:only]
+        if @options[:only].instance_of? Symbol
+          @options[:only] == method_name
+        else
+          @options[:only].include?(method_name)
+        end
+      elsif @options[:except]
+        if @options[:except].instance_of? Symbol
+          @options[:except] != method_name
+        else
+          !@options[:except].include?(method_name)
+        end
+      else
+        true
+      end
+    end
+  end
+end

data/lib/typhoeus/hydra.rb

@@ -0,0 +1,243 @@
+module Typhoeus
+  class Hydra
+    def initialize(options = {})
+      @memoize_requests = true
+      @multi       = Multi.new
+      @easy_pool   = []
+      initial_pool_size = options[:initial_pool_size] || 10
+      @max_concurrency = options[:max_concurrency] || 200
+      initial_pool_size.times { @easy_pool << Easy.new }
+      @stubs       = []
+      @memoized_requests = {}
+      @retrieved_from_cache = {}
+      @queued_requests = []
+      @running_requests = 0
+      @stubbed_request_count = 0
+    end
+
+    def self.hydra
+      @hydra ||= new
+    end
+
+    def self.hydra=(val)
+      @hydra = val
+    end
+
+    def clear_cache_callbacks
+      @cache_setter = nil
+      @cache_getter = nil
+    end
+
+    def clear_stubs
+      @stubs = []
+    end
+
+    def fire_and_forget
+      @queued_requests.each {|r| queue(r, false)}
+      @multi.fire_and_forget
+    end
+
+    def queue(request, obey_concurrency_limit = true)
+      return if assign_to_stub(request)
+
+      if @running_requests >= @max_concurrency && obey_concurrency_limit
+        @queued_requests << request
+      else
+        if request.method == :get
+          if @memoize_requests && @memoized_requests.has_key?(request.url)
+            if response = @retrieved_from_cache[request.url]
+              request.response = response
+              request.call_handlers
+            else
+              @memoized_requests[request.url] << request
+            end
+          else
+            @memoized_requests[request.url] = [] if @memoize_requests
+            get_from_cache_or_queue(request)
+          end
+        else
+          get_from_cache_or_queue(request)
+        end
+      end
+    end
+
+    def run
+      while @stubbed_request_count > 0
+        @stubs.each do |m|
+          while request = m.requests.shift
+            @stubbed_request_count -= 1
+            m.response.request = request
+            handle_request(request, m.response)
+          end
+        end
+      end
+
+      @multi.perform
+      @memoized_requests = {}
+      @retrieved_from_cache = {}
+    end
+
+    def disable_memoization
+      @memoize_requests = false
+    end
+
+    def cache_getter(&block)
+      @cache_getter = block
+    end
+
+    def cache_setter(&block)
+      @cache_setter = block
+    end
+
+    def on_complete(&block)
+      @on_complete = block
+    end
+
+    def on_complete=(proc)
+      @on_complete = proc
+    end
+
+    def stub(method, url)
+      @stubs << HydraMock.new(url, method)
+      @stubs.last
+    end
+
+    def assign_to_stub(request)
+      m = @stubs.detect {|stub| stub.matches? request}
+      if m
+        m.add_request(request)
+        @stubbed_request_count += 1
+      else
+        nil
+      end
+    end
+    private :assign_to_stub
+
+    def get_from_cache_or_queue(request)
+      if @cache_getter
+        val = @cache_getter.call(request)
+        if val
+          @retrieved_from_cache[request.url] = val
+          handle_request(request, val, false)
+        else
+          @multi.add(get_easy_object(request))
+        end
+      else
+        @multi.add(get_easy_object(request))
+      end
+    end
+    private :get_from_cache_or_queue
+
+    def get_easy_object(request)
+      @running_requests += 1
+
+      easy = @easy_pool.pop || Easy.new
+      easy.verbose          = request.verbose
+      if request.username || request.password
+        auth = { :username => request.username, :password => request.password }
+        auth[:method] = Typhoeus::Easy::AUTH_TYPES["CURLAUTH_#{request.auth_method.to_s.upcase}".to_sym] if request.auth_method
+        easy.auth = auth
+      end
+      easy.url          = request.url
+      easy.method       = request.method
+      easy.params       = request.params  if request.method == :post && !request.params.nil?
+      easy.headers      = request.headers if request.headers
+      easy.request_body = request.body    if request.body
+      easy.timeout      = request.timeout if request.timeout
+      easy.connect_timeout = request.connect_timeout if request.connect_timeout
+      easy.follow_location = request.follow_location if request.follow_location
+      easy.enable_cookies = request.enable_cookies if request.enable_cookies
+      easy.max_redirects = request.max_redirects if request.max_redirects
+      easy.proxy = request.proxy if request.proxy
+      easy.disable_ssl_peer_verification if request.disable_ssl_peer_verification
+      easy.ssl_cert         = request.ssl_cert
+      easy.ssl_cert_type    = request.ssl_cert_type
+      easy.ssl_key          = request.ssl_key
+      easy.ssl_key_type     = request.ssl_key_type
+      easy.ssl_key_password = request.ssl_key_password
+      easy.ssl_cacert       = request.ssl_cacert
+      easy.ssl_capath       = request.ssl_capath
+      easy.verbose          = request.verbose
+
+      easy.on_success do |easy|
+        queue_next
+        handle_request(request, response_from_easy(easy, request))
+        release_easy_object(easy)
+      end
+      easy.on_failure do |easy|
+        queue_next
+        handle_request(request, response_from_easy(easy, request))
+        release_easy_object(easy)
+      end
+      easy.set_headers
+      easy
+    end
+    private :get_easy_object
+
+    def queue_next
+      @running_requests -= 1
+      queue(@queued_requests.pop) unless @queued_requests.empty?
+    end
+    private :queue_next
+
+    def release_easy_object(easy)
+      easy.reset
+      @easy_pool.push easy
+    end
+    private :release_easy_object
+
+    def handle_request(request, response, live_request = true)
+      request.response = response
+
+      if live_request && request.cache_timeout && @cache_setter
+        @cache_setter.call(request)
+      end
+      @on_complete.call(response) if @on_complete
+
+      request.call_handlers
+      if requests = @memoized_requests[request.url]
+        requests.each do |r|
+          r.response = response
+          r.call_handlers
+        end
+      end
+    end
+    private :handle_request
+
+    def response_from_easy(easy, request)
+      Response.new(:code    => easy.response_code,
+                   :headers => easy.response_header,
+                   :body    => easy.response_body,
+                   :time    => easy.total_time_taken,
+                   :effective_url => easy.effective_url,
+                   :request => request)
+    end
+    private :response_from_easy
+  end
+
+  class HydraMock
+    attr_reader :url, :method, :response, :requests
+
+    def initialize(url, method)
+      @url      = url
+      @method   = method
+      @requests = []
+    end
+
+    def add_request(request)
+      @requests << request
+    end
+
+    def and_return(val)
+      @response = val
+    end
+
+    def matches?(request)
+      if url.kind_of?(String)
+        request.method == method && request.url == url
+      else
+        request.method == method && url =~ request.url
+      end
+    end
+  end
+end

data/lib/typhoeus/multi.rb

@@ -0,0 +1,35 @@
+module Typhoeus
+  class Multi
+    attr_reader :easy_handles
+
+    def initialize
+      reset_easy_handles
+    end
+
+    def remove(easy)
+      multi_remove_handle(easy)
+    end
+
+    def add(easy)
+      easy.set_headers() if easy.headers.empty?
+      @easy_handles << easy
+      multi_add_handle(easy)
+    end
+
+    def perform()
+      while active_handle_count > 0 do
+        multi_perform
+      end
+      reset_easy_handles
+    end
+
+    def cleanup()
+      multi_cleanup
+    end
+
+    private
+    def reset_easy_handles
+      @easy_handles = []
+    end
+  end
+end