typhoeus 0.2.0 → 0.2.4

data/lib/typhoeus/easy.rb

@@ -1,6 +1,6 @@
 module Typhoeus
   class Easy
-    attr_reader :response_body, :response_header, :method, :headers, :url, :params
+    attr_reader :response_body, :response_header, :method, :headers, :url, :params, :curl_return_code
     attr_accessor :start_time
 
     # These integer codes are available in curl/curl.h
@@ -28,6 +28,9 @@ module Typhoeus
       :CURLOPT_USERPWD        => 10000 + 5,
       :CURLOPT_VERBOSE        => 41,
       :CURLOPT_PROXY          => 10004,
+      :CURLOPT_PROXYUSERPWD   => 10000 + 6,
+      :CURLOPT_PROXYTYPE      => 101,
+      :CURLOPT_PROXYAUTH      => 111,
       :CURLOPT_VERIFYPEER     => 64,
       :CURLOPT_NOBODY         => 44,
       :CURLOPT_ENCODING       => 10000 + 102,
@@ -37,13 +40,19 @@ module Typhoeus
       :CURLOPT_SSLKEYTYPE     => 10088,
       :CURLOPT_KEYPASSWD      => 10026,
       :CURLOPT_CAINFO         => 10065,
-      :CURLOPT_CAPATH         => 10097
+      :CURLOPT_CAPATH         => 10097,
     }
     INFO_VALUES = {
-      :CURLINFO_RESPONSE_CODE => 2097154,
-      :CURLINFO_TOTAL_TIME    => 3145731,
-      :CURLINFO_HTTPAUTH_AVAIL => 0x200000 + 23,
-      :CURLINFO_EFFECTIVE_URL => 0x100000 + 1
+      :CURLINFO_RESPONSE_CODE      => 2097154,
+      :CURLINFO_TOTAL_TIME         => 3145731,
+      :CURLINFO_HTTPAUTH_AVAIL     => 0x200000 + 23,
+      :CURLINFO_EFFECTIVE_URL      => 0x100000 + 1,
+      :CURLINFO_NAMELOOKUP_TIME    => 0x300000 + 4,
+      :CURLINFO_CONNECT_TIME       => 0x300000 + 5,
+      :CURLINFO_PRETRANSFER_TIME   => 0x300000 + 6,
+      :CURLINFO_STARTTRANSFER_TIME => 0x300000 + 17,
+      :CURLINFO_APPCONNECT_TIME    => 0x300000 + 33,
+
     }
     AUTH_TYPES = {
       :CURLAUTH_BASIC         => 1,
@@ -53,6 +62,14 @@ module Typhoeus
       :CURLAUTH_DIGEST_IE     => 16,
       :CURLAUTH_AUTO          => 16 | 8 | 4 | 2 | 1
     }
+    PROXY_TYPES = {
+      :CURLPROXY_HTTP         => 0,
+      :CURLPROXY_HTTP_1_0     => 1,
+      :CURLPROXY_SOCKS4       => 4,
+      :CURLPROXY_SOCKS5       => 5,
+      :CURLPROXY_SOCKS4A      => 6,
+    }
+
 
     def initialize
       @method = :get
@@ -67,7 +84,13 @@ module Typhoeus
     end
 
     def proxy=(proxy)
-      set_option(OPTION_VALUES[:CURLOPT_PROXY], proxy)
+      set_option(OPTION_VALUES[:CURLOPT_PROXY], proxy[:server])
+      set_option(OPTION_VALUES[:CURLOPT_PROXYTYPE], proxy[:type]) if proxy[:type]
+    end
+
+    def proxy_auth=(authinfo)
+      set_option(OPTION_VALUES[:CURLOPT_PROXYUSERPWD], "#{authinfo[:username]}:#{authinfo[:password]}")
+      set_option(OPTION_VALUES[:CURLOPT_PROXYAUTH], authinfo[:method]) if authinfo[:method]
     end
 
     def auth=(authinfo)
@@ -87,6 +110,26 @@ module Typhoeus
       get_info_double(INFO_VALUES[:CURLINFO_TOTAL_TIME])
     end
 
+    def start_transfer_time
+      get_info_double(INFO_VALUES[:CURLINFO_STARTTRANSFER_TIME])
+    end
+
+    def app_connect_time
+      get_info_double(INFO_VALUES[:CURLINFO_APPCONNECT_TIME])
+    end
+
+    def pretransfer_time
+      get_info_double(INFO_VALUES[:CURLINFO_PRETRANSFER_TIME])
+    end
+
+    def connect_time
+      get_info_double(INFO_VALUES[:CURLINFO_CONNECT_TIME])
+    end
+
+    def name_lookup_time
+      get_info_double(INFO_VALUES[:CURLINFO_NAMELOOKUP_TIME])
+    end
+
     def effective_url
       get_info_string(INFO_VALUES[:CURLINFO_EFFECTIVE_URL])
     end
@@ -106,7 +149,7 @@ module Typhoeus
     def max_redirects=(redirects)
       set_option(OPTION_VALUES[:CURLOPT_MAXREDIRS], redirects)
     end
-    
+
     def connect_timeout=(milliseconds)
       @connect_timeout = milliseconds
       set_option(OPTION_VALUES[:CURLOPT_NOSIGNAL], 1)
@@ -120,7 +163,7 @@ module Typhoeus
     end
 
     def timed_out?
-      @timeout && total_time_taken > @timeout && response_code == 0
+      curl_return_code == 28
     end
 
     def supports_zlib?
@@ -174,24 +217,22 @@ module Typhoeus
       set_option(OPTION_VALUES[:CURLOPT_COPYPOSTFIELDS], data)
     end
 
+    def params
+      @form.nil? ? {} : @form.params
+    end
+
     def params=(params)
-      @params = params
-      params_string = params.keys.collect do |k|
-        value = params[k]
-        if value.is_a? Hash
-          value.keys.collect {|sk| Typhoeus::Utils.escape("#{k}[#{sk}]") + "=" + Typhoeus::Utils.escape(value[sk].to_s)}
-        elsif value.is_a? Array
-          key = Typhoeus::Utils.escape(k.to_s)
-          value.collect { |v| "#{key}=#{Typhoeus::Utils.escape(v.to_s)}" }.join('&')
-        else
-          "#{Typhoeus::Utils.escape(k.to_s)}=#{Typhoeus::Utils.escape(params[k].to_s)}"
-        end
-      end.flatten.join("&")
+      @form = Typhoeus::Form.new(params)
 
       if method == :post
-        self.post_data = params_string
+        @form.process!
+        if @form.multipart?
+          set_option(OPTION_VALUES[:CURLOPT_HTTPPOST], @form)
+        else
+          self.post_data = @form.to_s
+        end
       else
-        self.url = "#{url}?#{params_string}"
+        self.url = "#{url}?#{@form.to_s}"
       end
     end
 
@@ -205,7 +246,7 @@ module Typhoeus
     # Set SSL certificate type
     # " The string should be the format of your certificate. Supported formats are "PEM" and "DER" "
     def ssl_cert_type=(cert_type)
-      raise "Invalid ssl cert type : '#{cert_type}'..." if cert_type and !%w(PEM DER).include?(cert_type) 
+      raise "Invalid ssl cert type : '#{cert_type}'..." if cert_type and !%w(PEM DER).include?(cert_type)
       set_option(OPTION_VALUES[:CURLOPT_SSLCERTTYPE], cert_type)
     end
 
@@ -244,17 +285,20 @@ module Typhoeus
     end
 
     def set_option(option, value)
-      if value.class == String
-        easy_setopt_string(option, value)
-      elsif value
-        easy_setopt_long(option, value)
+      case value
+        when String
+          easy_setopt_string(option, value)
+        when Typhoeus::Form
+          easy_setopt_form(option, value)
+        else
+          easy_setopt_long(option, value) if value
       end
     end
 
     def perform
       set_headers()
       easy_perform()
-      resp_code = response_code() 
+      resp_code = response_code()
       if resp_code >= 200 && resp_code <= 299
         success
       else
@@ -283,7 +327,7 @@ module Typhoeus
       @success = block
     end
 
-    # gets called when finished and response code is 300-599
+    # gets called when finished and response code is 300-599 or curl returns an error code
     def failure
       @failure.call(self) if @failure
     end
@@ -296,25 +340,7 @@ module Typhoeus
       @failure = block
     end
 
-    def retries
-      @retries ||= 0
-    end
-
-    def increment_retries
-      @retries ||= 0
-      @retries += 1
-    end
-
-    def max_retries
-      @max_retries ||= 40
-    end
-
-    def max_retries?
-      retries >= max_retries
-    end
-
     def reset
-      @retries = 0
       @response_code = 0
       @response_header = ""
       @response_body = ""

data/lib/typhoeus/form.rb

@@ -0,0 +1,32 @@
+require 'mime/types'
+
+module Typhoeus
+  class Form
+    attr_accessor :params
+    attr_reader :traversal
+
+    def initialize(params = {})
+      @params = params
+    end
+
+    def traversal
+      @traversal ||= Typhoeus::Utils.traverse_params_hash(params)
+    end
+
+    def process!
+      # add params
+      traversal[:params].each { |p| formadd_param(p[0], p[1]) }
+
+      # add files
+      traversal[:files].each { |file_args| formadd_file(*file_args) }
+    end
+
+    def multipart?
+      !traversal[:files].empty?
+    end
+
+    def to_s
+      Typhoeus::Utils.traversal_to_param_string(traversal, false)
+    end
+  end
+end

data/lib/typhoeus/hydra/connect_options.rb

@@ -12,9 +12,10 @@ module Typhoeus
       #
       # @raises NetConnectNotAllowedError
       def check_allow_net_connect!(request)
-        if !Typhoeus::Hydra.allow_net_connect? and (!Typhoeus::Hydra.ignore_localhost? or !request.localhost?)
-          raise NetConnectNotAllowedError, "Real HTTP requests are not allowed. Unregistered request: #{request.inspect}"
-        end
+        return if Typhoeus::Hydra.allow_net_connect?
+        return if Typhoeus::Hydra.ignore_hosts.include?(request.host_domain)
+
+        raise NetConnectNotAllowedError, "Real HTTP requests are not allowed. Unregistered request: #{request.inspect}"
       end
       private :check_allow_net_connect!
 
@@ -39,7 +40,22 @@ module Typhoeus
         def ignore_localhost?
           ignore_localhost
         end
+
+        def ignore_hosts
+          @ignore_hosts ||= []
+
+          if ignore_localhost?
+            @ignore_hosts + Typhoeus::Request::LOCALHOST_ALIASES
+          else
+            @ignore_hosts
+          end
+        end
+
+        def ignore_hosts=(hosts)
+          @ignore_hosts = hosts
+        end
       end
     end
   end
 end
+

data/lib/typhoeus/hydra.rb

@@ -32,6 +32,17 @@ module Typhoeus
       @hydra = val
     end
 
+    #
+    # Abort the run on a best-effort basis.
+    #
+    # It won't abort the current burst of @max_concurrency requests,
+    # however it won't fire the rest of the queued requests so the run
+    # will be aborted as soon as possible...
+    #
+    def abort
+      @queued_requests.clear
+    end
+
     def clear_cache_callbacks
       @cache_setter = nil
       @cache_getter = nil
@@ -82,8 +93,11 @@ module Typhoeus
       end
 
       @multi.perform
+    ensure
+      @multi.reset_easy_handles{|easy| release_easy_object(easy)}
       @memoized_requests = {}
       @retrieved_from_cache = {}
+      @running_requests = 0
     end
 
     def disable_memoization
@@ -131,6 +145,19 @@ module Typhoeus
         auth[:method] = Typhoeus::Easy::AUTH_TYPES["CURLAUTH_#{request.auth_method.to_s.upcase}".to_sym] if request.auth_method
         easy.auth = auth
       end
+
+      if request.proxy
+        proxy = { :server => request.proxy }
+        proxy[:type] = Typhoeus::Easy::PROXY_TYPES["CURLPROXY_#{request.proxy_type.to_s.upcase}".to_sym] if request.proxy_type
+        easy.proxy = proxy if request.proxy
+      end
+
+      if request.proxy_username || request.proxy_password
+        auth = { :username => request.proxy_username, :password => request.proxy_password }
+        auth[:method] = Typhoeus::Easy::AUTH_TYPES["CURLAUTH_#{request.proxy_auth_method.to_s.upcase}".to_sym] if request.proxy_auth_method
+        easy.proxy_auth = auth
+      end
+
       easy.url          = request.url
       easy.method       = request.method
       easy.params       = request.params  if request.method == :post && !request.params.nil?
@@ -140,7 +167,6 @@ module Typhoeus
       easy.connect_timeout = request.connect_timeout if request.connect_timeout
       easy.follow_location = request.follow_location if request.follow_location
       easy.max_redirects = request.max_redirects if request.max_redirects
-      easy.proxy = request.proxy if request.proxy
       easy.disable_ssl_peer_verification if request.disable_ssl_peer_verification
       easy.ssl_cert         = request.ssl_cert
       easy.ssl_cert_type    = request.ssl_cert_type
@@ -200,12 +226,19 @@ module Typhoeus
     private :handle_request
 
     def response_from_easy(easy, request)
-      Response.new(:code    => easy.response_code,
-                   :headers => easy.response_header,
-                   :body    => easy.response_body,
-                   :time    => easy.total_time_taken,
-                   :effective_url => easy.effective_url,
-                   :request => request)
+      Response.new(:code                => easy.response_code,
+                   :headers             => easy.response_header,
+                   :body                => easy.response_body,
+                   :time                => easy.total_time_taken,
+                   :start_transfer_time => easy.start_transfer_time,
+                   :app_connect_time    => easy.app_connect_time,
+                   :pretransfer_time    => easy.pretransfer_time,
+                   :connect_time        => easy.connect_time,
+                   :name_lookup_time    => easy.name_lookup_time,
+                   :effective_url       => easy.effective_url,
+                   :curl_return_code => easy.curl_return_code,
+                   :curl_error_message => easy.curl_error_message,
+                   :request             => request)
     end
     private :response_from_easy
   end

data/lib/typhoeus/multi.rb

@@ -3,16 +3,16 @@ module Typhoeus
     attr_reader :easy_handles
 
     def initialize
-      reset_easy_handles
+      @easy_handles = []
     end
 
     def remove(easy)
-      multi_remove_handle(easy)
+      multi_remove_handle(easy) if @easy_handles.include?(easy)
     end
 
     def add(easy)
+      raise "trying to add easy handle twice" if @easy_handles.include?(easy)
       easy.set_headers() if easy.headers.empty?
-      @easy_handles << easy
       multi_add_handle(easy)
     end
 
@@ -27,9 +27,11 @@ module Typhoeus
       multi_cleanup
     end
 
-    private
     def reset_easy_handles
-      @easy_handles = []
+      @easy_handles.dup.each do |easy|
+        multi_remove_handle(easy)
+        yield easy if block_given?
+      end
     end
   end
 end

data/lib/typhoeus/remote.rb

@@ -1,5 +1,5 @@
 module Typhoeus
-  USER_AGENT = "Typhoeus - http://github.com/pauldix/typhoeus/tree/master"
+  USER_AGENT = "Typhoeus - http://github.com/dbalatero/typhoeus/tree/master"
   
   def self.included(base)
     base.extend ClassMethods

data/lib/typhoeus/remote_proxy_object.rb

@@ -22,6 +22,8 @@ module Typhoeus
         unless @proxied_object
           Typhoeus.perform_easy_requests
           response = Response.new(:code => @easy.response_code,
+                                  :curl_return_code => @easy.curl_return_code,
+                                  :curl_error_message => @easy.curl_error_message,
                                   :headers => @easy.response_header,
                                   :body => @easy.response_body,
                                   :time => @easy.total_time_taken,

data/lib/typhoeus/request.rb

@@ -6,10 +6,12 @@ module Typhoeus
     attr_writer   :headers
     attr_accessor :method, :params, :body, :connect_timeout, :timeout,
                   :user_agent, :response, :cache_timeout, :follow_location,
-                  :max_redirects, :proxy, :disable_ssl_peer_verification,
+                  :max_redirects, :proxy, :proxy_username,:proxy_password,
+                  :disable_ssl_peer_verification,
                   :ssl_cert, :ssl_cert_type, :ssl_key, :ssl_key_type,
                   :ssl_key_password, :ssl_cacert, :ssl_capath, :verbose,
-                  :username, :password, :auth_method, :user_agent
+                  :username, :password, :auth_method, :user_agent,
+                  :proxy_auth_method, :proxy_type
 
     # Initialize a new Request
     #
@@ -52,6 +54,10 @@ module Typhoeus
       @follow_location  = options[:follow_location]
       @max_redirects    = options[:max_redirects]
       @proxy            = options[:proxy]
+      @proxy_type       = options[:proxy_type]
+      @proxy_username   = options[:proxy_username]
+      @proxy_password   = options[:proxy_password]
+      @proxy_auth_method = options[:proxy_auth_method]
       @disable_ssl_peer_verification = options[:disable_ssl_peer_verification]
       @ssl_cert         = options[:ssl_cert]
       @ssl_cert_type    = options[:ssl_cert_type]
@@ -78,8 +84,10 @@ module Typhoeus
       @handled_response = nil
     end
 
+    LOCALHOST_ALIASES = %w[ localhost 127.0.0.1 0.0.0.0 ]
+
     def localhost?
-      %(localhost 127.0.0.1 0.0.0.0).include?(@parsed_uri.host)
+      LOCALHOST_ALIASES.include?(@parsed_uri.host)
     end
 
     def host
@@ -92,23 +100,18 @@ module Typhoeus
       end
     end
 
+    def host_domain
+      @parsed_uri.host
+    end
+
     def headers
       @headers["User-Agent"] = @user_agent
       @headers
     end
 
     def params_string
-      params.keys.sort { |a, b| a.to_s <=> b.to_s }.collect do |k|
-        value = params[k]
-        if value.is_a? Hash
-          value.keys.collect {|sk| Typhoeus::Utils.escape("#{k}[#{sk}]") + "=" + Typhoeus::Utils.escape(value[sk].to_s)}
-        elsif value.is_a? Array
-          key = Typhoeus::Utils.escape(k.to_s)
-          value.collect { |v| "#{key}[]=#{Typhoeus::Utils.escape(v.to_s)}" }.join('&')
-        else
-          "#{Typhoeus::Utils.escape(k.to_s)}=#{Typhoeus::Utils.escape(params[k].to_s)}"
-        end
-      end.flatten.join("&")
+      traversal = Typhoeus::Utils.traverse_params_hash(params)
+      Typhoeus::Utils.traversal_to_param_string(traversal)
     end
 
     def on_complete(&block)

data/lib/typhoeus/response.rb

@@ -4,11 +4,17 @@ module Typhoeus
     attr_reader :code, :headers, :body, :time,
                 :requested_url, :requested_remote_method,
                 :requested_http_method, :start_time,
-                :effective_url
+                :effective_url, :start_transfer_time,
+                :app_connect_time, :pretransfer_time,
+                :connect_time, :name_lookup_time,
+                :curl_return_code, :curl_error_message
+
     attr_writer :headers_hash
 
     def initialize(params = {})
       @code                  = params[:code]
+      @curl_return_code      = params[:curl_return_code]
+      @curl_error_message    = params[:curl_error_message]
       @status_message        = params[:status_message]
       @http_version          = params[:http_version]
       @headers               = params[:headers] || ''
@@ -17,6 +23,11 @@ module Typhoeus
       @requested_url         = params[:requested_url]
       @requested_http_method = params[:requested_http_method]
       @start_time            = params[:start_time]
+      @start_transfer_time   = params[:start_transfer_time]
+      @app_connect_time      = params[:app_connect_time]
+      @pretransfer_time      = params[:pretransfer_time]
+      @connect_time          = params[:connect_time]
+      @name_lookup_time      = params[:name_lookup_time]
       @request               = params[:request]
       @effective_url         = params[:effective_url]
       @mock                  = params[:mock] || false  # default
@@ -67,6 +78,10 @@ module Typhoeus
       @code != 304
     end
 
+    def timed_out?
+      curl_return_code == 28
+    end
+
     private
 
       def first_header_line

data/lib/typhoeus/utils.rb

@@ -8,6 +8,44 @@ module Typhoeus
     end
     module_function :escape
 
+    # Params are NOT escaped.
+    def traverse_params_hash(hash, result = nil, current_key = nil)
+      result ||= { :files => [], :params => [] }
+
+      hash.keys.sort { |a, b| a.to_s <=> b.to_s }.collect do |key|
+        new_key = (current_key ? "#{current_key}[#{key}]" : key).to_s
+        case hash[key]
+        when Hash
+          traverse_params_hash(hash[key], result, new_key)
+        when Array
+          array_key = "#{new_key}[]"
+          hash[key].each do |v|
+            result[:params] << [array_key, v.to_s]
+          end
+        when File
+          filename = File.basename(hash[key].path)
+          types = MIME::Types.type_for(filename)
+          result[:files] << [
+            new_key,
+            filename,
+            types.empty? ? 'application/octet-stream' : types[0].to_s,
+            File.expand_path(hash[key].path)
+          ]
+        else
+          result[:params] << [new_key, hash[key].to_s]
+        end
+      end
+      result
+    end
+    module_function :traverse_params_hash
+
+    def traversal_to_param_string(traversal, escape = true)
+      traversal[:params].collect { |param|
+        "#{Typhoeus::Utils.escape(param[0])}=#{Typhoeus::Utils.escape(param[1])}"
+      }.join('&')
+    end
+    module_function :traversal_to_param_string
+
     # Return the bytesize of String; uses String#size under Ruby 1.8 and
     # String#bytesize under 1.9.
     if ''.respond_to?(:bytesize)

data/lib/typhoeus.rb

@@ -5,6 +5,7 @@ require 'digest/sha2'
 require 'typhoeus/utils'
 require 'typhoeus/normalized_header_hash'
 require 'typhoeus/easy'
+require 'typhoeus/form'
 require 'typhoeus/multi'
 require 'typhoeus/native'
 require 'typhoeus/filter'

data/spec/fixtures/placeholder.txt

@@ -0,0 +1 @@
+This file is used to test uploading.

data/spec/servers/app.rb

@@ -5,6 +5,16 @@ require 'json'
 require 'zlib'
 
 @@fail_count = 0
+
+post '/file' do
+  {
+    'content-type' => params[:file][:type],
+    'filename' => params[:file][:filename],
+    'content' => params[:file][:tempfile].read,
+    'request-content-type' => request.env['CONTENT_TYPE']
+  }.to_json
+end
+
 get '/fail/:number' do
   if @@fail_count >= params[:number].to_i
     "ok"

data/spec/spec.opts

@@ -1,2 +1,3 @@
 --diff
 --color
+--backtrace

data/spec/spec_helper.rb

@@ -9,3 +9,6 @@ path = File.expand_path(File.dirname(__FILE__) + "/../lib/")
 $LOAD_PATH.unshift(path) unless $LOAD_PATH.include?(path)
 
 require path + '/typhoeus'
+
+Spec::Runner.configure do |config|
+end