typhoeus 0.2.0 → 0.2.1

data/ext/typhoeus/typhoeus_form.h

@@ -0,0 +1,13 @@
+#ifndef TYPHOEUS_FORM
+#define TYPHOEUS_FORM
+
+#include <native.h>
+
+typedef struct {
+  struct curl_httppost *first;
+  struct curl_httppost *last;
+} CurlForm;
+
+void init_typhoeus_form();
+
+#endif

data/ext/typhoeus/typhoeus_multi.c

@@ -16,12 +16,15 @@ static VALUE multi_add_handle(VALUE self, VALUE easy) {
 
   mcode = curl_multi_add_handle(curl_multi->multi, curl_easy->curl);
   if (mcode != CURLM_CALL_MULTI_PERFORM && mcode != CURLM_OK) {
-    rb_raise((VALUE)mcode, "An error occured adding the handle");
+    rb_raise(rb_eRuntimeError, "An error occured adding the handle: %d: %s", mcode, curl_multi_strerror(mcode));
   }
 
   curl_easy_setopt(curl_easy->curl, CURLOPT_PRIVATE, easy);
   curl_multi->active++;
 
+  VALUE easy_handles = rb_iv_get(self, "@easy_handles");
+  rb_ary_push(easy_handles, easy);
+
   if (mcode == CURLM_CALL_MULTI_PERFORM) {
     curl_multi_perform(curl_multi->multi, &(curl_multi->running));
   }
@@ -43,6 +46,9 @@ static VALUE multi_remove_handle(VALUE self, VALUE easy) {
   curl_multi->active--;
   curl_multi_remove_handle(curl_multi->multi, curl_easy->curl);
 
+  VALUE easy_handles = rb_iv_get(self, "@easy_handles");
+  rb_ary_delete(easy_handles, easy);
+
   return easy;
 }
 
@@ -65,36 +71,14 @@ static void multi_read_info(VALUE self, CURLM *multi_handle) {
     if (easy_handle) {
       ecode = curl_easy_getinfo(easy_handle, CURLINFO_PRIVATE, &easy);
       if (ecode != 0) {
-        rb_raise(ecode, "error getting easy object");
+        rb_raise(rb_eRuntimeError, "error getting easy object: %d: %s", ecode, curl_easy_strerror(ecode));
       }
 
       long response_code = -1;
       curl_easy_getinfo(easy_handle, CURLINFO_RESPONSE_CODE, &response_code);
 
-      // TODO: find out what the real problem is here and fix it.
-      // this next bit is a horrible hack. For some reason my tests against a local server on my laptop
-      // fail intermittently and return this result number. However, it will succeed if you try it a few
-      // more times. Also noteworthy is that this doens't happen when hitting an external server. WTF?!
-
-      // Sandofsky says:
-      // This is caused by OS X first attempting to resolve using IPV6.
-      // Hack solution: connect to yourself with 127.0.0.1, not localhost
-      // http://curl.haxx.se/mail/tracker-2009-09/0018.html
-      if (result == 7) {
-        VALUE max_retries = rb_funcall(easy, rb_intern("max_retries?"), 0);
-        if (max_retries != Qtrue) {
-          multi_remove_handle(self, easy);
-          multi_add_handle(self, easy);
-          CurlMulti *curl_multi;
-          Data_Get_Struct(self, CurlMulti, curl_multi);
-          curl_multi_perform(curl_multi->multi, &(curl_multi->running));
-
-          rb_funcall(easy, rb_intern("increment_retries"), 0);
-
-          continue;
-        }
-      }
       multi_remove_handle(self, easy);
+      rb_iv_set(easy, "@curl_return_code", INT2FIX(result));
 
       if (result != 0) {
         rb_funcall(easy, rb_intern("failure"), 0);
@@ -118,7 +102,7 @@ static void rb_curl_multi_run(VALUE self, CURLM *multi_handle, int *still_runnin
   } while (mcode == CURLM_CALL_MULTI_PERFORM);
 
   if (mcode != CURLM_OK) {
-    rb_raise((VALUE)mcode, "an error occured while running perform");
+    rb_raise(rb_eRuntimeError, "an error occured while running perform: %d: %s", mcode, curl_multi_strerror(mcode));
   }
 
   multi_read_info( self, multi_handle );
@@ -128,6 +112,8 @@ static VALUE fire_and_forget(VALUE self) {
   CurlMulti *curl_multi;
   Data_Get_Struct(self, CurlMulti, curl_multi);
   rb_curl_multi_run( self, curl_multi->multi, &(curl_multi->running) );
+
+  return Qnil;
 }
 
 static VALUE multi_perform(VALUE self) {
@@ -150,7 +136,7 @@ static VALUE multi_perform(VALUE self) {
     /* get the curl suggested time out */
     mcode = curl_multi_timeout(curl_multi->multi, &timeout);
     if (mcode != CURLM_OK) {
-      rb_raise((VALUE)mcode, "an error occured getting the timeout");
+      rb_raise(rb_eRuntimeError, "an error occured getting the timeout: %d: %s", mcode, curl_multi_strerror(mcode));
           }
 
     if (timeout == 0) { /* no delay */
@@ -167,7 +153,7 @@ static VALUE multi_perform(VALUE self) {
     /* load the fd sets from the multi handle */
     mcode = curl_multi_fdset(curl_multi->multi, &fdread, &fdwrite, &fdexcep, &maxfd);
     if (mcode != CURLM_OK) {
-      rb_raise((VALUE)mcode, "an error occured getting the fdset");
+      rb_raise(rb_eRuntimeError, "an error occured getting the fdset: %d: %s", mcode, curl_multi_strerror(mcode));
     }
 
     rc = rb_thread_select(maxfd+1, &fdread, &fdwrite, &fdexcep, &tv);
@@ -185,7 +171,7 @@ static VALUE active_handle_count(VALUE self) {
   CurlMulti *curl_multi;
   Data_Get_Struct(self, CurlMulti, curl_multi);
 
-  return INT2NUM(curl_multi->active);
+  return INT2FIX(curl_multi->active);
 }
 
 static VALUE multi_cleanup(VALUE self) {

data/lib/typhoeus.rb

@@ -5,6 +5,7 @@ require 'digest/sha2'
 require 'typhoeus/utils'
 require 'typhoeus/normalized_header_hash'
 require 'typhoeus/easy'
+require 'typhoeus/form'
 require 'typhoeus/multi'
 require 'typhoeus/native'
 require 'typhoeus/filter'

data/lib/typhoeus/easy.rb

@@ -1,6 +1,6 @@
 module Typhoeus
   class Easy
-    attr_reader :response_body, :response_header, :method, :headers, :url, :params
+    attr_reader :response_body, :response_header, :method, :headers, :url, :params, :curl_return_code
     attr_accessor :start_time
 
     # These integer codes are available in curl/curl.h
@@ -28,6 +28,9 @@ module Typhoeus
       :CURLOPT_USERPWD        => 10000 + 5,
       :CURLOPT_VERBOSE        => 41,
       :CURLOPT_PROXY          => 10004,
+      :CURLOPT_PROXYUSERPWD   => 10000 + 6,
+      :CURLOPT_PROXYTYPE      => 101,
+      :CURLOPT_PROXYAUTH      => 111,
       :CURLOPT_VERIFYPEER     => 64,
       :CURLOPT_NOBODY         => 44,
       :CURLOPT_ENCODING       => 10000 + 102,
@@ -37,13 +40,19 @@ module Typhoeus
       :CURLOPT_SSLKEYTYPE     => 10088,
       :CURLOPT_KEYPASSWD      => 10026,
       :CURLOPT_CAINFO         => 10065,
-      :CURLOPT_CAPATH         => 10097
+      :CURLOPT_CAPATH         => 10097,
     }
     INFO_VALUES = {
-      :CURLINFO_RESPONSE_CODE => 2097154,
-      :CURLINFO_TOTAL_TIME    => 3145731,
-      :CURLINFO_HTTPAUTH_AVAIL => 0x200000 + 23,
-      :CURLINFO_EFFECTIVE_URL => 0x100000 + 1
+      :CURLINFO_RESPONSE_CODE      => 2097154,
+      :CURLINFO_TOTAL_TIME         => 3145731,
+      :CURLINFO_HTTPAUTH_AVAIL     => 0x200000 + 23,
+      :CURLINFO_EFFECTIVE_URL      => 0x100000 + 1,
+      :CURLINFO_NAMELOOKUP_TIME    => 0x300000 + 4,
+      :CURLINFO_CONNECT_TIME       => 0x300000 + 5,
+      :CURLINFO_PRETRANSFER_TIME   => 0x300000 + 6,
+      :CURLINFO_STARTTRANSFER_TIME => 0x300000 + 17,
+      :CURLINFO_APPCONNECT_TIME    => 0x300000 + 33,
+
     }
     AUTH_TYPES = {
       :CURLAUTH_BASIC         => 1,
@@ -53,6 +62,14 @@ module Typhoeus
       :CURLAUTH_DIGEST_IE     => 16,
       :CURLAUTH_AUTO          => 16 | 8 | 4 | 2 | 1
     }
+    PROXY_TYPES = {
+      :CURLPROXY_HTTP         => 0,
+      :CURLPROXY_HTTP_1_0     => 1,
+      :CURLPROXY_SOCKS4       => 4,
+      :CURLPROXY_SOCKS5       => 5,
+      :CURLPROXY_SOCKS4A      => 6,
+    }
+
 
     def initialize
       @method = :get
@@ -67,7 +84,13 @@ module Typhoeus
     end
 
     def proxy=(proxy)
-      set_option(OPTION_VALUES[:CURLOPT_PROXY], proxy)
+      set_option(OPTION_VALUES[:CURLOPT_PROXY], proxy[:server])
+      set_option(OPTION_VALUES[:CURLOPT_PROXYTYPE], proxy[:type]) if proxy[:type]
+    end
+
+    def proxy_auth=(authinfo)
+      set_option(OPTION_VALUES[:CURLOPT_PROXYUSERPWD], "#{authinfo[:username]}:#{authinfo[:password]}")
+      set_option(OPTION_VALUES[:CURLOPT_PROXYAUTH], authinfo[:method]) if authinfo[:method]
     end
 
     def auth=(authinfo)
@@ -87,6 +110,26 @@ module Typhoeus
       get_info_double(INFO_VALUES[:CURLINFO_TOTAL_TIME])
     end
 
+    def start_transfer_time
+      get_info_double(INFO_VALUES[:CURLINFO_STARTTRANSFER_TIME])
+    end
+
+    def app_connect_time
+      get_info_double(INFO_VALUES[:CURLINFO_APPCONNECT_TIME])
+    end
+
+    def pretransfer_time
+      get_info_double(INFO_VALUES[:CURLINFO_PRETRANSFER_TIME])
+    end
+
+    def connect_time
+      get_info_double(INFO_VALUES[:CURLINFO_CONNECT_TIME])
+    end
+
+    def name_lookup_time
+      get_info_double(INFO_VALUES[:CURLINFO_NAMELOOKUP_TIME])
+    end
+
     def effective_url
       get_info_string(INFO_VALUES[:CURLINFO_EFFECTIVE_URL])
     end
@@ -106,7 +149,7 @@ module Typhoeus
     def max_redirects=(redirects)
       set_option(OPTION_VALUES[:CURLOPT_MAXREDIRS], redirects)
     end
-    
+
     def connect_timeout=(milliseconds)
       @connect_timeout = milliseconds
       set_option(OPTION_VALUES[:CURLOPT_NOSIGNAL], 1)
@@ -120,7 +163,7 @@ module Typhoeus
     end
 
     def timed_out?
-      @timeout && total_time_taken > @timeout && response_code == 0
+      curl_return_code == 28
     end
 
     def supports_zlib?
@@ -174,24 +217,18 @@ module Typhoeus
       set_option(OPTION_VALUES[:CURLOPT_COPYPOSTFIELDS], data)
     end
 
+    def params
+      @form.nil? ? {} : @form.params
+    end
+    
     def params=(params)
-      @params = params
-      params_string = params.keys.collect do |k|
-        value = params[k]
-        if value.is_a? Hash
-          value.keys.collect {|sk| Typhoeus::Utils.escape("#{k}[#{sk}]") + "=" + Typhoeus::Utils.escape(value[sk].to_s)}
-        elsif value.is_a? Array
-          key = Typhoeus::Utils.escape(k.to_s)
-          value.collect { |v| "#{key}=#{Typhoeus::Utils.escape(v.to_s)}" }.join('&')
-        else
-          "#{Typhoeus::Utils.escape(k.to_s)}=#{Typhoeus::Utils.escape(params[k].to_s)}"
-        end
-      end.flatten.join("&")
+      @form = Typhoeus::Form.new(params)
 
       if method == :post
-        self.post_data = params_string
+        @form.process!
+        set_option(OPTION_VALUES[:CURLOPT_HTTPPOST], @form)
       else
-        self.url = "#{url}?#{params_string}"
+        self.url = "#{url}?#{@form.to_s}"
       end
     end
 
@@ -205,7 +242,7 @@ module Typhoeus
     # Set SSL certificate type
     # " The string should be the format of your certificate. Supported formats are "PEM" and "DER" "
     def ssl_cert_type=(cert_type)
-      raise "Invalid ssl cert type : '#{cert_type}'..." if cert_type and !%w(PEM DER).include?(cert_type) 
+      raise "Invalid ssl cert type : '#{cert_type}'..." if cert_type and !%w(PEM DER).include?(cert_type)
       set_option(OPTION_VALUES[:CURLOPT_SSLCERTTYPE], cert_type)
     end
 
@@ -244,17 +281,20 @@ module Typhoeus
     end
 
     def set_option(option, value)
-      if value.class == String
-        easy_setopt_string(option, value)
-      elsif value
-        easy_setopt_long(option, value)
+      case value
+        when String
+          easy_setopt_string(option, value)
+        when Typhoeus::Form
+          easy_setopt_form(option, value)
+        else
+          easy_setopt_long(option, value) if value
       end
     end
 
     def perform
       set_headers()
       easy_perform()
-      resp_code = response_code() 
+      resp_code = response_code()
       if resp_code >= 200 && resp_code <= 299
         success
       else
@@ -283,7 +323,7 @@ module Typhoeus
       @success = block
     end
 
-    # gets called when finished and response code is 300-599
+    # gets called when finished and response code is 300-599 or curl returns an error code
     def failure
       @failure.call(self) if @failure
     end
@@ -296,25 +336,7 @@ module Typhoeus
       @failure = block
     end
 
-    def retries
-      @retries ||= 0
-    end
-
-    def increment_retries
-      @retries ||= 0
-      @retries += 1
-    end
-
-    def max_retries
-      @max_retries ||= 40
-    end
-
-    def max_retries?
-      retries >= max_retries
-    end
-
     def reset
-      @retries = 0
       @response_code = 0
       @response_header = ""
       @response_body = ""

data/lib/typhoeus/form.rb

@@ -0,0 +1,47 @@
+require 'mime/types'
+
+module Typhoeus
+  class Form
+    attr_accessor :params
+
+    def initialize(params = {})
+      @params = params
+    end
+
+    def process!
+      params.each do |key, value|
+        case value
+          when Hash
+            value.keys.each {|sub_key| formadd_param("#{key}[#{sub_key}]", value[sub_key].to_s)}
+          when Array
+            value.each {|v| formadd_param(key.to_s, v.to_s)}
+          when File
+            filename = File.basename(value.path)
+            types = MIME::Types.type_for(filename)
+            formadd_file(
+              key.to_s,
+              filename,
+              types.empty? ? 'application/octet-stream' : types[0].to_s,
+              File.expand_path(value.path)
+            )
+          else
+            formadd_param(key.to_s, value.to_s)
+        end
+      end
+    end
+
+    def to_s
+      params.keys.sort_by{|k|k.to_s}.collect do |k|
+        value = params[k]
+        if value.is_a? Hash
+          value.keys.sort_by{|sk|sk.to_s}.collect {|sk| Typhoeus::Utils.escape("#{k}[#{sk}]") + "=" + Typhoeus::Utils.escape(value[sk].to_s)}
+        elsif value.is_a? Array
+          key = Typhoeus::Utils.escape(k.to_s)
+          value.collect { |v| "#{key}=#{Typhoeus::Utils.escape(v.to_s)}" }.join('&')
+        else
+          "#{Typhoeus::Utils.escape(k.to_s)}=#{Typhoeus::Utils.escape(params[k].to_s)}"
+        end
+      end.flatten.join("&")
+    end
+  end
+end

data/lib/typhoeus/hydra.rb

@@ -32,6 +32,17 @@ module Typhoeus
       @hydra = val
     end
 
+    #
+    # Abort the run on a best-effort basis.
+    #
+    # It won't abort the current burst of @max_concurrency requests,
+    # however it won't fire the rest of the queued requests so the run
+    # will be aborted as soon as possible...
+    #
+    def abort
+      @queued_requests.clear
+    end
+
     def clear_cache_callbacks
       @cache_setter = nil
       @cache_getter = nil
@@ -82,8 +93,11 @@ module Typhoeus
       end
 
       @multi.perform
+    ensure
+      @multi.reset_easy_handles{|easy| release_easy_object(easy)}
       @memoized_requests = {}
       @retrieved_from_cache = {}
+      @running_requests = 0
     end
 
     def disable_memoization
@@ -131,6 +145,19 @@ module Typhoeus
         auth[:method] = Typhoeus::Easy::AUTH_TYPES["CURLAUTH_#{request.auth_method.to_s.upcase}".to_sym] if request.auth_method
         easy.auth = auth
       end
+
+      if request.proxy
+        proxy = { :server => request.proxy }
+        proxy[:type] = Typhoeus::Easy::PROXY_TYPES["CURLPROXY_#{request.proxy_type.to_s.upcase}".to_sym] if request.proxy_type
+        easy.proxy = proxy if request.proxy
+      end
+
+      if request.proxy_username || request.proxy_password
+        auth = { :username => request.proxy_username, :password => request.proxy_password }
+        auth[:method] = Typhoeus::Easy::AUTH_TYPES["CURLAUTH_#{request.proxy_auth_method.to_s.upcase}".to_sym] if request.proxy_auth_method
+        easy.proxy_auth = auth
+      end
+
       easy.url          = request.url
       easy.method       = request.method
       easy.params       = request.params  if request.method == :post && !request.params.nil?
@@ -140,7 +167,6 @@ module Typhoeus
       easy.connect_timeout = request.connect_timeout if request.connect_timeout
       easy.follow_location = request.follow_location if request.follow_location
       easy.max_redirects = request.max_redirects if request.max_redirects
-      easy.proxy = request.proxy if request.proxy
       easy.disable_ssl_peer_verification if request.disable_ssl_peer_verification
       easy.ssl_cert         = request.ssl_cert
       easy.ssl_cert_type    = request.ssl_cert_type
@@ -200,12 +226,19 @@ module Typhoeus
     private :handle_request
 
     def response_from_easy(easy, request)
-      Response.new(:code    => easy.response_code,
-                   :headers => easy.response_header,
-                   :body    => easy.response_body,
-                   :time    => easy.total_time_taken,
-                   :effective_url => easy.effective_url,
-                   :request => request)
+      Response.new(:code                => easy.response_code,
+                   :headers             => easy.response_header,
+                   :body                => easy.response_body,
+                   :time                => easy.total_time_taken,
+                   :start_transfer_time => easy.start_transfer_time,
+                   :app_connect_time    => easy.app_connect_time,
+                   :pretransfer_time    => easy.pretransfer_time,
+                   :connect_time        => easy.connect_time,
+                   :name_lookup_time    => easy.name_lookup_time,
+                   :effective_url       => easy.effective_url,
+                   :curl_return_code => easy.curl_return_code,
+                   :curl_error_message => easy.curl_error_message,
+                   :request             => request)
     end
     private :response_from_easy
   end