typhoeus 0.2.4 → 0.3.2

data/CHANGELOG.markdown

@@ -1,3 +1,25 @@
+0.3.0
+-----
+* Fix array params to be consistent with HTTP spec [gridaphobe]
+* traversal_to_params_hash should use the escape option [itsmeduncan]
+* Fix > 1024 open file descriptors [mschulkind]
+* Fixed a bug with internally queued requests being dropped [mschulkind]
+* Use gemspec in bundler to avoid duplication [mschulkind]
+* Run internally queued requests in FIFO order [mschulkind]
+* Moved Typhoeus::VERSION to a separate file, to fix rake build_native [mschulkind]
+* Fixed problems related to put requests with empty bodies [skaes, GH-84]
+* Added CURLOPT_INTERFACE option via Request#interface=. [spiegela]
+* Added Tempfile support to Form#process! [richievos]
+* Hydra won't forget to accept gzip/deflate encoding [codesnik]
+* Accept and convert strings to integers in Typhoeus::Request#initialize for timeout/cache_timeout/connect_timeout values when using ruby 1.9.x. [djnawara]
+* Added interface for registering stub finders [myronmarston]
+* Fixed header stubbing [myronmarston]
+* Added PKCS12 support [jodell]
+* Make a request with handlers marshallable [bernerdschaefer]
+* Upgraded to RSpec 2 [bernerdschaefer]
+* Fix HTTP status edge-case [balexis]
+* Expose primary_ip to easy object [balexis]
+
 0.2.4
 -----
 * Fix form POSTs to only use multipart for file uploads, otherwise use application/x-www-form-urlencoded [dbalatero]

data/Gemfile

@@ -1,11 +1,3 @@
 source :rubygems
 
-gem "mime-types"
-
-group :test do
-  gem 'rspec', '1.3.1'
-  gem 'jeweler'
-  gem 'json'
-  gem 'sinatra'
-  gem 'diff-lcs'
-end
+gemspec

data/Gemfile.lock

@@ -1,32 +1,35 @@
+PATH
+  remote: .
+  specs:
+    typhoeus (0.3.0)
+      mime-types
+
 GEM
   remote: http://rubygems.org/
   specs:
     diff-lcs (1.1.2)
-    gemcutter (0.6.1)
-    git (1.2.5)
-    jeweler (1.4.0)
-      gemcutter (>= 0.1.0)
-      git (>= 1.2.5)
-      rubyforge (>= 2.0.0)
-    json (1.4.6)
-    json_pure (1.4.6)
+    json (1.5.3)
     mime-types (1.16)
-    rack (1.2.1)
-    rspec (1.3.1)
-    rubyforge (2.0.4)
-      json_pure (>= 1.1.7)
-    sinatra (1.1.0)
+    rack (1.3.0)
+    rspec (2.6.0)
+      rspec-core (~> 2.6.0)
+      rspec-expectations (~> 2.6.0)
+      rspec-mocks (~> 2.6.0)
+    rspec-core (2.6.4)
+    rspec-expectations (2.6.0)
+      diff-lcs (~> 1.1.2)
+    rspec-mocks (2.6.0)
+    sinatra (1.2.6)
       rack (~> 1.1)
-      tilt (~> 1.1)
-    tilt (1.1)
+      tilt (< 2.0, >= 1.2.2)
+    tilt (1.3.2)
 
 PLATFORMS
   ruby
 
 DEPENDENCIES
   diff-lcs
-  jeweler
   json
-  mime-types
-  rspec (= 1.3.1)
+  rspec (~> 2.6)
   sinatra
+  typhoeus!

data/Rakefile

@@ -1,8 +1,6 @@
 $LOAD_PATH.unshift(File.dirname(__FILE__))
 
-require "spec"
-require "spec/rake/spectask"
-require 'lib/typhoeus'
+require "lib/typhoeus/version"
 
 begin
   require 'jeweler'
@@ -14,7 +12,7 @@ begin
     gemspec.homepage = "http://github.com/dbalatero/typhoeus"
     gemspec.authors = ["Paul Dix", "David Balatero"]
     gemspec.add_dependency "mime-types"
-    gemspec.add_development_dependency "rspec"
+    gemspec.add_development_dependency "rspec", "~> 2.6"
     gemspec.add_development_dependency "jeweler"
     gemspec.add_development_dependency "diff-lcs"
     gemspec.add_development_dependency "sinatra"
@@ -26,9 +24,8 @@ rescue LoadError
   puts "Jeweler not available. Install it with: gem install jeweler"
 end
 
-Spec::Rake::SpecTask.new do |t|
-  t.spec_opts = ['--options', "\"#{File.dirname(__FILE__)}/spec/spec.opts\""]
-  t.spec_files = FileList['spec/**/*_spec.rb']
+require 'rspec/core/rake_task'
+RSpec::Core::RakeTask.new do |t|
 end
 
 task :install do
@@ -37,5 +34,20 @@ task :install do
   puts `gem install typhoeus-#{Typhoeus::VERSION}.gem`
 end
 
+desc "Builds the native code"
+task :build_native do
+  system("cd ext/typhoeus && ruby extconf.rb && make")
+end
+
+desc "Start up the test servers"
+task :start_test_servers do
+  puts "Starting 3 test servers"
+  (3000..3002).map do |port|
+    Thread.new do
+      system("ruby spec/servers/app.rb -p #{port}")
+    end
+  end.each(&:join)
+end
+
 desc "Run all the tests"
 task :default => :spec

data/ext/typhoeus/typhoeus_multi.c

@@ -1,4 +1,10 @@
+/* Make sure select() works with >1024 file handles open. */
+#include <sys/types.h>
+#undef __FD_SETSIZE
+#define __FD_SETSIZE 524288 
+
 #include <typhoeus_multi.h>
+#include <errno.h>
 
 static void multi_read_info(VALUE self, CURLM *multi_handle);
 
@@ -158,7 +164,7 @@ static VALUE multi_perform(VALUE self) {
 
     rc = rb_thread_select(maxfd+1, &fdread, &fdwrite, &fdexcep, &tv);
     if (rc < 0) {
-      rb_raise(rb_eRuntimeError, "error on thread select");
+      rb_raise(rb_eRuntimeError, "error on thread select: %d", errno);
     }
     rb_curl_multi_run( self, curl_multi->multi, &(curl_multi->running) );
 

data/lib/typhoeus.rb

@@ -16,10 +16,9 @@ require 'typhoeus/response'
 require 'typhoeus/request'
 require 'typhoeus/hydra'
 require 'typhoeus/hydra_mock'
+require 'typhoeus/version'
 
 module Typhoeus
-  VERSION = File.read(File.dirname(__FILE__) + "/../VERSION").chomp
-
   def self.easy_object_pool
     @easy_objects ||= []
   end

data/lib/typhoeus/easy.rb

@@ -20,6 +20,7 @@ module Typhoeus
       # [Only works on unix-style/SIGALRM operating systems. IOW, does
       # not work on Windows.
       :CURLOPT_CONNECTTIMEOUT_MS  => 156,
+      :CURLOPT_INTERFACE      => 10000 + 62,
       :CURLOPT_NOSIGNAL       => 99,
       :CURLOPT_HTTPHEADER     => 10023,
       :CURLOPT_FOLLOWLOCATION => 52,
@@ -32,6 +33,7 @@ module Typhoeus
       :CURLOPT_PROXYTYPE      => 101,
       :CURLOPT_PROXYAUTH      => 111,
       :CURLOPT_VERIFYPEER     => 64,
+      :CURLOPT_VERIFYHOST    => 81,
       :CURLOPT_NOBODY         => 44,
       :CURLOPT_ENCODING       => 10000 + 102,
       :CURLOPT_SSLCERT        => 10025,
@@ -47,6 +49,7 @@ module Typhoeus
       :CURLINFO_TOTAL_TIME         => 3145731,
       :CURLINFO_HTTPAUTH_AVAIL     => 0x200000 + 23,
       :CURLINFO_EFFECTIVE_URL      => 0x100000 + 1,
+      :CURLINFO_PRIMARY_IP         => 0x100000 + 32,
       :CURLINFO_NAMELOOKUP_TIME    => 0x300000 + 4,
       :CURLINFO_CONNECT_TIME       => 0x300000 + 5,
       :CURLINFO_PRETRANSFER_TIME   => 0x300000 + 6,
@@ -75,6 +78,10 @@ module Typhoeus
       @method = :get
       @headers = {}
 
+      set_defaults
+    end
+
+    def set_defaults
       # Enable encoding/compression support
       set_option(OPTION_VALUES[:CURLOPT_ENCODING], '')
     end
@@ -82,6 +89,11 @@ module Typhoeus
     def headers=(hash)
       @headers = hash
     end
+    
+    def interface=(interface)
+      @interface = interface
+      set_option(OPTION_VALUES[:CURLOPT_INTERFACE], interface)
+    end
 
     def proxy=(proxy)
       set_option(OPTION_VALUES[:CURLOPT_PROXY], proxy[:server])
@@ -133,6 +145,10 @@ module Typhoeus
     def effective_url
       get_info_string(INFO_VALUES[:CURLINFO_EFFECTIVE_URL])
     end
+    
+    def primary_ip
+      get_info_string(INFO_VALUES[:CURLINFO_PRIMARY_IP])
+    end
 
     def response_code
       get_info_long(INFO_VALUES[:CURLINFO_RESPONSE_CODE])
@@ -173,9 +189,7 @@ module Typhoeus
     def request_body=(request_body)
       @request_body = request_body
       if @method == :put
-        easy_set_request_body(@request_body)
-        headers["Transfer-Encoding"] = ""
-        headers["Expect"] = ""
+        easy_set_request_body(@request_body.to_s)
       else
         self.post_data = request_body
       end
@@ -194,6 +208,10 @@ module Typhoeus
       set_option(OPTION_VALUES[:CURLOPT_VERIFYPEER], 0)
     end
 
+    def disable_ssl_host_verification
+      set_option(OPTION_VALUES[:CURLOPT_VERIFYHOST], 0)
+    end
+
     def method=(method)
       @method = method
       if method == :get
@@ -203,7 +221,7 @@ module Typhoeus
         self.post_data = ""
       elsif method == :put
         set_option(OPTION_VALUES[:CURLOPT_UPLOAD], 1)
-        self.request_body = "" unless @request_body
+        self.request_body = @request_body.to_s
       elsif method == :head
         set_option(OPTION_VALUES[:CURLOPT_NOBODY], 1)
       else
@@ -246,7 +264,7 @@ module Typhoeus
     # Set SSL certificate type
     # " The string should be the format of your certificate. Supported formats are "PEM" and "DER" "
     def ssl_cert_type=(cert_type)
-      raise "Invalid ssl cert type : '#{cert_type}'..." if cert_type and !%w(PEM DER).include?(cert_type)
+      raise "Invalid ssl cert type : '#{cert_type}'..." if cert_type and !%w(PEM DER p12).include?(cert_type)
       set_option(OPTION_VALUES[:CURLOPT_SSLCERTTYPE], cert_type)
     end
 
@@ -344,7 +362,9 @@ module Typhoeus
       @response_code = 0
       @response_header = ""
       @response_body = ""
+      @request_body = ""
       easy_reset()
+      set_defaults
     end
 
     def get_info_string(option)

data/lib/typhoeus/hydra.rb

@@ -125,6 +125,7 @@ module Typhoeus
         val = @cache_getter.call(request)
         if val
           @retrieved_from_cache[request.url] = val
+          queue_next
           handle_request(request, val, false)
         else
           @multi.add(get_easy_object(request))
@@ -165,9 +166,11 @@ module Typhoeus
       easy.request_body = request.body    if request.body
       easy.timeout      = request.timeout if request.timeout
       easy.connect_timeout = request.connect_timeout if request.connect_timeout
+      easy.interface       = request.interface if request.interface
       easy.follow_location = request.follow_location if request.follow_location
       easy.max_redirects = request.max_redirects if request.max_redirects
       easy.disable_ssl_peer_verification if request.disable_ssl_peer_verification
+      easy.disable_ssl_host_verification if request.disable_ssl_host_verification
       easy.ssl_cert         = request.ssl_cert
       easy.ssl_cert_type    = request.ssl_cert_type
       easy.ssl_key          = request.ssl_key
@@ -194,7 +197,7 @@ module Typhoeus
 
     def queue_next
       @running_requests -= 1
-      queue(@queued_requests.pop) unless @queued_requests.empty?
+      queue(@queued_requests.shift) unless @queued_requests.empty?
     end
     private :queue_next
 
@@ -236,6 +239,7 @@ module Typhoeus
                    :connect_time        => easy.connect_time,
                    :name_lookup_time    => easy.name_lookup_time,
                    :effective_url       => easy.effective_url,
+                   :primary_ip          => easy.primary_ip,
                    :curl_return_code => easy.curl_return_code,
                    :curl_error_message => easy.curl_error_message,
                    :request             => request)

data/lib/typhoeus/hydra/stubbing.rb

@@ -11,10 +11,26 @@ module Typhoeus
           self.stubs = []
         end
 
+        def register_stub_finder(&block)
+          stub_finders << block
+        end
+
         def find_stub_from_request(request)
+          stub_finders.each do |finder|
+            if response = finder.call(request)
+              mock = HydraMock.new(/.*/, :any)
+              mock.and_return(response)
+              return mock
+            end
+          end
+
           stubs.detect { |stub| stub.matches?(request) }
         end
 
+        def stub_finders
+          @stub_finders ||= []
+        end
+
         def self.extended(base)
           class << base
             attr_accessor :stubs

data/lib/typhoeus/request.rb

@@ -7,7 +7,7 @@ module Typhoeus
     attr_accessor :method, :params, :body, :connect_timeout, :timeout,
                   :user_agent, :response, :cache_timeout, :follow_location,
                   :max_redirects, :proxy, :proxy_username,:proxy_password,
-                  :disable_ssl_peer_verification,
+                  :disable_ssl_peer_verification, :disable_ssl_host_verification, :interface,
                   :ssl_cert, :ssl_cert_type, :ssl_key, :ssl_key_type,
                   :ssl_key_password, :ssl_cacert, :ssl_capath, :verbose,
                   :username, :password, :auth_method, :user_agent,
@@ -22,6 +22,7 @@ module Typhoeus
     # ** +:params+  : params as a Hash
     # ** +:body+
     # ** +:timeout+ : timeout (ms)
+    # ** +:interface+ : interface or ip address (string)
     # ** +:connect_timeout+ : connect timeout (ms)
     # ** +:headers+  : headers as Hash
     # ** +:user_agent+ : user agent (string)
@@ -30,6 +31,7 @@ module Typhoeus
     # ** +:max_redirects
     # ** +:proxy
     # ** +:disable_ssl_peer_verification
+    # ** +:disable_ssl_host_verification
     # ** +:ssl_cert
     # ** +:ssl_cert_type
     # ** +:ssl_key
@@ -46,11 +48,12 @@ module Typhoeus
       @method           = options[:method] || :get
       @params           = options[:params]
       @body             = options[:body]
-      @timeout          = options[:timeout]
-      @connect_timeout  = options[:connect_timeout]
+      @timeout          = safe_to_i(options[:timeout])
+      @connect_timeout  = safe_to_i(options[:connect_timeout])
+      @interface        = options[:interface]
       @headers          = options[:headers] || {}
       @user_agent       = options[:user_agent] || Typhoeus::USER_AGENT
-      @cache_timeout    = options[:cache_timeout]
+      @cache_timeout    = safe_to_i(options[:cache_timeout])
       @follow_location  = options[:follow_location]
       @max_redirects    = options[:max_redirects]
       @proxy            = options[:proxy]
@@ -59,6 +62,7 @@ module Typhoeus
       @proxy_password   = options[:proxy_password]
       @proxy_auth_method = options[:proxy_auth_method]
       @disable_ssl_peer_verification = options[:disable_ssl_peer_verification]
+      @disable_ssl_host_verification = options[:disable_ssl_host_verification]
       @ssl_cert         = options[:ssl_cert]
       @ssl_cert_type    = options[:ssl_cert_type]
       @ssl_key          = options[:ssl_key]
@@ -197,5 +201,29 @@ module Typhoeus
     def self.head(url, params = {})
       run(url, params.merge(:method => :head))
     end
+
+  protected
+
+    # Return the important data needed to serialize this Request, except the
+    # `on_complete` and `after_complete` handlers, since they cannot be
+    # marshalled.
+    def marshal_dump
+      (instance_variables - ['@on_complete', '@after_complete', :@on_complete, :@after_complete]).map do |name|
+        [name, instance_variable_get(name)]
+      end
+    end
+
+    def marshal_load(attributes)
+      attributes.each { |name, value| instance_variable_set(name, value) }
+    end
+
+  private
+
+    def safe_to_i(value)
+      return value if value.is_a?(Fixnum)
+      return nil if value.nil?
+      return nil if value.respond_to?(:empty?) && value.empty?
+      value.to_i
+    end
   end
 end

data/lib/typhoeus/response.rb

@@ -7,7 +7,8 @@ module Typhoeus
                 :effective_url, :start_transfer_time,
                 :app_connect_time, :pretransfer_time,
                 :connect_time, :name_lookup_time,
-                :curl_return_code, :curl_error_message
+                :curl_return_code, :curl_error_message,
+                :primary_ip
 
     attr_writer :headers_hash
 
@@ -17,7 +18,7 @@ module Typhoeus
       @curl_error_message    = params[:curl_error_message]
       @status_message        = params[:status_message]
       @http_version          = params[:http_version]
-      @headers               = params[:headers] || ''
+      @headers               = params[:headers]
       @body                  = params[:body]
       @time                  = params[:time]
       @requested_url         = params[:requested_url]
@@ -30,6 +31,7 @@ module Typhoeus
       @name_lookup_time      = params[:name_lookup_time]
       @request               = params[:request]
       @effective_url         = params[:effective_url]
+      @primary_ip            = params[:primary_ip]
       @mock                  = params[:mock] || false  # default
       @headers_hash          = NormalizedHeaderHash.new(params[:headers_hash]) if params[:headers_hash]
     end
@@ -39,6 +41,10 @@ module Typhoeus
       @mock
     end
 
+    def headers
+      @headers ||= @headers_hash ? construct_header_string : ''
+    end
+
     def headers_hash
       @headers_hash ||= begin
         headers.split("\n").map {|o| o.strip}.inject(Typhoeus::NormalizedHeaderHash.new) do |hash, o|
@@ -62,8 +68,20 @@ module Typhoeus
     end
 
     def status_message
-      # http://rubular.com/r/eAr1oVYsVa
-      @status_message ||= first_header_line ? first_header_line[/\d{3} (.*)$/, 1].chomp : nil
+      return @status_message if @status_message != nil
+
+      # HTTP servers can choose not to include the explanation to HTTP codes. The RFC
+      # states this (http://www.w3.org/Protocols/rfc2616/rfc2616-sec10.html#sec10.4):
+      # Except when responding to a HEAD request, the server SHOULD include an entity containing
+      # an explanation of the error situation [...]
+      # This means 'HTTP/1.1 404' is as valid as 'HTTP/1.1 404 Not Found' and we have to handle it.
+
+      # Regexp doc: http://rubular.com/r/eAr1oVYsVa
+      if first_header_line != nil and first_header_line[/\d{3} (.*)$/, 1] != nil
+        @status_message = first_header_line[/\d{3} (.*)$/, 1].chomp
+      else
+        @status_message = nil
+      end
     end
 
     def http_version
@@ -85,7 +103,20 @@ module Typhoeus
     private
 
       def first_header_line
-        @first_header_line ||= headers.split("\n").first
+        @first_header_line ||= @headers.to_s.split("\n").first
+      end
+
+      def construct_header_string
+        lines = ["HTTP/#{http_version} #{code} #{status_message}"]
+
+        @headers_hash.each do |key, values|
+          [values].flatten.each do |value|
+            lines << "#{key}: #{value}"
+          end
+        end
+
+        lines << '' << ''
+        lines.join("\r\n")
       end
   end
 end