typhoeus 0.2.0 → 0.2.1

data/lib/typhoeus/hydra/connect_options.rb

@@ -12,9 +12,10 @@ module Typhoeus
       #
       # @raises NetConnectNotAllowedError
       def check_allow_net_connect!(request)
-        if !Typhoeus::Hydra.allow_net_connect? and (!Typhoeus::Hydra.ignore_localhost? or !request.localhost?)
-          raise NetConnectNotAllowedError, "Real HTTP requests are not allowed. Unregistered request: #{request.inspect}"
-        end
+        return if Typhoeus::Hydra.allow_net_connect?
+        return if Typhoeus::Hydra.ignore_hosts.include?(request.host_domain)
+
+        raise NetConnectNotAllowedError, "Real HTTP requests are not allowed. Unregistered request: #{request.inspect}"
       end
       private :check_allow_net_connect!
 
@@ -39,7 +40,22 @@ module Typhoeus
         def ignore_localhost?
           ignore_localhost
         end
+
+        def ignore_hosts
+          @ignore_hosts ||= []
+
+          if ignore_localhost?
+            @ignore_hosts + Typhoeus::Request::LOCALHOST_ALIASES
+          else
+            @ignore_hosts
+          end
+        end
+
+        def ignore_hosts=(hosts)
+          @ignore_hosts = hosts
+        end
       end
     end
   end
 end
+

data/lib/typhoeus/multi.rb

@@ -3,16 +3,16 @@ module Typhoeus
     attr_reader :easy_handles
 
     def initialize
-      reset_easy_handles
+      @easy_handles = []
     end
 
     def remove(easy)
-      multi_remove_handle(easy)
+      multi_remove_handle(easy) if @easy_handles.include?(easy)
     end
 
     def add(easy)
+      raise "trying to add easy handle twice" if @easy_handles.include?(easy)
       easy.set_headers() if easy.headers.empty?
-      @easy_handles << easy
       multi_add_handle(easy)
     end
 
@@ -27,9 +27,11 @@ module Typhoeus
       multi_cleanup
     end
 
-    private
     def reset_easy_handles
-      @easy_handles = []
+      @easy_handles.dup.each do |easy|
+        multi_remove_handle(easy)
+        yield easy if block_given?
+      end
     end
   end
 end

data/lib/typhoeus/remote.rb

@@ -1,5 +1,5 @@
 module Typhoeus
-  USER_AGENT = "Typhoeus - http://github.com/pauldix/typhoeus/tree/master"
+  USER_AGENT = "Typhoeus - http://github.com/dbalatero/typhoeus/tree/master"
   
   def self.included(base)
     base.extend ClassMethods

data/lib/typhoeus/remote_proxy_object.rb

@@ -22,6 +22,8 @@ module Typhoeus
         unless @proxied_object
           Typhoeus.perform_easy_requests
           response = Response.new(:code => @easy.response_code,
+                                  :curl_return_code => @easy.curl_return_code,
+                                  :curl_error_message => @easy.curl_error_message,
                                   :headers => @easy.response_header,
                                   :body => @easy.response_body,
                                   :time => @easy.total_time_taken,

data/lib/typhoeus/request.rb

@@ -6,10 +6,12 @@ module Typhoeus
     attr_writer   :headers
     attr_accessor :method, :params, :body, :connect_timeout, :timeout,
                   :user_agent, :response, :cache_timeout, :follow_location,
-                  :max_redirects, :proxy, :disable_ssl_peer_verification,
+                  :max_redirects, :proxy, :proxy_username,:proxy_password,
+                  :disable_ssl_peer_verification,
                   :ssl_cert, :ssl_cert_type, :ssl_key, :ssl_key_type,
                   :ssl_key_password, :ssl_cacert, :ssl_capath, :verbose,
-                  :username, :password, :auth_method, :user_agent
+                  :username, :password, :auth_method, :user_agent,
+                  :proxy_auth_method, :proxy_type
 
     # Initialize a new Request
     #
@@ -52,6 +54,10 @@ module Typhoeus
       @follow_location  = options[:follow_location]
       @max_redirects    = options[:max_redirects]
       @proxy            = options[:proxy]
+      @proxy_type       = options[:proxy_type]
+      @proxy_username   = options[:proxy_username]
+      @proxy_password   = options[:proxy_password]
+      @proxy_auth_method = options[:proxy_auth_method]
       @disable_ssl_peer_verification = options[:disable_ssl_peer_verification]
       @ssl_cert         = options[:ssl_cert]
       @ssl_cert_type    = options[:ssl_cert_type]
@@ -78,8 +84,10 @@ module Typhoeus
       @handled_response = nil
     end
 
+    LOCALHOST_ALIASES = %w[ localhost 127.0.0.1 0.0.0.0 ]
+
     def localhost?
-      %(localhost 127.0.0.1 0.0.0.0).include?(@parsed_uri.host)
+      LOCALHOST_ALIASES.include?(@parsed_uri.host)
     end
 
     def host
@@ -92,6 +100,10 @@ module Typhoeus
       end
     end
 
+    def host_domain
+      @parsed_uri.host
+    end
+
     def headers
       @headers["User-Agent"] = @user_agent
       @headers

data/lib/typhoeus/response.rb

@@ -4,11 +4,17 @@ module Typhoeus
     attr_reader :code, :headers, :body, :time,
                 :requested_url, :requested_remote_method,
                 :requested_http_method, :start_time,
-                :effective_url
+                :effective_url, :start_transfer_time,
+                :app_connect_time, :pretransfer_time,
+                :connect_time, :name_lookup_time,
+                :curl_return_code, :curl_error_message
+
     attr_writer :headers_hash
 
     def initialize(params = {})
       @code                  = params[:code]
+      @curl_return_code      = params[:curl_return_code]
+      @curl_error_message    = params[:curl_error_message]
       @status_message        = params[:status_message]
       @http_version          = params[:http_version]
       @headers               = params[:headers] || ''
@@ -17,6 +23,11 @@ module Typhoeus
       @requested_url         = params[:requested_url]
       @requested_http_method = params[:requested_http_method]
       @start_time            = params[:start_time]
+      @start_transfer_time   = params[:start_transfer_time]
+      @app_connect_time      = params[:app_connect_time]
+      @pretransfer_time      = params[:pretransfer_time]
+      @connect_time          = params[:connect_time]
+      @name_lookup_time      = params[:name_lookup_time]
       @request               = params[:request]
       @effective_url         = params[:effective_url]
       @mock                  = params[:mock] || false  # default
@@ -67,6 +78,10 @@ module Typhoeus
       @code != 304
     end
 
+    def timed_out?
+      curl_return_code == 28
+    end
+
     private
 
       def first_header_line

data/spec/fixtures/placeholder.txt

@@ -0,0 +1 @@
+This file is used to test uploading.

data/spec/servers/app.rb

@@ -5,6 +5,15 @@ require 'json'
 require 'zlib'
 
 @@fail_count = 0
+
+post '/file' do
+  {
+    'content-type' => params[:file][:type],
+    'filename' => params[:file][:filename],
+    'content' => params[:file][:tempfile].read
+  }.to_json
+end
+
 get '/fail/:number' do
   if @@fail_count >= params[:number].to_i
     "ok"

data/spec/spec.opts

@@ -1,2 +1,3 @@
 --diff
 --color
+--backtrace

data/spec/spec_helper.rb

@@ -9,3 +9,6 @@ path = File.expand_path(File.dirname(__FILE__) + "/../lib/")
 $LOAD_PATH.unshift(path) unless $LOAD_PATH.include?(path)
 
 require path + '/typhoeus'
+
+Spec::Runner.configure do |config|
+end

data/spec/typhoeus/easy_spec.rb

@@ -1,4 +1,4 @@
-require File.dirname(__FILE__) + '/../spec_helper'
+require File.expand_path(File.dirname(__FILE__) + '/../spec_helper')
 
 describe Typhoeus::Easy do
   describe "#supports_zlib" do
@@ -17,6 +17,42 @@ describe Typhoeus::Easy do
     end
   end
 
+  describe "curl errors" do
+    it "should provide the CURLE_OPERATION_TIMEDOUT return code when a request times out" do
+      e = Typhoeus::Easy.new
+      e.url = "http://localhost:3001/?delay=1"
+      e.method = :get
+      e.timeout = 100
+      e.perform
+      e.curl_return_code.should == 28
+      e.curl_error_message.should == "Timeout was reached"
+      e.response_code.should == 0
+    end
+
+    it "should provide the CURLE_COULDNT_CONNECT return code when trying to connect to a non existent port" do
+      e = Typhoeus::Easy.new
+      e.url = "http://localhost:3999"
+      e.method = :get
+      e.connect_timeout = 100
+      e.perform
+      e.curl_return_code.should == 7
+      e.curl_error_message.should == "Couldn't connect to server"
+      e.response_code.should == 0
+    end
+
+    it "should not return an error message on a successful easy operation" do
+      easy = Typhoeus::Easy.new
+      easy.url    = "http://localhost:3002"
+      easy.method = :get
+      easy.curl_error_message.should == nil
+      easy.perform
+      easy.response_code.should == 200
+      easy.curl_return_code.should == 0
+      easy.curl_error_message.should == "No error"
+    end
+
+  end
+
   describe "options" do
     it "should not follow redirects if not instructed to" do
       e = Typhoeus::Easy.new
@@ -48,12 +84,11 @@ describe Typhoeus::Easy do
 
     it "should provide a timeout in milliseconds" do
       e = Typhoeus::Easy.new
-      e.url = "http://localhost:3001"
+      e.url = "http://localhost:3001/?delay=1"
       e.method = :get
-      e.timeout = 50
+      e.timeout = 10
       e.perform
-      # this doesn't work on a mac for some reason
-#      e.timed_out?.should == true
+      e.timed_out?.should == true
     end
 
     it "should allow the setting of the max redirects to follow" do
@@ -209,7 +244,21 @@ describe Typhoeus::Easy do
       easy.params = {:foo => "bar"}
       easy.perform
       easy.response_code.should == 200
-      easy.response_body.should include("foo=bar")
+      easy.response_body.should include("name=\\\"foo\\\"\\r\\n\\r\\nbar")
+    end
+
+    it "should handle a file upload" do
+      easy = Typhoeus::Easy.new
+      easy.url    = "http://localhost:3002/file"
+      easy.method = :post
+      easy.params = {:file => File.open(File.expand_path(File.dirname(__FILE__) + "/../fixtures/placeholder.txt"), "r")}
+      easy.perform
+      easy.response_code.should == 200
+      JSON.parse(easy.response_body).should == {
+        'content-type' => 'text/plain',
+        'filename' => 'placeholder.txt',
+        'content' => 'This file is used to test uploading.'
+      }
     end
   end
   

data/spec/typhoeus/filter_spec.rb

@@ -1,4 +1,4 @@
-require File.dirname(__FILE__) + '/../spec_helper'
+require File.expand_path(File.dirname(__FILE__) + '/../spec_helper')
 
 describe Typhoeus::Filter do
   it "should take a method name and optionally take options" do
@@ -32,4 +32,4 @@ describe Typhoeus::Filter do
       filter.apply_filter?(:foo).should be_false
     end
   end
-end
+end

data/spec/typhoeus/form_spec.rb

@@ -0,0 +1,106 @@
+require File.expand_path(File.dirname(__FILE__) + '/../spec_helper')
+
+describe Typhoeus::Form do
+  describe "#process!" do
+    it "should generate a valid form object" do
+      form = Typhoeus::Form.new({
+        :name => "John Smith",
+        :age => "29"
+      })
+      form.should_receive(:formadd_param).with("name", "John Smith")
+      form.should_receive(:formadd_param).with("age", "29")
+      form.process!
+    end
+
+    it "should handle params that are a hash" do
+      form = Typhoeus::Form.new({
+        :attributes => {
+          :eyes => "brown",
+          :hair => "green",
+          :teeth => "white"
+        },
+        :name => "John Smith",
+        :age => "29"
+      })
+      form.should_receive(:formadd_param).with("attributes[eyes]", "brown")
+      form.should_receive(:formadd_param).with("attributes[hair]", "green")
+      form.should_receive(:formadd_param).with("attributes[teeth]", "white")
+      form.should_receive(:formadd_param).with("name", "John Smith")
+      form.should_receive(:formadd_param).with("age", "29")
+      form.process!
+    end
+
+    it "should params that have mutliple values" do
+      form = Typhoeus::Form.new({
+        :colors => ["brown", "green", "white"],
+        :name => "John Smith",
+        :age => "29"
+      })
+      form.should_receive(:formadd_param).with("colors", "brown")
+      form.should_receive(:formadd_param).with("colors", "green")
+      form.should_receive(:formadd_param).with("colors", "white")
+      form.should_receive(:formadd_param).with("name", "John Smith")
+      form.should_receive(:formadd_param).with("age", "29")
+      form.process!
+    end
+
+    context "when a File object is a param" do
+      it "should handle one file" do
+        form = Typhoeus::Form.new(
+          :file => File.open(File.expand_path(File.dirname(__FILE__) + "/../fixtures/placeholder.txt"), "r")
+        )
+        form.should_receive(:formadd_file).with("file", "placeholder.txt", "text/plain", anything)
+        form.process!
+      end
+      it "should handle more than one file" do
+        form = Typhoeus::Form.new(
+          :text_file => File.open(File.expand_path(File.dirname(__FILE__) + "/../fixtures/placeholder.txt"), "r"),
+          :gif_file => File.open(File.expand_path(File.dirname(__FILE__) + "/../fixtures/placeholder.gif"), "r")
+        )
+        form.should_receive(:formadd_file).with("text_file", "placeholder.txt", "text/plain", anything)
+        form.should_receive(:formadd_file).with("gif_file", "placeholder.gif", "image/gif", anything)
+        form.process!
+      end
+      it "should default to 'application/octet-stream' if no content type can be determined" do
+        pending
+        form = Typhoeus::Form.new(
+          :file => File.open(File.expand_path(File.dirname(__FILE__) + "/../fixtures/placeholder.txt"), "r")
+        )
+        form.should_receive(:formadd_file).with("file", "placeholder.ukn", "application/octet-stream", anything)
+        form.process!
+      end
+    end
+  end
+
+  describe "#to_s" do
+    it "should generate a valid query string" do
+      form = Typhoeus::Form.new({
+        :name => "John Smith",
+        :age => "29"
+      })
+      form.to_s.should == "age=29&name=John+Smith"
+    end
+
+    it "should handle params that are a hash" do
+      form = Typhoeus::Form.new({
+        :attributes => {
+          :eyes => "brown",
+          :hair => "green",
+          :teeth => "white"
+        },
+        :name => "John Smith",
+        :age => "29"
+      })
+      form.to_s.should == "age=29&attributes%5Beyes%5D=brown&attributes%5Bhair%5D=green&attributes%5Bteeth%5D=white&name=John+Smith"
+    end
+
+    it "should params that have mutliple values" do
+      form = Typhoeus::Form.new({
+        :colors => ["brown", "green", "white"],
+        :name => "John Smith",
+        :age => "29"
+      })
+      form.to_s.should == "age=29&colors=brown&colors=green&colors=white&name=John+Smith"
+    end
+  end
+end

data/spec/typhoeus/hydra_mock_spec.rb

@@ -1,4 +1,4 @@
-require File.dirname(__FILE__) + "/../spec_helper"
+require File.expand_path(File.dirname(__FILE__) + "/../spec_helper")
 
 describe Typhoeus::HydraMock do
   it "should mark all responses as mocks" do

data/spec/typhoeus/hydra_spec.rb

@@ -1,4 +1,4 @@
-require File.dirname(__FILE__) + '/../spec_helper'
+require File.expand_path(File.dirname(__FILE__) + '/../spec_helper')
 
 # some of these tests assume that you have some local services running.
 # ruby spec/servers/app.rb -p 3000
@@ -46,6 +46,47 @@ describe Typhoeus::Hydra do
     second.response.body.should include("second")
   end
 
+  it "should store the curl return codes on the reponses" do
+    hydra  = Typhoeus::Hydra.new
+    first  = Typhoeus::Request.new("http://localhost:3001/?delay=1", :timeout => 100)
+    second = Typhoeus::Request.new("http://localhost:3999", :connect_timout => 100)
+    hydra.queue first
+    hydra.queue second
+    hydra.run
+    first.response.curl_return_code == 28
+    second.response.curl_return_code == 7
+  end
+
+  it "aborts a batch of requests" do
+    urls = [
+        'http://localhost:3000',
+        'http://localhost:3001',
+        'http://localhost:3002'
+    ]
+
+    # this will make testing easier
+    hydra     = Typhoeus::Hydra.new( :max_concurrency => 1 )
+    completed = 0
+
+    10.times {
+        |i|
+
+        req = Typhoeus::Request.new( urls[ i % urls.size], :params => { :cnt => i } )
+        req.on_complete {
+            |res|
+            completed += 1
+            hydra.abort if completed == 5
+        }
+
+        hydra.queue( req )
+    }
+
+    hydra.run
+
+    # technically this should be '== 6' but I don't trust it...
+    completed.should < 10
+  end
+
   it "has a cache_setter proc" do
     hydra = Typhoeus::Hydra.new
     hydra.cache_setter do |request|
@@ -390,37 +431,77 @@ describe Typhoeus::Hydra::ConnectOptions do
     @klass = Typhoeus::Hydra
   end
 
+  let!(:old_net_connect) { @klass.allow_net_connect }
+  let!(:old_ignore_localhost) { @klass.ignore_localhost }
+  let(:hydra) { @klass.new }
+
+  after(:each) do
+    @klass.allow_net_connect = old_net_connect
+    @klass.ignore_localhost = old_ignore_localhost
+  end
+
+  def request_for(host)
+    Typhoeus::Request.new("http://#{host}:3000")
+  end
+
   describe "#ignore_localhost" do
-    it "should allow localhost requests over the wire if true" do
-      begin
-        old_net_connect = @klass.allow_net_connect
-        old_ignore_localhost = @klass.ignore_localhost
+    context "when set to true" do
+      before(:each) { @klass.ignore_localhost = true }
+
+      [true, false].each do |val|
+        it "allows localhost requests when allow_net_connect is #{val}" do
+          @klass.allow_net_connect = val
+          expect { hydra.queue(request_for('localhost')) }.to_not raise_error
+        end
+      end
+    end
+
+    context "when set to false" do
+      before(:each) { @klass.ignore_localhost = false }
+
+      it "allows localhost requests when allow_net_connect is true" do
+        @klass.allow_net_connect = true
+        expect { hydra.queue(request_for('localhost')) }.to_not raise_error
+      end
+
+      it "does not allow localhost requests when allow_net_connect is false" do
         @klass.allow_net_connect = false
-        @klass.ignore_localhost = true
+        expect { hydra.queue(request_for('localhost')) }.to raise_error(Typhoeus::Hydra::NetConnectNotAllowedError)
+      end
+    end
+  end
 
-        req = Typhoeus::Request.new("http://localhost:3000")
-        hydra = @klass.new
+  describe "#ignore_hosts" do
+    context 'when allow_net_connect is set to false' do
+      before(:each) do
+        @klass.ignore_localhost = false
+        @klass.allow_net_connect = false
+      end
 
-        lambda {
-          hydra.queue(req)
-        }.should_not raise_error
-      ensure
-        @klass.allow_net_connect = old_net_connect
-        @klass.ignore_localhost = old_ignore_localhost
+      Typhoeus::Request::LOCALHOST_ALIASES.each do |disallowed_host|
+        ignore_hosts = Typhoeus::Request::LOCALHOST_ALIASES - [disallowed_host]
+
+        context "when set to #{ignore_hosts.join(' and ')}" do
+          before(:each) { @klass.ignore_hosts = ignore_hosts }
+
+          it "does not allow a request to #{disallowed_host}" do
+            expect { hydra.queue(request_for(disallowed_host)) }.to raise_error(Typhoeus::Hydra::NetConnectNotAllowedError)
+          end
+
+          ignore_hosts.each do |host|
+            it "allows a request to #{host}" do
+              expect { hydra.queue(request_for(host)) }.to_not raise_error
+            end
+          end
+        end
       end
     end
   end
 
   describe "#allow_net_connect" do
     it "should be settable" do
-      begin
-        # make sure to not mess up other specs.
-        old = @klass.allow_net_connect
-        @klass.allow_net_connect = true
-        @klass.allow_net_connect.should be_true
-      ensure
-        @klass.allow_net_connect = old
-      end
+      @klass.allow_net_connect = true
+      @klass.allow_net_connect.should be_true
     end
 
     it "should default to true" do
@@ -428,23 +509,12 @@ describe Typhoeus::Hydra::ConnectOptions do
     end
 
     it "should raise an error if we queue a request while its false" do
-      begin
-        request = Typhoeus::Request.new("http://localhost:3000")
-        old_net_connect = @klass.allow_net_connect
-        old_ignore_localhost = @klass.ignore_localhost
+      @klass.allow_net_connect = false
+      @klass.ignore_localhost = false
 
-        @klass.allow_net_connect = false
-        @klass.ignore_localhost = false
-
-        hydra = Typhoeus::Hydra.new
-
-        lambda {
-          hydra.queue(request)
-        }.should raise_error(Typhoeus::Hydra::NetConnectNotAllowedError)
-      ensure
-        @klass.allow_net_connect = old_net_connect
-        @klass.ignore_localhost = old_ignore_localhost
-      end
+      expect {
+        hydra.queue(request_for('example.com'))
+      }.to raise_error(Typhoeus::Hydra::NetConnectNotAllowedError)
     end
   end