typhoeus 0.1.24 → 0.1.25

data/lib/typhoeus/hydra.rb

@@ -130,11 +130,20 @@ module Typhoeus
 
     def get_easy_object(request)
       @running_requests += 1
+
       easy = @easy_pool.pop || Easy.new
+
+      # Force gzip if we support it.
+      headers = {}
+      if easy.supports_zlib?
+        headers['Accept-Encoding'] = 'gzip'
+      end
+      headers.merge!(request.headers) if request.headers
+
       easy.url          = request.url
       easy.method       = request.method
       easy.params       = request.params  if request.method == :post && !request.params.nil?
-      easy.headers      = request.headers if request.headers
+      easy.headers      = headers
       easy.request_body = request.body    if request.body
       easy.timeout      = request.timeout if request.timeout
       easy.follow_location = request.follow_location if request.follow_location
@@ -192,6 +201,7 @@ module Typhoeus
                    :headers => easy.response_header,
                    :body    => easy.response_body,
                    :time    => easy.total_time_taken,
+                   :effective_url => easy.effective_url,
                    :request => request)
     end
     private :response_from_easy

data/lib/typhoeus/request.rb

@@ -42,7 +42,7 @@ module Typhoeus
     end
 
     def params_string
-      params.keys.sort.collect do |k|
+      params.keys.sort { |a, b| a.to_s <=> b.to_s }.collect do |k|
         value = params[k]
         if value.is_a? Hash
           value.keys.collect {|sk| Rack::Utils.escape("#{k}[#{sk}]") + "=" + Rack::Utils.escape(value[sk].to_s)}

data/lib/typhoeus/response.rb

@@ -3,8 +3,9 @@ module Typhoeus
     attr_accessor :request
     attr_reader :code, :headers, :body, :time,
                 :requested_url, :requested_remote_method,
-                :requested_http_method, :start_time
-    
+                :requested_http_method, :start_time,
+                :effective_url
+
     def initialize(params = {})
       @code                  = params[:code]
       @headers               = params[:headers]
@@ -14,8 +15,9 @@ module Typhoeus
       @requested_http_method = params[:requested_http_method]
       @start_time            = params[:start_time]
       @request               = params[:request]
+      @effective_url         = params[:effective_url]
     end
-    
+
     def headers_hash
       headers.split("\n").map {|o| o.strip}.inject({}) do |hash, o|
         if o.empty?
@@ -30,16 +32,16 @@ module Typhoeus
           else
             hash[key] = value
           end
-          
+
           hash
         end
       end
     end
-    
+
     def success?
       @code >= 200 && @code < 300
     end
-    
+
     def modified?
       @code != 304
     end

data/lib/typhoeus/service.rb

@@ -0,0 +1,20 @@
+module Typhoeus
+  class Service
+    def initialize(host, port)
+      @host = host
+      @port = port
+    end
+    
+    def get(resource, params)
+    end
+    
+    def put(resource, params)
+    end
+    
+    def post(resource, params)
+    end
+    
+    def delete(resource, params)
+    end
+  end
+end

data/profilers/valgrind.rb

@@ -0,0 +1,24 @@
+#!/usr/bin/env ruby
+# go to ext/typhoeus and run ruby extconf.rb && make before running
+# this.
+
+$LOAD_PATH.unshift(File.dirname(__FILE__) + "/../ext")
+require File.dirname(__FILE__) + "/../lib/typhoeus"
+
+klass = Class.new { include Typhoeus }
+
+loops = ENV['LOOPS'].to_i
+url = ARGV.first || (raise "requires URL!")
+
+loops.times do |i|
+  puts "On loop #{i}" if i % 10 == 0
+  results = []
+  5.times do
+    results << klass.get(url)
+  end
+ 
+  # fire requests
+  results[0].code
+end
+
+puts "Ran #{loops} loops on #{url}!"

data/spec/fixtures/result_set.xml

@@ -0,0 +1,60 @@
+<result_set>
+  <ttl>20</ttl>
+  <result>
+    <id>1</id>
+    <name>hello</name>
+    <description>
+      this is a long description for a text field of some kind.
+      this is a long description for a text field of some kind.
+      this is a long description for a text field of some kind.
+      this is a long description for a text field of some kind.
+      this is a long description for a text field of some kind.
+      this is a long description for a text field of some kind.
+      this is a long description for a text field of some kind.
+      this is a long description for a text field of some kind.
+      this is a long description for a text field of some kind.
+      this is a long description for a text field of some kind.
+      this is a long description for a text field of some kind.
+      this is a long description for a text field of some kind.
+      this is a long description for a text field of some kind.
+    </description>
+  </result>
+  <result>
+    <id>2</id>
+    <name>hello</name>
+    <description>
+      this is a long description for a text field of some kind.
+      this is a long description for a text field of some kind.
+      this is a long description for a text field of some kind.
+      this is a long description for a text field of some kind.
+      this is a long description for a text field of some kind.
+      this is a long description for a text field of some kind.
+      this is a long description for a text field of some kind.
+      this is a long description for a text field of some kind.
+      this is a long description for a text field of some kind.
+      this is a long description for a text field of some kind.
+      this is a long description for a text field of some kind.
+      this is a long description for a text field of some kind.
+      this is a long description for a text field of some kind.
+    </description>
+  </result>
+  <result>
+    <id>3</id>
+    <name>hello</name>
+    <description>
+      this is a long description for a text field of some kind.
+      this is a long description for a text field of some kind.
+      this is a long description for a text field of some kind.
+      this is a long description for a text field of some kind.
+      this is a long description for a text field of some kind.
+      this is a long description for a text field of some kind.
+      this is a long description for a text field of some kind.
+      this is a long description for a text field of some kind.
+      this is a long description for a text field of some kind.
+      this is a long description for a text field of some kind.
+      this is a long description for a text field of some kind.
+      this is a long description for a text field of some kind.
+      this is a long description for a text field of some kind.
+    </description>
+  </result>
+</result_set>

data/spec/typhoeus/easy_spec.rb

@@ -1,6 +1,22 @@
 require File.dirname(__FILE__) + '/../spec_helper'
 
-describe Typhoeus::Easy do  
+describe Typhoeus::Easy do
+  describe "#supports_zlib" do
+    before(:each) do
+      @easy = Typhoeus::Easy.new
+    end
+
+    it "should return true if the version string has zlib" do
+      @easy.stub!(:version).and_return("libcurl/7.20.0 OpenSSL/0.9.8l zlib/1.2.3 libidn/1.16")
+      @easy.supports_zlib?.should be_true
+    end
+
+    it "should return false if the version string doesn't have zlib" do
+      @easy.stub!(:version).and_return("libcurl/7.20.0 OpenSSL/0.9.8l libidn/1.16")
+      @easy.supports_zlib?.should be_false
+    end
+  end
+
   describe "options" do
     it "should not follow redirects if not instructed to" do
       e = Typhoeus::Easy.new

data/spec/typhoeus/hydra_spec.rb

@@ -0,0 +1,311 @@
+require File.dirname(__FILE__) + '/../spec_helper'
+
+# some of these tests assume that you have some local services running.
+# ruby spec/servers/app.rb -p 3000
+# ruby spec/servers/app.rb -p 3001
+# ruby spec/servers/app.rb -p 3002
+describe Typhoeus::Hydra do
+  before(:all) do
+    cache_class = Class.new do
+      def initialize
+        @cache = {}
+      end
+      def get(key)
+        @cache[key]
+      end
+      def set(key, object, timeout = 0)
+        @cache[key] = object
+      end
+    end
+    @cache = cache_class.new
+  end
+
+  it "has a singleton" do
+    Typhoeus::Hydra.hydra.should be_a Typhoeus::Hydra
+  end
+
+  it "has a setter for the singleton" do
+    Typhoeus::Hydra.hydra = :foo
+    Typhoeus::Hydra.hydra.should == :foo
+    Typhoeus::Hydra.hydra = Typhoeus::Hydra.new
+  end
+
+  context "#stub" do
+    before do
+      @hydra = Typhoeus::Hydra.new
+      @on_complete_handler_called = nil
+      @request  = Typhoeus::Request.new("http://localhost:3000/foo")
+      @request.on_complete do |response|
+        @on_complete_handler_called = true
+        response.code.should == 404
+        response.headers.should == "whatever"
+      end
+      @response = Typhoeus::Response.new(:code => 404, :headers => "whatever", :body => "not found", :time => 0.1)
+    end
+
+    it "stubs requests to a specific URI" do
+      @hydra.stub(:get, "http://localhost:3000/foo").and_return(@response)
+      @hydra.queue(@request)
+      @hydra.run
+      @on_complete_handler_called.should be_true
+      @response.request.should == @request
+    end
+
+    it "stubs requests to URIs matching a pattern" do
+      @hydra.stub(:get, /foo/).and_return(@response)
+      @hydra.queue(@request)
+      @hydra.run
+      @on_complete_handler_called.should be_true
+      @response.request.should == @request
+    end
+
+    it "can clear stubs" do
+      @hydra.clear_stubs
+    end
+    
+    it "clears out previously queued requests once they are called" do
+      @hydra.stub(:get, "asdf").and_return(@response)
+
+      call_count = 0
+      request = Typhoeus::Request.new("asdf")
+      request.on_complete do |response|
+        call_count += 1
+      end
+      @hydra.queue(request)
+      @hydra.run
+      call_count.should == 1
+      @hydra.run
+      call_count.should == 1
+    end
+    
+    it "calls stubs for requests that are queued up in the on_complete of a first stub" do
+      @hydra.stub(:get, "asdf").and_return(@response)
+      @hydra.stub(:get, "bar").and_return(@response)
+      
+      second_handler_called = false
+      request = Typhoeus::Request.new("asdf")
+      request.on_complete do |response|
+        r = Typhoeus::Request.new("bar")
+        r.on_complete do |res|
+          second_handler_called = true
+        end
+        @hydra.queue(r)
+      end
+      @hydra.queue(request)
+      @hydra.run
+      
+      second_handler_called.should be_true
+    end
+
+    it "matches a stub only when the HTTP method also matches"
+  end
+
+  it "queues a request" do
+    hydra = Typhoeus::Hydra.new
+    hydra.queue Typhoeus::Request.new("http://localhost:3000")
+  end
+
+  it "runs a batch of requests" do
+    hydra  = Typhoeus::Hydra.new
+    first  = Typhoeus::Request.new("http://localhost:3000/first")
+    second = Typhoeus::Request.new("http://localhost:3001/second")
+    hydra.queue first
+    hydra.queue second
+    hydra.run
+    first.response.body.should include("first")
+    second.response.body.should include("second")
+  end
+
+  it "has a cache_setter proc" do
+    hydra = Typhoeus::Hydra.new
+    hydra.cache_setter do |request|
+      # @cache.set(request.cache_key, request.response, request.cache_timeout)
+    end
+  end
+
+  it "has a cache_getter" do
+    hydra = Typhoeus::Hydra.new
+    hydra.cache_getter do |request|
+      # @cache.get(request.cache_key) rescue nil
+    end
+  end
+
+  it "memoizes GET reqeusts" do
+    hydra  = Typhoeus::Hydra.new
+    first  = Typhoeus::Request.new("http://localhost:3000/foo", :params => {:delay => 1})
+    second = Typhoeus::Request.new("http://localhost:3000/foo", :params => {:delay => 1})
+    hydra.queue first
+    hydra.queue second
+    start_time = Time.now
+    hydra.run
+    first.response.body.should include("foo")
+    first.handled_response.body.should include("foo")
+    first.response.should == second.response
+    first.handled_response.should == second.handled_response
+    (Time.now - start_time).should < 1.2 # if it had run twice it would be ~ 2 seconds
+  end
+
+  it "can turn off memoization for GET requests" do
+    hydra  = Typhoeus::Hydra.new
+    hydra.disable_memoization
+    first  = Typhoeus::Request.new("http://localhost:3000/foo")
+    second = Typhoeus::Request.new("http://localhost:3000/foo")
+    hydra.queue first
+    hydra.queue second
+    hydra.run
+    first.response.body.should include("foo")
+    first.response.object_id.should_not == second.response.object_id
+  end
+
+  it "pulls GETs from cache" do
+    hydra  = Typhoeus::Hydra.new
+    start_time = Time.now
+    hydra.cache_getter do |request|
+      @cache.get(request.cache_key) rescue nil
+    end
+    hydra.cache_setter do |request|
+      @cache.set(request.cache_key, request.response, request.cache_timeout)
+    end
+
+    first  = Typhoeus::Request.new("http://localhost:3000/foo", :params => {:delay => 1})
+    @cache.set(first.cache_key, :foo, 60)
+    hydra.queue first
+    hydra.run
+    (Time.now - start_time).should < 0.1
+    first.response.should == :foo
+  end
+
+  it "sets GET responses to cache when the request has a cache_timeout value" do
+    hydra  = Typhoeus::Hydra.new
+    hydra.cache_getter do |request|
+      @cache.get(request.cache_key) rescue nil
+    end
+    hydra.cache_setter do |request|
+      @cache.set(request.cache_key, request.response, request.cache_timeout)
+    end
+
+    first  = Typhoeus::Request.new("http://localhost:3000/first", :cache_timeout => 0)
+    second = Typhoeus::Request.new("http://localhost:3000/second")
+    hydra.queue first
+    hydra.queue second
+    hydra.run
+    first.response.body.should include("first")
+    @cache.get(first.cache_key).should == first.response
+    @cache.get(second.cache_key).should be_nil
+  end
+
+  it "has a global on_complete" do
+    foo = nil
+    hydra  = Typhoeus::Hydra.new
+    hydra.on_complete do |response|
+      foo = :called
+    end
+
+    first  = Typhoeus::Request.new("http://localhost:3000/first")
+    hydra.queue first
+    hydra.run
+    first.response.body.should include("first")
+    foo.should == :called
+  end
+
+  it "has a global on_complete setter" do
+    foo = nil
+    hydra  = Typhoeus::Hydra.new
+    proc = Proc.new {|response| foo = :called}
+    hydra.on_complete = proc
+
+    first  = Typhoeus::Request.new("http://localhost:3000/first")
+    hydra.queue first
+    hydra.run
+    first.response.body.should include("first")
+    foo.should == :called
+  end
+
+  it "should reuse connections from the pool for a host"
+
+  it "should queue up requests while others are running" do
+    hydra   = Typhoeus::Hydra.new
+
+    start_time = Time.now
+    @responses = []
+
+    request = Typhoeus::Request.new("http://localhost:3000/first", :params => {:delay => 1})
+    request.on_complete do |response|
+      @responses << response
+      response.body.should include("first")
+    end
+
+    request.after_complete do |object|
+      second_request = Typhoeus::Request.new("http://localhost:3001/second", :params => {:delay => 2})
+      second_request.on_complete do |response|
+        @responses << response
+        response.body.should include("second")
+      end
+      hydra.queue second_request
+    end
+    hydra.queue request
+
+    third_request = Typhoeus::Request.new("http://localhost:3002/third", :params => {:delay => 3})
+    third_request.on_complete do |response|
+      @responses << response
+      response.body.should include("third")
+    end
+    hydra.queue third_request
+
+    hydra.run
+    @responses.size.should == 3
+    (Time.now - start_time).should < 3.3
+  end
+
+  it "should fire and forget" do
+    # this test is totally hacky. I have no clue how to make it verify. I just look at the test servers
+    # to verify that stuff is running
+    hydra  = Typhoeus::Hydra.new
+    first  = Typhoeus::Request.new("http://localhost:3000/first?delay=1")
+    second = Typhoeus::Request.new("http://localhost:3001/second?delay=2")
+    hydra.queue first
+    hydra.queue second
+    hydra.fire_and_forget
+    third = Typhoeus::Request.new("http://localhost:3002/third?delay=3")
+    hydra.queue third
+    hydra.fire_and_forget
+    sleep 3 # have to do this or future tests may break.
+  end
+
+  it "should take the maximum number of concurrent requests as an argument" do
+    hydra = Typhoeus::Hydra.new(:max_concurrency => 2)
+    first  = Typhoeus::Request.new("http://localhost:3000/first?delay=1")
+    second = Typhoeus::Request.new("http://localhost:3001/second?delay=1")
+    third  = Typhoeus::Request.new("http://localhost:3002/third?delay=1")
+    hydra.queue first
+    hydra.queue second
+    hydra.queue third
+
+    start_time = Time.now
+    hydra.run
+    finish_time = Time.now
+
+    first.response.code.should == 200
+    second.response.code.should == 200
+    third.response.code.should == 200
+    (finish_time - start_time).should > 2.0
+  end
+
+  it "should respect the follow_location option when set on a request" do
+    hydra = Typhoeus::Hydra.new
+    request = Typhoeus::Request.new("http://localhost:3000/redirect", :follow_location => true)
+    hydra.queue request
+    hydra.run
+
+    request.response.code.should == 200
+  end
+
+  it "should pass through the max_redirects option when set on a request" do
+    hydra = Typhoeus::Hydra.new
+    request = Typhoeus::Request.new("http://localhost:3000/bad_redirect", :max_redirects => 5)
+    hydra.queue request
+    hydra.run
+
+    request.response.code.should == 302
+  end
+end