tech-angels-typhoeus 0.1.36

data/spec/servers/app.rb

@@ -0,0 +1,73 @@
+#!/usr/bin/env ruby
+require 'rubygems'
+require 'sinatra'
+require 'json'
+
+@@fail_count = 0
+get '/fail/:number' do
+  if @@fail_count >= params[:number].to_i
+    "ok"
+  else
+    @@fail_count += 1
+    error 500, "oh noes!"
+   end
+end
+
+get '/fail_forever' do
+  error 500, "oh noes!"
+end
+
+get '/redirect' do
+  redirect '/'
+end
+
+get '/bad_redirect' do
+  redirect '/bad_redirect'
+end
+
+get '/auth_basic/:username/:password' do
+  @auth ||=  Rack::Auth::Basic::Request.new(request.env)
+  # Check that we've got a basic auth, and that it's credentials match the ones
+  # provided in the request
+  if @auth.provided? && @auth.basic? && @auth.credentials == [ params[:username], params[:password] ]
+    # auth is valid - confirm it
+    true
+  else
+    # invalid auth - request the authentication
+    response['WWW-Authenticate'] = %(Basic realm="Testing HTTP Auth")
+    throw(:halt, [401, "Not authorized\n"])
+  end
+end
+
+get '/auth_ntlm' do
+  # we're just checking for the existence if NTLM auth header here. It's validation
+  # is too troublesome and really doesn't bother is much, it's up to libcurl to make
+  # it valid
+  is_ntlm_auth = /^NTLM/ =~ request.env['HTTP_AUTHORIZATION']
+  true if is_ntlm_auth
+  throw(:halt, [401, "Not authorized\n"]) if !is_ntlm_auth
+end
+
+get '/**' do
+  sleep params["delay"].to_i if params.has_key?("delay")
+  request.env.merge!(:body => request.body.read).to_json
+end
+
+head '/**' do
+  sleep params["delay"].to_i if params.has_key?("delay")
+end
+
+put '/**' do
+  puts request.inspect
+  request.env.merge!(:body => request.body.read).to_json
+end
+
+post '/**' do
+  puts request.inspect
+  request.env.merge!(:body => request.body.read).to_json
+end
+
+delete '/**' do
+  puts request.inspect
+  request.env.merge!(:body => request.body.read).to_json
+end

data/spec/spec.opts

@@ -0,0 +1,2 @@
+--diff
+--color

data/spec/spec_helper.rb

@@ -0,0 +1,11 @@
+require "rubygems"
+require 'json'
+require "spec"
+
+# gem install redgreen for colored test output
+begin require "redgreen" unless ENV['TM_CURRENT_LINE']; rescue LoadError; end
+
+path = File.expand_path(File.dirname(__FILE__) + "/../lib/")
+$LOAD_PATH.unshift(path) unless $LOAD_PATH.include?(path)
+
+require "lib/typhoeus"

data/spec/typhoeus/easy_spec.rb

@@ -0,0 +1,236 @@
+require File.dirname(__FILE__) + '/../spec_helper'
+
+describe Typhoeus::Easy do
+  describe "#supports_zlib" do
+    before(:each) do
+      @easy = Typhoeus::Easy.new
+    end
+
+    it "should return true if the version string has zlib" do
+      @easy.stub!(:curl_version).and_return("libcurl/7.20.0 OpenSSL/0.9.8l zlib/1.2.3 libidn/1.16")
+      @easy.supports_zlib?.should be_true
+    end
+
+    it "should return false if the version string doesn't have zlib" do
+      @easy.stub!(:curl_version).and_return("libcurl/7.20.0 OpenSSL/0.9.8l libidn/1.16")
+      @easy.supports_zlib?.should be_false
+    end
+  end
+
+  describe "options" do
+    it "should not follow redirects if not instructed to" do
+      e = Typhoeus::Easy.new
+      e.url = "http://localhost:3001/redirect"
+      e.method = :get
+      e.perform
+      e.response_code.should == 302
+    end
+
+    it "should allow for following redirects" do
+      e = Typhoeus::Easy.new
+      e.url = "http://localhost:3001/redirect"
+      e.method = :get
+      e.follow_location = true
+      e.perform
+      e.response_code.should == 200
+      JSON.parse(e.response_body)["REQUEST_METHOD"].should == "GET"
+    end
+
+    it "should allow you to set the user agent" do
+      easy = Typhoeus::Easy.new
+      easy.url    = "http://localhost:3002"
+      easy.method = :get
+      easy.user_agent = "myapp"
+      easy.perform
+      easy.response_code.should == 200
+      JSON.parse(easy.response_body)["HTTP_USER_AGENT"].should == "myapp"
+    end
+
+    it "should provide a timeout in milliseconds" do
+      e = Typhoeus::Easy.new
+      e.url = "http://localhost:3001"
+      e.method = :get
+      e.timeout = 50
+      e.perform
+      # this doesn't work on a mac for some reason
+#      e.timed_out?.should == true
+    end
+
+    it "should allow the setting of the max redirects to follow" do
+      e = Typhoeus::Easy.new
+      e.url = "http://localhost:3001/redirect"
+      e.method = :get
+      e.follow_location = true
+      e.max_redirects = 5
+      e.perform
+      e.response_code.should == 200
+    end
+
+    it "should handle our bad redirect action, provided we've set max_redirects properly" do
+      e = Typhoeus::Easy.new
+      e.url = "http://localhost:3001/bad_redirect"
+      e.method = :get
+      e.follow_location = true
+      e.max_redirects = 5
+      e.perform
+      e.response_code.should == 302
+    end
+  end
+  
+  describe "authentication" do
+    it "should allow to set username and password" do
+      e = Typhoeus::Easy.new
+      username, password = 'foo', 'bar'
+      e.auth = { :username => username, :password => password }
+      e.url = "http://localhost:3001/auth_basic/#{username}/#{password}"
+      e.method = :get
+      e.perform
+      e.response_code.should == 200
+    end
+    
+    it "should allow to query auth methods support by the server" do
+      e = Typhoeus::Easy.new
+      e.url = "http://localhost:3001/auth_basic/foo/bar"
+      e.method = :get
+      e.perform
+      e.auth_methods.should == Typhoeus::Easy::AUTH_TYPES[:CURLAUTH_BASIC]
+    end
+
+    it "should allow to set authentication method" do
+      e = Typhoeus::Easy.new
+      e.auth = { :username => 'username', :password => 'password', :method => Typhoeus::Easy::AUTH_TYPES[:CURLAUTH_NTLM] }
+      e.url = "http://localhost:3001/auth_ntlm"
+      e.method = :get
+      e.perform
+      e.response_code.should == 200
+    end
+  end
+  
+  describe "get" do
+    it "should perform a get" do
+      easy = Typhoeus::Easy.new
+      easy.url    = "http://localhost:3002"
+      easy.method = :get
+      easy.perform
+      easy.response_code.should == 200
+      JSON.parse(easy.response_body)["REQUEST_METHOD"].should == "GET"
+    end
+  end
+
+  describe "purge" do
+    it "should set custom request to purge" do
+      easy = Typhoeus::Easy.new
+      easy.should_receive(:set_option).with(Typhoeus::Easy::OPTION_VALUES[:CURLOPT_CUSTOMREQUEST], "PURGE").once
+      easy.method = :purge
+    end
+  end
+
+  describe "head" do
+    it "should perform a head" do
+      easy = Typhoeus::Easy.new
+      easy.url    = "http://localhost:3002"
+      easy.method = :head
+      easy.perform
+      easy.response_code.should == 200
+    end
+  end
+
+  describe "start_time" do
+    it "should be get/settable" do
+      time = Time.now
+      easy = Typhoeus::Easy.new
+      easy.start_time.should be_nil
+      easy.start_time = time
+      easy.start_time.should == time
+    end
+  end
+
+  describe "params=" do
+    it "should handle arrays of params" do
+      easy = Typhoeus::Easy.new
+      easy.url = "http://localhost:3002/index.html"
+      easy.method = :get
+      easy.request_body = "this is a body!"
+      easy.params = {
+        :foo => 'bar',
+        :username => ['dbalatero', 'dbalatero2']
+      }
+      
+      easy.url.should =~ /\?.*username=dbalatero&username=dbalatero2/
+    end
+  end
+
+
+  describe "put" do
+    it "should perform a put" do
+      easy = Typhoeus::Easy.new
+      easy.url    = "http://localhost:3002"
+      easy.method = :put
+      easy.perform
+      easy.response_code.should == 200
+      JSON.parse(easy.response_body)["REQUEST_METHOD"].should == "PUT"
+    end
+    
+    it "should send a request body" do
+      easy = Typhoeus::Easy.new
+      easy.url    = "http://localhost:3002"
+      easy.method = :put
+      easy.request_body = "this is a body!"
+      easy.perform
+      easy.response_code.should == 200
+      easy.response_body.should include("this is a body!")
+    end
+  end
+  
+  describe "post" do
+    it "should perform a post" do
+      easy = Typhoeus::Easy.new
+      easy.url    = "http://localhost:3002"
+      easy.method = :post
+      easy.perform
+      easy.response_code.should == 200
+      JSON.parse(easy.response_body)["REQUEST_METHOD"].should == "POST"
+    end
+    
+    it "should send a request body" do
+      easy = Typhoeus::Easy.new
+      easy.url    = "http://localhost:3002"
+      easy.method = :post
+      easy.request_body = "this is a body!"
+      easy.perform
+      easy.response_code.should == 200
+      easy.response_body.should include("this is a body!")
+    end
+    
+    it "should handle params" do
+      easy = Typhoeus::Easy.new
+      easy.url    = "http://localhost:3002"
+      easy.method = :post
+      easy.params = {:foo => "bar"}
+      easy.perform
+      easy.response_code.should == 200
+      easy.response_body.should include("foo=bar")
+    end
+  end
+  
+  describe "delete" do
+    it "should perform a delete" do
+      easy = Typhoeus::Easy.new
+      easy.url    = "http://localhost:3002"
+      easy.method = :delete
+      easy.perform
+      easy.response_code.should == 200
+      JSON.parse(easy.response_body)["REQUEST_METHOD"].should == "DELETE"
+    end
+    
+    it "should send a request body" do
+      easy = Typhoeus::Easy.new
+      easy.url    = "http://localhost:3002"
+      easy.method = :delete
+      easy.request_body = "this is a body!"
+      easy.perform
+      easy.response_code.should == 200
+      easy.response_body.should include("this is a body!")
+    end
+  end  
+end

data/spec/typhoeus/filter_spec.rb

@@ -0,0 +1,35 @@
+require File.dirname(__FILE__) + '/../spec_helper'
+
+describe Typhoeus::Filter do
+  it "should take a method name and optionally take options" do
+    filter = Typhoeus::Filter.new(:bar, :only => :foo)
+    filter = Typhoeus::Filter.new(:bar)
+  end
+  
+  describe "#apply_filter?" do
+    it "should return true for any method when :only and :except aren't specified" do
+      filter = Typhoeus::Filter.new(:bar)
+      filter.apply_filter?(:asdf).should be_true
+    end
+    
+    it "should return true if a method is in only" do
+      filter = Typhoeus::Filter.new(:bar, :only => :foo)
+      filter.apply_filter?(:foo).should be_true
+    end
+    
+    it "should return false if a method isn't in only" do
+      filter = Typhoeus::Filter.new(:bar, :only => :foo)
+      filter.apply_filter?(:bar).should be_false
+    end
+    
+    it "should return true if a method isn't in except" do
+      filter = Typhoeus::Filter.new(:bar, :except => :foo)
+      filter.apply_filter?(:bar).should be_true
+    end
+    
+    it "should return false if a method is in except" do
+      filter = Typhoeus::Filter.new(:bar, :except => :foo)
+      filter.apply_filter?(:foo).should be_false
+    end
+  end
+end

data/spec/typhoeus/hydra_spec.rb

@@ -0,0 +1,311 @@
+require File.dirname(__FILE__) + '/../spec_helper'
+
+# some of these tests assume that you have some local services running.
+# ruby spec/servers/app.rb -p 3000
+# ruby spec/servers/app.rb -p 3001
+# ruby spec/servers/app.rb -p 3002
+describe Typhoeus::Hydra do
+  before(:all) do
+    cache_class = Class.new do
+      def initialize
+        @cache = {}
+      end
+      def get(key)
+        @cache[key]
+      end
+      def set(key, object, timeout = 0)
+        @cache[key] = object
+      end
+    end
+    @cache = cache_class.new
+  end
+
+  it "has a singleton" do
+    Typhoeus::Hydra.hydra.should be_a Typhoeus::Hydra
+  end
+
+  it "has a setter for the singleton" do
+    Typhoeus::Hydra.hydra = :foo
+    Typhoeus::Hydra.hydra.should == :foo
+    Typhoeus::Hydra.hydra = Typhoeus::Hydra.new
+  end
+
+  context "#stub" do
+    before do
+      @hydra = Typhoeus::Hydra.new
+      @on_complete_handler_called = nil
+      @request  = Typhoeus::Request.new("http://localhost:3000/foo")
+      @request.on_complete do |response|
+        @on_complete_handler_called = true
+        response.code.should == 404
+        response.headers.should == "whatever"
+      end
+      @response = Typhoeus::Response.new(:code => 404, :headers => "whatever", :body => "not found", :time => 0.1)
+    end
+
+    it "stubs requests to a specific URI" do
+      @hydra.stub(:get, "http://localhost:3000/foo").and_return(@response)
+      @hydra.queue(@request)
+      @hydra.run
+      @on_complete_handler_called.should be_true
+      @response.request.should == @request
+    end
+
+    it "stubs requests to URIs matching a pattern" do
+      @hydra.stub(:get, /foo/).and_return(@response)
+      @hydra.queue(@request)
+      @hydra.run
+      @on_complete_handler_called.should be_true
+      @response.request.should == @request
+    end
+
+    it "can clear stubs" do
+      @hydra.clear_stubs
+    end
+    
+    it "clears out previously queued requests once they are called" do
+      @hydra.stub(:get, "asdf").and_return(@response)
+
+      call_count = 0
+      request = Typhoeus::Request.new("asdf")
+      request.on_complete do |response|
+        call_count += 1
+      end
+      @hydra.queue(request)
+      @hydra.run
+      call_count.should == 1
+      @hydra.run
+      call_count.should == 1
+    end
+    
+    it "calls stubs for requests that are queued up in the on_complete of a first stub" do
+      @hydra.stub(:get, "asdf").and_return(@response)
+      @hydra.stub(:get, "bar").and_return(@response)
+      
+      second_handler_called = false
+      request = Typhoeus::Request.new("asdf")
+      request.on_complete do |response|
+        r = Typhoeus::Request.new("bar")
+        r.on_complete do |res|
+          second_handler_called = true
+        end
+        @hydra.queue(r)
+      end
+      @hydra.queue(request)
+      @hydra.run
+      
+      second_handler_called.should be_true
+    end
+
+    it "matches a stub only when the HTTP method also matches"
+  end
+
+  it "queues a request" do
+    hydra = Typhoeus::Hydra.new
+    hydra.queue Typhoeus::Request.new("http://localhost:3000")
+  end
+
+  it "runs a batch of requests" do
+    hydra  = Typhoeus::Hydra.new
+    first  = Typhoeus::Request.new("http://localhost:3000/first")
+    second = Typhoeus::Request.new("http://localhost:3001/second")
+    hydra.queue first
+    hydra.queue second
+    hydra.run
+    first.response.body.should include("first")
+    second.response.body.should include("second")
+  end
+
+  it "has a cache_setter proc" do
+    hydra = Typhoeus::Hydra.new
+    hydra.cache_setter do |request|
+      # @cache.set(request.cache_key, request.response, request.cache_timeout)
+    end
+  end
+
+  it "has a cache_getter" do
+    hydra = Typhoeus::Hydra.new
+    hydra.cache_getter do |request|
+      # @cache.get(request.cache_key) rescue nil
+    end
+  end
+
+  it "memoizes GET reqeusts" do
+    hydra  = Typhoeus::Hydra.new
+    first  = Typhoeus::Request.new("http://localhost:3000/foo", :params => {:delay => 1})
+    second = Typhoeus::Request.new("http://localhost:3000/foo", :params => {:delay => 1})
+    hydra.queue first
+    hydra.queue second
+    start_time = Time.now
+    hydra.run
+    first.response.body.should include("foo")
+    first.handled_response.body.should include("foo")
+    first.response.should == second.response
+    first.handled_response.should == second.handled_response
+    (Time.now - start_time).should < 1.2 # if it had run twice it would be ~ 2 seconds
+  end
+
+  it "can turn off memoization for GET requests" do
+    hydra  = Typhoeus::Hydra.new
+    hydra.disable_memoization
+    first  = Typhoeus::Request.new("http://localhost:3000/foo")
+    second = Typhoeus::Request.new("http://localhost:3000/foo")
+    hydra.queue first
+    hydra.queue second
+    hydra.run
+    first.response.body.should include("foo")
+    first.response.object_id.should_not == second.response.object_id
+  end
+
+  it "pulls GETs from cache" do
+    hydra  = Typhoeus::Hydra.new
+    start_time = Time.now
+    hydra.cache_getter do |request|
+      @cache.get(request.cache_key) rescue nil
+    end
+    hydra.cache_setter do |request|
+      @cache.set(request.cache_key, request.response, request.cache_timeout)
+    end
+
+    first  = Typhoeus::Request.new("http://localhost:3000/foo", :params => {:delay => 1})
+    @cache.set(first.cache_key, :foo, 60)
+    hydra.queue first
+    hydra.run
+    (Time.now - start_time).should < 0.1
+    first.response.should == :foo
+  end
+
+  it "sets GET responses to cache when the request has a cache_timeout value" do
+    hydra  = Typhoeus::Hydra.new
+    hydra.cache_getter do |request|
+      @cache.get(request.cache_key) rescue nil
+    end
+    hydra.cache_setter do |request|
+      @cache.set(request.cache_key, request.response, request.cache_timeout)
+    end
+
+    first  = Typhoeus::Request.new("http://localhost:3000/first", :cache_timeout => 0)
+    second = Typhoeus::Request.new("http://localhost:3000/second")
+    hydra.queue first
+    hydra.queue second
+    hydra.run
+    first.response.body.should include("first")
+    @cache.get(first.cache_key).should == first.response
+    @cache.get(second.cache_key).should be_nil
+  end
+
+  it "has a global on_complete" do
+    foo = nil
+    hydra  = Typhoeus::Hydra.new
+    hydra.on_complete do |response|
+      foo = :called
+    end
+
+    first  = Typhoeus::Request.new("http://localhost:3000/first")
+    hydra.queue first
+    hydra.run
+    first.response.body.should include("first")
+    foo.should == :called
+  end
+
+  it "has a global on_complete setter" do
+    foo = nil
+    hydra  = Typhoeus::Hydra.new
+    proc = Proc.new {|response| foo = :called}
+    hydra.on_complete = proc
+
+    first  = Typhoeus::Request.new("http://localhost:3000/first")
+    hydra.queue first
+    hydra.run
+    first.response.body.should include("first")
+    foo.should == :called
+  end
+
+  it "should reuse connections from the pool for a host"
+
+  it "should queue up requests while others are running" do
+    hydra   = Typhoeus::Hydra.new
+
+    start_time = Time.now
+    @responses = []
+
+    request = Typhoeus::Request.new("http://localhost:3000/first", :params => {:delay => 1})
+    request.on_complete do |response|
+      @responses << response
+      response.body.should include("first")
+    end
+
+    request.after_complete do |object|
+      second_request = Typhoeus::Request.new("http://localhost:3001/second", :params => {:delay => 2})
+      second_request.on_complete do |response|
+        @responses << response
+        response.body.should include("second")
+      end
+      hydra.queue second_request
+    end
+    hydra.queue request
+
+    third_request = Typhoeus::Request.new("http://localhost:3002/third", :params => {:delay => 3})
+    third_request.on_complete do |response|
+      @responses << response
+      response.body.should include("third")
+    end
+    hydra.queue third_request
+
+    hydra.run
+    @responses.size.should == 3
+    (Time.now - start_time).should < 3.3
+  end
+
+  it "should fire and forget" do
+    # this test is totally hacky. I have no clue how to make it verify. I just look at the test servers
+    # to verify that stuff is running
+    hydra  = Typhoeus::Hydra.new
+    first  = Typhoeus::Request.new("http://localhost:3000/first?delay=1")
+    second = Typhoeus::Request.new("http://localhost:3001/second?delay=2")
+    hydra.queue first
+    hydra.queue second
+    hydra.fire_and_forget
+    third = Typhoeus::Request.new("http://localhost:3002/third?delay=3")
+    hydra.queue third
+    hydra.fire_and_forget
+    sleep 3 # have to do this or future tests may break.
+  end
+
+  it "should take the maximum number of concurrent requests as an argument" do
+    hydra = Typhoeus::Hydra.new(:max_concurrency => 2)
+    first  = Typhoeus::Request.new("http://localhost:3000/first?delay=1")
+    second = Typhoeus::Request.new("http://localhost:3001/second?delay=1")
+    third  = Typhoeus::Request.new("http://localhost:3002/third?delay=1")
+    hydra.queue first
+    hydra.queue second
+    hydra.queue third
+
+    start_time = Time.now
+    hydra.run
+    finish_time = Time.now
+
+    first.response.code.should == 200
+    second.response.code.should == 200
+    third.response.code.should == 200
+    (finish_time - start_time).should > 2.0
+  end
+
+  it "should respect the follow_location option when set on a request" do
+    hydra = Typhoeus::Hydra.new
+    request = Typhoeus::Request.new("http://localhost:3000/redirect", :follow_location => true)
+    hydra.queue request
+    hydra.run
+
+    request.response.code.should == 200
+  end
+
+  it "should pass through the max_redirects option when set on a request" do
+    hydra = Typhoeus::Hydra.new
+    request = Typhoeus::Request.new("http://localhost:3000/bad_redirect", :max_redirects => 5)
+    hydra.queue request
+    hydra.run
+
+    request.response.code.should == 302
+  end
+end