typhoeus 0.1.6 → 0.1.28

data/lib/typhoeus/request.rb

@@ -1,8 +1,34 @@
 module Typhoeus
   class Request
-    attr_accessor :method, :params, :body, :headers, :timeout, :user_agent, :response, :cache_timeout
+    attr_accessor :method, :params, :body, :headers, :timeout, :user_agent, :response, :cache_timeout, :follow_location, :max_redirects, :proxy, :disable_ssl_peer_verification, :ssl_cert, :ssl_cert_type, :ssl_key, :ssl_key_type, :ssl_key_password, :ssl_cacert, :ssl_capath
+
+
     attr_reader   :url
-    
+
+    # Initialize a new Request
+    #
+    # Options:
+    # * +url+ : Endpoint (URL) of the request
+    # * +options+   : A hash containing options among :
+    # ** +:method+  : :get (default) / :post / :put
+    # ** +:params+  : params as a Hash
+    # ** +:body+
+    # ** +:timeout+ : timeout (ms)
+    # ** +:headers+  : headers as Hash
+    # ** +:user_agent+ : user agent (string)
+    # ** +:cache_timeout+ : cache timeout (ms)
+    # ** +:follow_location
+    # ** +:max_redirects
+    # ** +:proxy
+    # ** +:disable_ssl_peer_verification
+    # ** +:ssl_cert
+    # ** +:ssl_cert_type
+    # ** +:ssl_key
+    # ** +:ssl_key_type
+    # ** +:ssl_key_password
+    # ** +:ssl_cacert
+    # ** +:ssl_capath
+    #
     def initialize(url, options = {})
       @method           = options[:method] || :get
       @params           = options[:params]
@@ -11,12 +37,28 @@ module Typhoeus
       @headers          = options[:headers] || {}
       @user_agent       = options[:user_agent] || Typhoeus::USER_AGENT
       @cache_timeout    = options[:cache_timeout]
-      @url              = @params ? "#{url}?#{params_string}" : url
+      @follow_location  = options[:follow_location]
+      @max_redirects    = options[:max_redirects]
+      @proxy            = options[:proxy]
+      @disable_ssl_peer_verification = options[:disable_ssl_peer_verification]
+      @ssl_cert         = options[:ssl_cert]
+      @ssl_cert_type    = options[:ssl_cert_type]
+      @ssl_key          = options[:ssl_key]
+      @ssl_key_type     = options[:ssl_key_type]
+      @ssl_key_password = options[:ssl_key_password]
+      @ssl_cacert       = options[:ssl_cacert]
+      @ssl_capath       = options[:ssl_capath]
+
+      if @method == :post
+        @url = url
+      else
+        @url = @params ? "#{url}?#{params_string}" : url
+      end
       @on_complete      = nil
       @after_complete   = nil
       @handled_response = nil
     end
-    
+
     def host
       slash_location = @url.index('/', 8)
       if slash_location
@@ -26,14 +68,14 @@ module Typhoeus
         return query_string_location ? @url.slice(0, query_string_location) : @url
       end
     end
-    
+
     def headers
       @headers["User-Agent"] = @user_agent
       @headers
     end
-    
+
     def params_string
-      params.keys.sort.collect do |k|
+      params.keys.sort { |a, b| a.to_s <=> b.to_s }.collect do |k|
         value = params[k]
         if value.is_a? Hash
           value.keys.collect {|sk| Rack::Utils.escape("#{k}[#{sk}]") + "=" + Rack::Utils.escape(value[sk].to_s)}
@@ -45,67 +87,71 @@ module Typhoeus
         end
       end.flatten.join("&")
     end
-    
+
     def on_complete(&block)
       @on_complete = block
     end
-    
+
     def on_complete=(proc)
       @on_complete = proc
     end
-    
+
     def after_complete(&block)
       @after_complete = block
     end
-    
+
     def after_complete=(proc)
       @after_complete = proc
     end
-    
+
     def call_handlers
       if @on_complete
         @handled_response = @on_complete.call(response)
         call_after_complete
       end
     end
-    
+
     def call_after_complete
        @after_complete.call(@handled_response) if @after_complete
     end
-    
+
     def handled_response=(val)
       @handled_response = val
     end
-    
+
     def handled_response
       @handled_response || response
     end
-    
+
     def cache_key
       Digest::SHA1.hexdigest(url)
     end
-    
+
     def self.run(url, params)
       r = new(url, params)
       Typhoeus::Hydra.hydra.queue r
       Typhoeus::Hydra.hydra.run
       r.response
     end
-    
-    def self.get(url, params)
+
+    def self.get(url, params = {})
       run(url, params.merge(:method => :get))
     end
-    
-    def self.post(url, params)
+
+    def self.post(url, params = {})
       run(url, params.merge(:method => :post))
     end
-    
-    def self.put(url, params)
+
+    def self.put(url, params = {})
       run(url, params.merge(:method => :put))
     end
-    
-    def self.delete(url, params)
+
+    def self.delete(url, params = {})
       run(url, params.merge(:method => :delete))
     end
+
+    def self.head(url, params = {})
+      run(url, params.merge(:method => :head))
+    end
   end
-end
+end

data/lib/typhoeus/response.rb

@@ -3,8 +3,9 @@ module Typhoeus
     attr_accessor :request
     attr_reader :code, :headers, :body, :time,
                 :requested_url, :requested_remote_method,
-                :requested_http_method, :start_time
-    
+                :requested_http_method, :start_time,
+                :effective_url
+
     def initialize(params = {})
       @code                  = params[:code]
       @headers               = params[:headers]
@@ -14,6 +15,35 @@ module Typhoeus
       @requested_http_method = params[:requested_http_method]
       @start_time            = params[:start_time]
       @request               = params[:request]
+      @effective_url         = params[:effective_url]
+    end
+
+    def headers_hash
+      headers.split("\n").map {|o| o.strip}.inject({}) do |hash, o|
+        if o.empty?
+          hash
+        else
+          i = o.index(":") || o.size
+          key = o.slice(0, i)
+          value = o.slice(i + 1, o.size)
+          value = value.strip unless value.nil?
+          if hash.has_key? key
+            hash[key] = [hash[key], value].flatten
+          else
+            hash[key] = value
+          end
+
+          hash
+        end
+      end
+    end
+
+    def success?
+      @code >= 200 && @code < 300
+    end
+
+    def modified?
+      @code != 304
     end
   end
 end

data/lib/typhoeus/service.rb

@@ -0,0 +1,20 @@
+module Typhoeus
+  class Service
+    def initialize(host, port)
+      @host = host
+      @port = port
+    end
+    
+    def get(resource, params)
+    end
+    
+    def put(resource, params)
+    end
+    
+    def post(resource, params)
+    end
+    
+    def delete(resource, params)
+    end
+  end
+end

data/lib/typhoeus.rb

@@ -14,23 +14,23 @@ require 'typhoeus/request'
 require 'typhoeus/hydra'
 
 module Typhoeus
-  VERSION = "0.1.6"
-  
+  VERSION = File.read(File.dirname(__FILE__) + "/../VERSION").chomp
+
   def self.easy_object_pool
     @easy_objects ||= []
   end
-  
+
   def self.init_easy_object_pool
     20.times do
       easy_object_pool << Typhoeus::Easy.new
     end
   end
-  
+
   def self.release_easy_object(easy)
     easy.reset
     easy_object_pool << easy
   end
-  
+
   def self.get_easy_object
     if easy_object_pool.empty?
       Typhoeus::Easy.new
@@ -38,12 +38,12 @@ module Typhoeus
       easy_object_pool.pop
     end
   end
-  
+
   def self.add_easy_request(easy_object)
     Thread.current[:curl_multi] ||= Typhoeus::Multi.new
     Thread.current[:curl_multi].add(easy_object)
   end
-  
+
   def self.perform_easy_requests
     multi = Thread.current[:curl_multi]
     start_time = Time.now

data/profilers/valgrind.rb

@@ -0,0 +1,24 @@
+#!/usr/bin/env ruby
+# go to ext/typhoeus and run ruby extconf.rb && make before running
+# this.
+
+$LOAD_PATH.unshift(File.dirname(__FILE__) + "/../ext")
+require File.dirname(__FILE__) + "/../lib/typhoeus"
+
+klass = Class.new { include Typhoeus }
+
+loops = ENV['LOOPS'].to_i
+url = ARGV.first || (raise "requires URL!")
+
+loops.times do |i|
+  puts "On loop #{i}" if i % 10 == 0
+  results = []
+  5.times do
+    results << klass.get(url)
+  end
+ 
+  # fire requests
+  results[0].code
+end
+
+puts "Ran #{loops} loops on #{url}!"

data/spec/fixtures/result_set.xml

@@ -0,0 +1,60 @@
+<result_set>
+  <ttl>20</ttl>
+  <result>
+    <id>1</id>
+    <name>hello</name>
+    <description>
+      this is a long description for a text field of some kind.
+      this is a long description for a text field of some kind.
+      this is a long description for a text field of some kind.
+      this is a long description for a text field of some kind.
+      this is a long description for a text field of some kind.
+      this is a long description for a text field of some kind.
+      this is a long description for a text field of some kind.
+      this is a long description for a text field of some kind.
+      this is a long description for a text field of some kind.
+      this is a long description for a text field of some kind.
+      this is a long description for a text field of some kind.
+      this is a long description for a text field of some kind.
+      this is a long description for a text field of some kind.
+    </description>
+  </result>
+  <result>
+    <id>2</id>
+    <name>hello</name>
+    <description>
+      this is a long description for a text field of some kind.
+      this is a long description for a text field of some kind.
+      this is a long description for a text field of some kind.
+      this is a long description for a text field of some kind.
+      this is a long description for a text field of some kind.
+      this is a long description for a text field of some kind.
+      this is a long description for a text field of some kind.
+      this is a long description for a text field of some kind.
+      this is a long description for a text field of some kind.
+      this is a long description for a text field of some kind.
+      this is a long description for a text field of some kind.
+      this is a long description for a text field of some kind.
+      this is a long description for a text field of some kind.
+    </description>
+  </result>
+  <result>
+    <id>3</id>
+    <name>hello</name>
+    <description>
+      this is a long description for a text field of some kind.
+      this is a long description for a text field of some kind.
+      this is a long description for a text field of some kind.
+      this is a long description for a text field of some kind.
+      this is a long description for a text field of some kind.
+      this is a long description for a text field of some kind.
+      this is a long description for a text field of some kind.
+      this is a long description for a text field of some kind.
+      this is a long description for a text field of some kind.
+      this is a long description for a text field of some kind.
+      this is a long description for a text field of some kind.
+      this is a long description for a text field of some kind.
+      this is a long description for a text field of some kind.
+    </description>
+  </result>
+</result_set>

data/spec/servers/app.rb

@@ -1,3 +1,4 @@
+#!/usr/bin/env ruby
 require 'rubygems'
 require 'sinatra'
 require 'json'
@@ -16,11 +17,46 @@ get '/fail_forever' do
   error 500, "oh noes!"
 end
 
+get '/redirect' do
+  redirect '/'
+end
+
+get '/bad_redirect' do
+  redirect '/bad_redirect'
+end
+
+get '/auth_basic/:username/:password' do
+  @auth ||=  Rack::Auth::Basic::Request.new(request.env)
+  # Check that we've got a basic auth, and that it's credentials match the ones
+  # provided in the request
+  if @auth.provided? && @auth.basic? && @auth.credentials == [ params[:username], params[:password] ]
+    # auth is valid - confirm it
+    true
+  else
+    # invalid auth - request the authentication
+    response['WWW-Authenticate'] = %(Basic realm="Testing HTTP Auth")
+    throw(:halt, [401, "Not authorized\n"])
+  end
+end
+
+get '/auth_ntlm' do
+  # we're just checking for the existence if NTLM auth header here. It's validation
+  # is too troublesome and really doesn't bother is much, it's up to libcurl to make
+  # it valid
+  is_ntlm_auth = /^NTLM/ =~ request.env['HTTP_AUTHORIZATION']
+  true if is_ntlm_auth
+  throw(:halt, [401, "Not authorized\n"]) if !is_ntlm_auth
+end
+
 get '/**' do
   sleep params["delay"].to_i if params.has_key?("delay")
   request.env.merge!(:body => request.body.read).to_json
 end
 
+head '/**' do
+  sleep params["delay"].to_i if params.has_key?("delay")
+end
+
 put '/**' do
   puts request.inspect
   request.env.merge!(:body => request.body.read).to_json
@@ -34,4 +70,4 @@ end
 delete '/**' do
   puts request.inspect
   request.env.merge!(:body => request.body.read).to_json
-end
+end

data/spec/typhoeus/easy_spec.rb

@@ -1,9 +1,51 @@
 require File.dirname(__FILE__) + '/../spec_helper'
 
-describe Typhoeus::Easy do  
+describe Typhoeus::Easy do
+  describe "#supports_zlib" do
+    before(:each) do
+      @easy = Typhoeus::Easy.new
+    end
+
+    it "should return true if the version string has zlib" do
+      @easy.stub!(:curl_version).and_return("libcurl/7.20.0 OpenSSL/0.9.8l zlib/1.2.3 libidn/1.16")
+      @easy.supports_zlib?.should be_true
+    end
+
+    it "should return false if the version string doesn't have zlib" do
+      @easy.stub!(:curl_version).and_return("libcurl/7.20.0 OpenSSL/0.9.8l libidn/1.16")
+      @easy.supports_zlib?.should be_false
+    end
+  end
+
   describe "options" do
-    it "should allow for following redirects"
-    it "should allow you to set the user agent"
+    it "should not follow redirects if not instructed to" do
+      e = Typhoeus::Easy.new
+      e.url = "http://localhost:3001/redirect"
+      e.method = :get
+      e.perform
+      e.response_code.should == 302
+    end
+
+    it "should allow for following redirects" do
+      e = Typhoeus::Easy.new
+      e.url = "http://localhost:3001/redirect"
+      e.method = :get
+      e.follow_location = true
+      e.perform
+      e.response_code.should == 200
+      JSON.parse(e.response_body)["REQUEST_METHOD"].should == "GET"
+    end
+
+    it "should allow you to set the user agent" do
+      easy = Typhoeus::Easy.new
+      easy.url    = "http://localhost:3002"
+      easy.method = :get
+      easy.user_agent = "myapp"
+      easy.perform
+      easy.response_code.should == 200
+      JSON.parse(easy.response_body)["HTTP_USER_AGENT"].should == "myapp"
+    end
+
     it "should provide a timeout in milliseconds" do
       e = Typhoeus::Easy.new
       e.url = "http://localhost:3001"
@@ -13,6 +55,55 @@ describe Typhoeus::Easy do
       # this doesn't work on a mac for some reason
 #      e.timed_out?.should == true
     end
+
+    it "should allow the setting of the max redirects to follow" do
+      e = Typhoeus::Easy.new
+      e.url = "http://localhost:3001/redirect"
+      e.method = :get
+      e.follow_location = true
+      e.max_redirects = 5
+      e.perform
+      e.response_code.should == 200
+    end
+
+    it "should handle our bad redirect action, provided we've set max_redirects properly" do
+      e = Typhoeus::Easy.new
+      e.url = "http://localhost:3001/bad_redirect"
+      e.method = :get
+      e.follow_location = true
+      e.max_redirects = 5
+      e.perform
+      e.response_code.should == 302
+    end
+  end
+  
+  describe "authentication" do
+    it "should allow to set username and password" do
+      e = Typhoeus::Easy.new
+      username, password = 'foo', 'bar'
+      e.auth = { :username => username, :password => password }
+      e.url = "http://localhost:3001/auth_basic/#{username}/#{password}"
+      e.method = :get
+      e.perform
+      e.response_code.should == 200
+    end
+    
+    it "should allow to query auth methods support by the server" do
+      e = Typhoeus::Easy.new
+      e.url = "http://localhost:3001/auth_basic/foo/bar"
+      e.method = :get
+      e.perform
+      e.auth_methods.should == Typhoeus::Easy::AUTH_TYPES[:CURLAUTH_BASIC]
+    end
+
+    it "should allow to set authentication method" do
+      e = Typhoeus::Easy.new
+      e.auth = { :username => 'username', :password => 'password', :method => Typhoeus::Easy::AUTH_TYPES[:CURLAUTH_NTLM] }
+      e.url = "http://localhost:3001/auth_ntlm"
+      e.method = :get
+      e.perform
+      e.response_code.should == 200
+    end
   end
   
   describe "get" do
@@ -26,6 +117,24 @@ describe Typhoeus::Easy do
     end
   end
 
+  describe "purge" do
+    it "should set custom request to purge" do
+      easy = Typhoeus::Easy.new
+      easy.should_receive(:set_option).with(Typhoeus::Easy::OPTION_VALUES[:CURLOPT_CUSTOMREQUEST], "PURGE").once
+      easy.method = :purge
+    end
+  end
+
+  describe "head" do
+    it "should perform a head" do
+      easy = Typhoeus::Easy.new
+      easy.url    = "http://localhost:3002"
+      easy.method = :head
+      easy.perform
+      easy.response_code.should == 200
+    end
+  end
+
   describe "start_time" do
     it "should be get/settable" do
       time = Time.now