typhoeus 0.3.3 → 0.4.0

data/lib/typhoeus/{normalized_header_hash.rb → header.rb}

@@ -1,5 +1,5 @@
 module Typhoeus
-  class NormalizedHeaderHash < ::Hash
+  class Header < ::Hash
     def initialize(constructor = {})
       if constructor.is_a?(Hash)
         super
@@ -17,10 +17,6 @@ module Typhoeus
       super(convert_key(key))
     end
 
-    [:include?, :has_key?, :member?].each do |method|
-      alias_method method, :key?
-    end
-
     def [](key)
       super(convert_key(key))
     end
@@ -39,7 +35,7 @@ module Typhoeus
     alias_method :merge!, :update
 
     def dup
-      self.class.new(self)
+      self.class.new(Marshal.load(Marshal.dump(self)))
     end
 
     def merge(hash)

data/lib/typhoeus/hydra.rb

@@ -64,7 +64,7 @@ module Typhoeus
         @queued_requests << request
       else
         if request.method == :get
-          if @memoize_requests && @memoized_requests.has_key?(request.url)
+          if @memoize_requests && @memoized_requests.key?(request.url)
             if response = @retrieved_from_cache[request.url]
               request.response = response
               request.call_handlers
@@ -120,6 +120,8 @@ module Typhoeus
       @on_complete = proc
     end
 
+    private
+
     def get_from_cache_or_queue(request)
       if @cache_getter
         val = @cache_getter.call(request)
@@ -134,7 +136,6 @@ module Typhoeus
         @multi.add(get_easy_object(request))
       end
     end
-    private :get_from_cache_or_queue
 
     def get_easy_object(request)
       @running_requests += 1
@@ -143,27 +144,27 @@ module Typhoeus
       easy.verbose          = request.verbose
       if request.username || request.password
         auth = { :username => request.username, :password => request.password }
-        auth[:method] = Typhoeus::Easy::AUTH_TYPES["CURLAUTH_#{request.auth_method.to_s.upcase}".to_sym] if request.auth_method
+        auth[:method] = request.auth_method if request.auth_method
         easy.auth = auth
       end
 
       if request.proxy
         proxy = { :server => request.proxy }
-        proxy[:type] = Typhoeus::Easy::PROXY_TYPES["CURLPROXY_#{request.proxy_type.to_s.upcase}".to_sym] if request.proxy_type
+        proxy[:type] = request.proxy_type if request.proxy_type
         easy.proxy = proxy if request.proxy
       end
 
       if request.proxy_username || request.proxy_password
         auth = { :username => request.proxy_username, :password => request.proxy_password }
-        auth[:method] = Typhoeus::Easy::AUTH_TYPES["CURLAUTH_#{request.proxy_auth_method.to_s.upcase}".to_sym] if request.proxy_auth_method
+        auth[:method] = request.proxy_auth_method if request.proxy_auth_method
         easy.proxy_auth = auth
       end
 
       easy.url          = request.url
       easy.method       = request.method
-      easy.params       = request.params  if request.method == :post && !request.params.nil?
+      easy.params       = request.params  if [:post, :put].include?(request.method) && !request.params.nil?
       easy.headers      = request.headers if request.headers
-      easy.request_body = request.body    if request.body
+      easy.request_body = request.body    if [:post, :put].include?(request.method) && !request.body.nil?
       easy.timeout      = request.timeout if request.timeout
       easy.connect_timeout = request.connect_timeout if request.connect_timeout
       easy.interface       = request.interface if request.interface
@@ -194,19 +195,16 @@ module Typhoeus
       easy.set_headers
       easy
     end
-    private :get_easy_object
 
     def queue_next
       @running_requests -= 1
       queue(@queued_requests.shift) unless @queued_requests.empty?
     end
-    private :queue_next
 
     def release_easy_object(easy)
       easy.reset
       @easy_pool.push easy
     end
-    private :release_easy_object
 
     def handle_request(request, response, live_request = true)
       request.response = response
@@ -227,7 +225,6 @@ module Typhoeus
         end
       end
     end
-    private :handle_request
 
     def response_from_easy(easy, request)
       Response.new(:code                => easy.response_code,
@@ -243,8 +240,8 @@ module Typhoeus
                    :primary_ip          => easy.primary_ip,
                    :curl_return_code => easy.curl_return_code,
                    :curl_error_message => easy.curl_error_message,
+                   :redirect_count      => easy.redirect_count,
                    :request             => request)
     end
-    private :response_from_easy
   end
 end

data/lib/typhoeus/hydra_mock.rb

@@ -9,7 +9,7 @@ module Typhoeus
       @requests = []
       @options = options
       if @options[:headers]
-        @options[:headers] = Typhoeus::NormalizedHeaderHash.new(@options[:headers])
+        @options[:headers] = Typhoeus::Header.new(@options[:headers])
       end
 
       @current_response_index = 0
@@ -93,7 +93,7 @@ module Typhoeus
     end
 
     def headers_match?(request)
-      request_headers = NormalizedHeaderHash.new(request.headers)
+      request_headers = Header.new(request.headers)
 
       if empty_headers?(self.headers)
         empty_headers?(request_headers)

data/lib/typhoeus/multi.rb

@@ -3,35 +3,144 @@ module Typhoeus
     attr_reader :easy_handles
 
     def initialize
+      Curl.init
+
+      @handle = Curl.multi_init
+      @active = 0
+      @running = 0
       @easy_handles = []
+
+      @timeout = ::FFI::MemoryPointer.new(:long)
+      @timeval = Curl::Timeval.new
+      @fd_read = Curl::FDSet.new
+      @fd_write = Curl::FDSet.new
+      @fd_excep = Curl::FDSet.new
+      @max_fd = ::FFI::MemoryPointer.new(:int)
+
+      ObjectSpace.define_finalizer(self, self.class.finalizer(self))
     end
 
-    def remove(easy)
-      multi_remove_handle(easy) if @easy_handles.include?(easy)
+    def self.finalizer(multi)
+      proc { Curl.multi_cleanup(multi.handle) }
     end
 
     def add(easy)
       raise "trying to add easy handle twice" if @easy_handles.include?(easy)
       easy.set_headers() if easy.headers.empty?
-      multi_add_handle(easy)
+
+      code = Curl.multi_add_handle(@handle, easy.handle)
+      raise RuntimeError.new("An error occured adding the handle: #{code}: #{Curl.multi_strerror(code)}") if code != :call_multi_perform and code != :ok
+
+      do_perform if code == :call_multi_perform
+
+      @active += 1
+      @easy_handles << easy
+      easy
     end
 
-    def perform()
-      while active_handle_count > 0 do
-        multi_perform
+    def remove(easy)
+      if @easy_handles.include?(easy)
+        @active -= 1
+        Curl.multi_remove_handle(@handle, easy.handle)
+        @easy_handles.delete(easy)
+      end
+    end
+
+    def perform
+      while @active > 0
+        run
+        while @running > 0
+          # get the curl-suggested timeout
+          code = Curl.multi_timeout(@handle, @timeout)
+          raise RuntimeError.new("an error occured getting the timeout: #{code}: #{Curl.multi_strerror(code)}") if code != :ok
+          timeout = @timeout.read_long
+          if timeout == 0 # no delay
+            run
+            next
+          elsif timeout < 0
+            timeout = 1
+          end
+
+          # load the fd sets from the multi handle
+          @fd_read.clear
+          @fd_write.clear
+          @fd_excep.clear
+          code = Curl.multi_fdset(@handle, @fd_read, @fd_write, @fd_excep, @max_fd)
+          raise RuntimeError.new("an error occured getting the fdset: #{code}: #{Curl.multi_strerror(code)}") if code != :ok
+
+          max_fd = @max_fd.read_int
+          if max_fd == -1
+            # curl is doing something special so let it run for a moment
+            sleep(0.001)
+          else
+            @timeval[:sec] = timeout / 1000
+            @timeval[:usec] = (timeout * 1000) % 1000000
+
+            code = Curl.select(max_fd + 1, @fd_read, @fd_write, @fd_excep, @timeval)
+            raise RuntimeError.new("error on thread select: #{::FFI.errno}") if code < 0
+          end
+
+          run
+        end
       end
       reset_easy_handles
     end
 
-    def cleanup()
-      multi_cleanup
+    def fire_and_forget
+      run
+    end
+
+    # check for finished easy handles and remove from the multi handle
+    def read_info
+      msgs_left = ::FFI::MemoryPointer.new(:int)
+      while not (msg = Curl.multi_info_read(@handle, msgs_left)).null?
+        next if msg[:code] != :done
+
+        easy = @easy_handles.find {|easy| easy.handle == msg[:easy_handle] }
+        next if not easy
+
+        response_code = ::FFI::MemoryPointer.new(:long)
+        response_code.write_long(-1)
+        Curl.easy_getinfo(easy.handle, :response_code, response_code)
+        response_code = response_code.read_long
+        remove(easy)
+
+        easy.curl_return_code = msg[:data][:code]
+        if easy.curl_return_code != 0 then easy.failure
+        elsif (200..299).member?(response_code) or response_code == 0 then easy.success
+        else easy.failure
+        end
+      end
+    end
+
+    def cleanup
+      Curl.multi_cleanup(@handle)
+      @active = 0
+      @running = 0
+      @easy_handles = []
     end
 
     def reset_easy_handles
       @easy_handles.dup.each do |easy|
-        multi_remove_handle(easy)
+        remove(easy)
         yield easy if block_given?
       end
     end
+
+    private
+
+    # called by perform and fire_and_forget
+    def run
+      begin code = do_perform end while code == :call_multi_perform
+      raise RuntimeError.new("an error occured while running perform: #{code}: #{Curl.multi_strerror(code)}") if code != :ok
+      read_info
+    end
+
+    def do_perform
+      running = ::FFI::MemoryPointer.new(:int)
+      code = Curl.multi_perform(@handle, running)
+      @running = running.read_int
+      code
+    end
   end
 end

data/lib/typhoeus/param_processor.rb

@@ -0,0 +1,43 @@
+require 'tempfile'
+
+module Typhoeus
+  class ParamProcessor
+    class << self
+      def traverse_params_hash(hash, result = nil, current_key = nil)
+        result ||= { :files => [], :params => [] }
+
+        hash.keys.sort { |a, b| a.to_s <=> b.to_s }.collect do |key|
+          new_key = (current_key ? "#{current_key}[#{key}]" : key).to_s
+          current_value = hash[key]
+          process_value current_value, :result => result, :new_key => new_key
+        end
+        result
+      end
+
+      def process_value(current_value, options)
+        result = options[:result]
+        new_key = options[:new_key]
+
+        case current_value
+        when Hash
+          traverse_params_hash(current_value, result, new_key)
+        when Array
+          current_value.each do |v|
+            result[:params] << [new_key, v.to_s]
+          end
+        when File, Tempfile
+          filename = File.basename(current_value.path)
+          types = MIME::Types.type_for(filename)
+          result[:files] << [
+            new_key,
+            filename,
+            types.empty? ? 'application/octet-stream' : types[0].to_s,
+            File.expand_path(current_value.path)
+          ]
+        else
+          result[:params] << [new_key, current_value.to_s]
+        end
+      end
+    end
+  end
+end

data/lib/typhoeus/request.rb

@@ -7,6 +7,7 @@ module Typhoeus
       :params,
       :body,
       :headers,
+      :cache_key_basis,
       :connect_timeout,
       :timeout,
       :user_agent,
@@ -32,12 +33,10 @@ module Typhoeus
       :username,
       :password,
       :auth_method,
-      :user_agent,
       :proxy_auth_method,
       :proxy_type
     ]
 
-    attr_reader   :url
     attr_accessor *ACCESSOR_OPTIONS
 
     # Initialize a new Request
@@ -69,9 +68,9 @@ module Typhoeus
     # ** +:username
     # ** +:password
     # ** +:auth_method
-    # ** +:user_agent+ : user agent (string) - DEPRECATED
     #
     def initialize(url, options = {})
+      @url = url
       @method           = options[:method] || :get
       @params           = options[:params]
       @body             = options[:body]
@@ -80,10 +79,6 @@ module Typhoeus
       @interface        = options[:interface]
       @headers          = options[:headers] || {}
 
-      if options.has_key?(:user_agent)
-        self.user_agent = options[:user_agent]
-      end
-
       @cache_timeout    = safe_to_i(options[:cache_timeout])
       @follow_location  = options[:follow_location]
       @max_redirects    = options[:max_redirects]
@@ -107,14 +102,6 @@ module Typhoeus
       @password         = options[:password]
       @auth_method      = options[:auth_method]
 
-      if @method == :post
-        @url = url
-      else
-        @url = @params ? "#{url}?#{params_string}" : url
-      end
-
-      @parsed_uri = URI.parse(@url)
-
       @on_complete      = nil
       @after_complete   = nil
       @handled_response = nil
@@ -123,16 +110,25 @@ module Typhoeus
     LOCALHOST_ALIASES = %w[ localhost 127.0.0.1 0.0.0.0 ]
 
     def localhost?
-      LOCALHOST_ALIASES.include?(@parsed_uri.host)
+      LOCALHOST_ALIASES.include?(parsed_uri.host)
     end
 
     def user_agent
       headers['User-Agent']
     end
 
-    def user_agent=(value)
-      puts "DEPRECATED: Typhoeus::Request#user_agent=(value). This will be removed in a later version."
-      headers['User-Agent'] = value
+    def url
+      if [:post, :put].include?(@method)
+        @url
+      else
+        url = "#{@url}?#{params_string}"
+        url += "&#{URI.escape(@body)}" if @body
+        url.gsub("?&", "?").gsub(/\?$/, '')
+      end
+    end
+
+    def parsed_uri
+      @parsed_uri ||= URI.parse(@url)
     end
 
     def host
@@ -146,10 +142,11 @@ module Typhoeus
     end
 
     def host_domain
-      @parsed_uri.host
+      parsed_uri.host
     end
 
     def params_string
+      return nil unless params
       traversal = Typhoeus::Utils.traverse_params_hash(params)
       Typhoeus::Utils.traversal_to_param_string(traversal)
     end
@@ -208,7 +205,7 @@ module Typhoeus
     end
 
     def cache_key
-      Digest::SHA1.hexdigest(url)
+      Digest::SHA1.hexdigest(cache_key_basis || url)
     end
 
     def self.run(url, params)

data/lib/typhoeus/response.rb

@@ -8,7 +8,7 @@ module Typhoeus
                 :app_connect_time, :pretransfer_time,
                 :connect_time, :name_lookup_time,
                 :curl_return_code, :curl_error_message,
-                :primary_ip
+                :primary_ip, :redirect_count
 
     attr_writer :headers_hash
 
@@ -32,8 +32,9 @@ module Typhoeus
       @request               = params[:request]
       @effective_url         = params[:effective_url]
       @primary_ip            = params[:primary_ip]
+      @redirect_count        = params[:redirect_count]
       @mock                  = params[:mock] || false  # default
-      @headers_hash          = NormalizedHeaderHash.new(params[:headers_hash]) if params[:headers_hash]
+      @headers_hash          = Header.new(params[:headers_hash]) if params[:headers_hash]
     end
 
     # Returns true if this is a mock response.
@@ -47,7 +48,7 @@ module Typhoeus
 
     def headers_hash
       @headers_hash ||= begin
-        headers.split("\n").map {|o| o.strip}.inject(Typhoeus::NormalizedHeaderHash.new) do |hash, o|
+        headers.split("\n").map {|o| o.strip}.inject(Typhoeus::Header.new) do |hash, o|
           if o.empty? || o =~ /^HTTP\/[\d\.]+/
             hash
           else
@@ -55,7 +56,7 @@ module Typhoeus
             key = o.slice(0, i)
             value = o.slice(i + 1, o.size)
             value = value.strip unless value.nil?
-            if hash.has_key? key
+            if hash.key? key
               hash[key] = [hash[key], value].flatten
             else
               hash[key] = value