typhoeus 0.6.3 → 0.6.4

data/lib/rack/typhoeus/middleware/params_decoder.rb

@@ -36,7 +36,7 @@ module Rack
         private
 
         # Persist params change in environment. Extracted from:
-        # https://github.com/rack/rack/blob/master/lib/rack/request.rb#L233
+        # https://github.com/rack/rack/blob/master/lib/rack/request.rb#L243
         def update_params(req, k, v)
           found = false
           if req.GET.has_key?(k)

data/lib/rack/typhoeus/middleware/params_decoder/helper.rb

@@ -48,8 +48,12 @@ module Rack
           # @return [Boolean] True if its a encoded Array, else false.
           def encoded?(hash)
             return false if hash.empty?
-            keys = hash.keys.map{|i| i.to_i if i.respond_to?(:to_i)}.sort
-            keys == hash.keys.size.times.to_a
+	          if hash.keys.size > 1
+              keys = hash.keys.map{|i| i.to_i if i.respond_to?(:to_i)}.sort
+              keys == hash.keys.size.times.to_a
+	          else
+              hash.keys.first =~ /0/
+	          end
           end
 
           # If the Hash is an array encoded by typhoeus an array is returned

data/lib/typhoeus/adapters/faraday.rb

@@ -110,8 +110,11 @@ module Faraday # :nodoc:
       def configure_ssl(req, env)
         ssl = env[:ssl]
 
-        ssl_verifyhost = (ssl && ssl.fetch(:verify, true)) ? 2 : 0
+        verify_p = (ssl && ssl.fetch(:verify, true))
+
+        ssl_verifyhost = verify_p ? 2 : 0
         req.options[:ssl_verifyhost] = ssl_verifyhost
+        req.options[:ssl_verifypeer] = verify_p
         req.options[:sslversion] = ssl[:version]     if ssl[:version]
         req.options[:sslcert]    = ssl[:client_cert] if ssl[:client_cert]
         req.options[:sslkey]     = ssl[:client_key]  if ssl[:client_key]

data/lib/typhoeus/easy_factory.rb

@@ -51,7 +51,7 @@ module Typhoeus
         easy.http_request(
           request.base_url,
           request.options.fetch(:method, :get),
-          request.options.reject{ |k,_| [:method, :cache_ttl].include?(k) }
+          sanitize(request.options)
         )
       rescue Ethon::Errors::InvalidOption => e
         help = provide_help(e.message.match(/:\s(\w+)/)[1])
@@ -63,6 +63,20 @@ module Typhoeus
 
     private
 
+    def sanitize(options)
+      sanitized = {}
+      request.options.each do |k,v|
+        next if [:method, :cache_ttl].include?(k.to_sym)
+        if new_option = renamed_options[k.to_sym]
+          warn("Deprecated option #{k}. Please use #{new_option} instead.")
+          sanitized[new_option] = v
+        else
+          sanitized[k] = v
+        end
+      end
+      sanitized
+    end
+
     # Sets on_complete callback on easy in order to be able to
     # track progress.
     #
@@ -72,28 +86,22 @@ module Typhoeus
     # @return [ Ethon::Easy ] The easy.
     def set_callback
       easy.on_complete do |easy|
-        request.finish(Response.new(easy.to_hash))
+        request.finish(Response.new(easy.mirror.options))
         Typhoeus::Pool.release(easy)
         if hydra && !hydra.queued_requests.empty?
-          hydra.add(hydra.queued_requests.shift)
+          hydra.dequeue
         end
       end
     end
 
-    def provide_help(option)
-      renamed = {
+    def renamed_options
+      {
         :auth_method => :httpauth,
         :connect_timeout => :connecttimeout,
-        :disable_ssl_host_verification => :ssl_verifyhost,
-        :disable_ssl_peer_verification => :ssl_verifypeer,
         :encoding => :accept_encoding,
         :follow_location => :followlocation,
         :max_redirects => :maxredirs,
-        :password => :userpwd,
-        :proxy_auth_method => :proxyauth,
-        :proxy_password => :proxyuserpwd,
         :proxy_type => :proxytype,
-        :proxy_username => :proxyuserpwd,
         :ssl_cacert => :cainfo,
         :ssl_capath => :capath,
         :ssl_cert => :sslcert,
@@ -102,12 +110,25 @@ module Typhoeus
         :ssl_key_password => :keypasswd,
         :ssl_key_type => :sslkeytype,
         :ssl_version => :sslversion,
-        :username => :userpwd
       }
-      removed = [:cache_key_basis, :cache_timout, :user_agent]
-      if new_option = renamed[option.to_sym]
+    end
+
+    def changed_options
+      {
+        :disable_ssl_host_verification => :ssl_verifyhost,
+        :disable_ssl_peer_verification => :ssl_verifypeer,
+        :proxy_auth_method => :proxyauth,
+      }
+    end
+
+    def removed_options
+      [:cache_key_basis, :cache_timeout, :user_agent]
+    end
+
+    def provide_help(option)
+      if new_option = changed_options[option.to_sym]
         "\nPlease try #{new_option} instead of #{option}." if new_option
-      elsif removed.include?(option.to_sym)
+      elsif removed_options.include?(option.to_sym)
         "\nThe option #{option} was removed."
       end
     end

data/lib/typhoeus/hydra/before.rb

@@ -11,7 +11,7 @@ module Typhoeus
 
       # Overrride add in order to execute callbacks in
       # Typhoeus.before. Will break and return when a
-      # callback returns nil or false. Calls super
+      # callback returns nil, false or a response. Calls super
       # otherwise.
       #
       # @example Add the request.
@@ -20,6 +20,7 @@ module Typhoeus
         Typhoeus.before.each do |callback|
           value = callback.call(request)
           if value.nil? || value == false || value.is_a?(Response)
+            dequeue
             return value
           end
         end

data/lib/typhoeus/hydra/cacheable.rb

@@ -4,6 +4,7 @@ module Typhoeus
       def add(request)
         if request.cacheable? && response = Typhoeus::Config.cache.get(request)
           request.finish(response)
+          dequeue
         else
           super
         end

data/lib/typhoeus/hydra/memoizable.rb

@@ -36,6 +36,7 @@ module Typhoeus
         if request.memoizable? && memory.has_key?(request)
           response = memory[request]
           request.finish(response, true)
+          dequeue
         else
           super
         end

data/lib/typhoeus/hydra/queueable.rb

@@ -10,7 +10,7 @@ module Typhoeus
       # Return the queued requests.
       #
       # @example Return queued requests.
-      #  hydra.queued_requests
+      #   hydra.queued_requests
       #
       # @return [ Array<Typhoeus::Request> ] The queued requests.
       def queued_requests
@@ -23,7 +23,7 @@ module Typhoeus
       # anything about already running requests.
       #
       # @example Abort hydra.
-      #  hydra.abort
+      #   hydra.abort
       def abort
         queued_requests.clear
       end
@@ -34,11 +34,23 @@ module Typhoeus
       # request.
       #
       # @example Queue request.
-      #  hydra.queue(request)
+      #   hydra.queue(request)
       def queue(request)
         request.hydra = self
         queued_requests << request
       end
+
+      # Removes a request from queued_requests and
+      # adds it to the hydra in order to be
+      # performed next.
+      #
+      # @example Dequeue request.
+      #   hydra.dequeue
+      #
+      # @since 0.6.4
+      def dequeue
+        add(queued_requests.shift) unless queued_requests.empty?
+      end
     end
   end
 end

data/lib/typhoeus/request.rb

@@ -1,3 +1,4 @@
+require 'zlib'
 require 'typhoeus/request/actions'
 require 'typhoeus/request/before'
 require 'typhoeus/request/block_connection'
@@ -123,7 +124,10 @@ module Typhoeus
     #
     # @since 0.5.5
     def url
-      EasyFactory.new(self).get.url
+      easy = EasyFactory.new(self).get
+      url = easy.url
+      Typhoeus::Pool.release(easy)
+      url
     end
 
     # Returns whether other is equal to self.
@@ -148,7 +152,7 @@ module Typhoeus
     #
     # @api private
     def hash
-      Digest::MD5.hexdigest("#{self.class.name}#{base_url}#{options}").to_i(16)
+      Zlib.crc32 "#{self.class.name}#{base_url}#{options}"
     end
 
     private

data/lib/typhoeus/request/marshal.rb

@@ -5,9 +5,10 @@ module Typhoeus
     module Marshal
 
       # Return the important data needed to serialize this Request, except the
-      # `on_complete` handler, since they cannot be marshalled.
+      # `on_complete`, `on_success`, `on_failure`, and `hydra`, since they cannot be marshalled.
       def marshal_dump
-        (instance_variables - ['@on_complete', :@on_complete]).map do |name|
+        unmarshallable = %w(@on_complete @on_success @on_failure @hydra)
+        (instance_variables - unmarshallable - unmarshallable.map(&:to_sym)).map do |name|
           [name, instance_variable_get(name)]
         end
       end

data/lib/typhoeus/response/informations.rb

@@ -188,6 +188,10 @@ module Typhoeus
         options[:redirect_count]
       end
 
+      def debug_info
+        options[:debug_info]
+      end
+
       # Returns the response header.
       #
       # @example Return headers.

data/lib/typhoeus/version.rb

@@ -1,5 +1,5 @@
 module Typhoeus
 
   # The current Typhoeus version.
-  VERSION = '0.6.3'
+  VERSION = '0.6.4'
 end

data/perf/profile.rb

@@ -0,0 +1,14 @@
+require 'typhoeus'
+require 'ruby-prof'
+
+calls = 50
+base_url = "http://127.0.0.1:3000/"
+
+RubyProf.start
+calls.times do |i|
+  Typhoeus::Request.get(base_url+i.to_s)
+end
+result = RubyProf.stop
+
+printer = RubyProf::FlatPrinter.new(result)
+printer.print(STDOUT)

data/perf/vs_nethttp.rb

@@ -0,0 +1,64 @@
+require 'typhoeus'
+require 'net/http'
+require 'open-uri'
+require 'benchmark'
+
+URL = "http://localhost:300"
+hydra = Typhoeus::Hydra.new(max_concurrency: 3)
+
+if defined? require_relative
+  require_relative '../spec/support/localhost_server.rb'
+  require_relative '../spec/support/server.rb'
+else
+  require '../spec/support/localhost_server.rb'
+  require '../spec/support/server.rb'
+end
+LocalhostServer.new(TESTSERVER.new, 3000)
+LocalhostServer.new(TESTSERVER.new, 3001)
+LocalhostServer.new(TESTSERVER.new, 3002)
+
+def url_for(i)
+  "#{URL}#{i%3}/"
+end
+
+Benchmark.bm do |bm|
+
+  [1000].each do |calls|
+    puts "[ #{calls} requests ]"
+
+    bm.report("net/http        ") do
+      calls.times do |i|
+        uri = URI.parse(url_for(i))
+        Net::HTTP.get_response(uri)
+      end
+    end
+
+    bm.report("open            ") do
+      calls.times do |i|
+        open(url_for(i))
+      end
+    end
+
+    bm.report("request         ") do
+      calls.times do |i|
+        Typhoeus::Request.get(url_for(i))
+      end
+    end
+
+    bm.report("hydra           ") do
+      calls.times do |i|
+        hydra.queue(Typhoeus::Request.new(url_for(i)))
+      end
+      hydra.run
+    end
+
+    bm.report("hydra memoize   ") do
+      Typhoeus::Config.memoize = true
+      calls.times do |i|
+        hydra.queue(Typhoeus::Request.new(url_for(i)))
+      end
+      hydra.run
+      Typhoeus::Config.memoize = false
+    end
+  end
+end

data/spec/rack/typhoeus/middleware/params_decoder/helper_spec.rb

@@ -0,0 +1,132 @@
+require 'spec_helper'
+require "rack/typhoeus"
+
+describe "Rack::Typhoeus::Middleware::ParamsDecoder::Helper" do
+
+  let(:klass) do
+    Class.new do
+      include Rack::Typhoeus::Middleware::ParamsDecoder::Helper
+    end.new
+  end
+
+  describe "#decode" do
+    let(:decoded) { klass.decode(params) }
+    let(:params) { { :array => {'0' => :a, '1' => :b } } }
+
+    it "decodes" do
+      expect(decoded[:array]).to match_array([:a, :b])
+    end
+
+    it "doesn't modify" do
+      expect(decoded).to_not be(params)
+    end
+  end
+
+  describe "#decode!" do
+    let(:decoded) { klass.decode!(params) }
+
+    context "when hash" do
+      context "when encoded" do
+        context "when simple" do
+          let(:params) { { :array => {'0' => :a, '1' => :b } } }
+
+          it "decodes" do
+            expect(decoded[:array]).to match_array([:a, :b])
+          end
+
+          it "modifies" do
+            expect(decoded).to eq(params)
+          end
+        end
+
+        context "when nested" do
+          let(:params) do
+            { :array => { '0' => 0, '1' => { '0' => 'sub0', '1' => 'sub1' } } }
+          end
+
+          it "decodes" do
+            expect(decoded[:array]).to include(0)
+            expect(decoded[:array].find{|e| e.is_a?(Array)}).to(
+              match_array(['sub0', 'sub1'])
+            )
+          end
+
+          it "modifies" do
+            expect(decoded).to eq(params)
+          end
+        end
+      end
+
+      context "when not encoded" do
+        let(:params) { {:a => :a} }
+
+        it "doesn't modify" do
+          expect(decoded).to be(params)
+        end
+      end
+    end
+
+    context "when no hash" do
+      let(:params) { "a" }
+
+      it "returns self" do
+        expect(decoded).to be(params)
+      end
+    end
+  end
+
+  describe "#encoded?" do
+    let(:encoded) { klass.send(:encoded?, params) }
+
+    context "when there is only one key" do
+      context "and its 0" do
+        let(:params){ {'0' => 1} }
+        it 'returns true' do
+          expect(encoded).to be_true
+        end
+      end
+      context "and its not 0" do
+        let(:params){ {'some-key' => 1}}
+        it 'returns false' do
+          expect(encoded).to be_false
+        end
+      end
+    end
+
+    context "when keys are ascending numbers starting with zero" do
+      let(:params) { Hash[12.times.map {|i| [i, (i+65).chr]}] }
+
+      it "returns true" do
+        expect(encoded).to be_true
+      end
+    end
+
+    context "when keys are not ascending numbers starting with zero" do
+      let(:params) { {:a => 1} }
+
+      it "returns false" do
+        expect(encoded).to be_false
+      end
+    end
+  end
+
+  describe "#convert" do
+    let(:converted) { klass.send(:convert, params) }
+
+    context "when encoded" do
+      let(:params) { {'0' => :a, '1' => :b} }
+
+      it "returns values" do
+        expect(converted).to match_array([:a, :b])
+      end
+    end
+
+    context "when not encoded" do
+      let(:params) { {:a => :a} }
+
+      it "returns unmodified" do
+        expect(converted).to be(params)
+      end
+    end
+  end
+end