typhoeus 0.4.2 → 1.4.0

data/lib/typhoeus.rb

@@ -1,56 +1,143 @@
 require 'digest/sha2'
+require 'ethon'
 
-require 'typhoeus/utils'
-require 'typhoeus/header'
-require 'typhoeus/curl'
-require 'typhoeus/easy'
-require 'typhoeus/form'
-require 'typhoeus/multi'
-require 'typhoeus/filter'
-require 'typhoeus/param_processor'
-require 'typhoeus/remote'
-require 'typhoeus/remote_proxy_object'
-require 'typhoeus/response'
-require 'typhoeus/request'
+require 'typhoeus/config'
+require 'typhoeus/easy_factory'
+require 'typhoeus/errors'
+require 'typhoeus/expectation'
 require 'typhoeus/hydra'
-require 'typhoeus/hydra_mock'
+require 'typhoeus/pool'
+require 'typhoeus/request'
+require 'typhoeus/response'
 require 'typhoeus/version'
 
+# If we are using any Rack-based application, then we need the Typhoeus rack
+# middleware to ensure our app is running properly.
+if defined?(Rack)
+  require "rack/typhoeus"
+end
+
+# If the Redis gem is available, load the redis cache adapter
+if defined?(Redis)
+  require "typhoeus/cache/redis"
+end
+
+# If the Dalli gem is available, load the Dalli cache adapter
+if defined?(Dalli)
+  require "typhoeus/cache/dalli"
+end
+
+# If we are using Rails, load the Rails cache adapter
+if defined?(Rails)
+  require "typhoeus/cache/rails"
+end
+
+# If we are using Rails, then we will include the Typhoeus railtie.
+# if defined?(Rails)
+#   require "typhoeus/railtie"
+# end
+
+# Typhoeus is a HTTP client library based on Ethon which
+# wraps libcurl. Sitting on top of libcurl makes Typhoeus
+# very reliable and fast.
+#
+# There are some gems using Typhoeus like
+# {https://github.com/myronmarston/vcr VCR},
+# {https://github.com/bblimke/webmock WebMock} or
+# {https://github.com/technoweenie/faraday Faraday}. VCR
+# and WebMock provide their own adapter whereas
+# Faraday relies on {Faraday::Adapter::Typhoeus}
+# since Typhoeus version 0.5.
+#
+# @example (see Typhoeus::Request)
+# @example (see Typhoeus::Hydra)
+#
+# @see Typhoeus::Request
+# @see Typhoeus::Hydra
+# @see Faraday::Adapter::Typhoeus
+#
+# @since 0.5.0
 module Typhoeus
-  def self.easy_object_pool
-    @easy_objects ||= []
-  end
+  extend Request::Actions
+  extend Request::Callbacks::Types
 
-  def self.init_easy_object_pool
-    20.times do
-      easy_object_pool << Typhoeus::Easy.new
-    end
-  end
+  # The default Typhoeus user agent.
+  USER_AGENT = "Typhoeus - https://github.com/typhoeus/typhoeus"
 
-  def self.release_easy_object(easy)
-    easy.reset
-    easy_object_pool << easy
+  # Set the Typhoeus configuration options by passing a block.
+  #
+  # @example (see Typhoeus::Config)
+  #
+  # @yield [ Typhoeus::Config ]
+  #
+  # @return [ Typhoeus::Config ] The configuration.
+  #
+  # @see Typhoeus::Config
+  def self.configure
+    yield Config
   end
 
-  def self.get_easy_object
-    if easy_object_pool.empty?
-      Typhoeus::Easy.new
-    else
-      easy_object_pool.pop
+  # Stub out a specific request.
+  #
+  # @example (see Typhoeus::Expectation)
+  #
+  # @param [ String ] base_url The url to stub out.
+  # @param [ Hash ] options The options to stub out.
+  #
+  # @return [ Typhoeus::Expectation ] The expecatation.
+  #
+  # @see Typhoeus::Expectation
+  def self.stub(base_url, options = {}, &block)
+    expectation = Expectation.all.find{ |e| e.base_url == base_url && e.options == options }
+    if expectation.nil?
+      expectation = Expectation.new(base_url, options)
+      Expectation.all << expectation
     end
+
+    expectation.and_return(&block) unless block.nil?
+    expectation
   end
 
-  def self.add_easy_request(easy_object)
-    Thread.current[:curl_multi] ||= Typhoeus::Multi.new
-    Thread.current[:curl_multi].add(easy_object)
+  # Add before callbacks.
+  #
+  # @example Add before callback.
+  #   Typhoeus.before { |request| p request.base_url }
+  #
+  # @param [ Block ] block The callback.
+  #
+  # @yield [ Typhoeus::Request ]
+  #
+  # @return [ Array<Block> ] All before blocks.
+  def self.before(&block)
+    @before ||= []
+    @before << block if block_given?
+    @before
   end
 
-  def self.perform_easy_requests
-    multi = Thread.current[:curl_multi]
-    start_time = Time.now
-    multi.easy_handles.each do |easy|
-      easy.start_time = start_time
-    end
-    multi.perform
+  # Execute given block as if block connection is turned off.
+  # The old block connection state is restored afterwards.
+  #
+  # @example Make a real request, no matter if it's blocked.
+  #   Typhoeus::Config.block_connection = true
+  #   Typhoeus.get("www.example.com").code
+  #   #=> raise Typhoeus::Errors::NoStub
+  #
+  #   Typhoeus.with_connection do
+  #     Typhoeus.get("www.example.com").code
+  #     #=> :ok
+  #   end
+  #
+  # @yield Yields control to the block after disabling block_connection.
+  #        Afterwards, the block_connection is set to its original
+  #        value.
+  # @return [ Object ] Returns the return value of the block.
+  #
+  # @see Typhoeus::Config.block_connection
+  def self.with_connection
+    old = Config.block_connection
+    Config.block_connection = false
+    result = yield if block_given?
+    Config.block_connection = old
+    result
   end
 end

data/perf/profile.rb

@@ -0,0 +1,14 @@
+require 'typhoeus'
+require 'ruby-prof'
+
+calls = 50
+base_url = "http://127.0.0.1:3000/"
+
+RubyProf.start
+calls.times do |i|
+  Typhoeus::Request.get(base_url+i.to_s)
+end
+result = RubyProf.stop
+
+printer = RubyProf::FlatPrinter.new(result)
+printer.print(STDOUT)

data/perf/vs_nethttp.rb

@@ -0,0 +1,64 @@
+require 'typhoeus'
+require 'net/http'
+require 'open-uri'
+require 'benchmark'
+
+URL = "http://localhost:300"
+hydra = Typhoeus::Hydra.new(max_concurrency: 3)
+
+if defined? require_relative
+  require_relative '../spec/support/localhost_server.rb'
+  require_relative '../spec/support/server.rb'
+else
+  require '../spec/support/localhost_server.rb'
+  require '../spec/support/server.rb'
+end
+LocalhostServer.new(TESTSERVER.new, 3000)
+LocalhostServer.new(TESTSERVER.new, 3001)
+LocalhostServer.new(TESTSERVER.new, 3002)
+
+def url_for(i)
+  "#{URL}#{i%3}/"
+end
+
+Benchmark.bm do |bm|
+
+  [1000].each do |calls|
+    puts "[ #{calls} requests ]"
+
+    bm.report("net/http        ") do
+      calls.times do |i|
+        uri = URI.parse(url_for(i))
+        Net::HTTP.get_response(uri)
+      end
+    end
+
+    bm.report("open            ") do
+      calls.times do |i|
+        open(url_for(i))
+      end
+    end
+
+    bm.report("request         ") do
+      calls.times do |i|
+        Typhoeus::Request.get(url_for(i))
+      end
+    end
+
+    bm.report("hydra           ") do
+      calls.times do |i|
+        hydra.queue(Typhoeus::Request.new(url_for(i)))
+      end
+      hydra.run
+    end
+
+    bm.report("hydra memoize   ") do
+      Typhoeus::Config.memoize = true
+      calls.times do |i|
+        hydra.queue(Typhoeus::Request.new(url_for(i)))
+      end
+      hydra.run
+      Typhoeus::Config.memoize = false
+    end
+  end
+end

data/spec/rack/typhoeus/middleware/params_decoder/helper_spec.rb

@@ -0,0 +1,156 @@
+require 'spec_helper'
+require "rack/typhoeus"
+
+describe "Rack::Typhoeus::Middleware::ParamsDecoder::Helper" do
+
+  let(:klass) do
+    Class.new do
+      include Rack::Typhoeus::Middleware::ParamsDecoder::Helper
+    end.new
+  end
+
+  describe "#decode" do
+    let(:decoded) { klass.decode(params) }
+    let(:params) { { :array => {'0' => :a, '1' => :b } } }
+
+    it "decodes" do
+      expect(decoded[:array]).to match_array([:a, :b])
+    end
+
+    it "doesn't modify" do
+      expect(decoded).to_not be(params)
+    end
+  end
+
+  describe "#decode!" do
+    let(:decoded) { klass.decode!(params) }
+
+    context "when hash" do
+      context "when encoded" do
+        context "when simple" do
+          let(:params) { { :array => {'0' => :a, '1' => :b } } }
+
+          it "decodes" do
+            expect(decoded[:array]).to match_array([:a, :b])
+          end
+
+          it "modifies" do
+            expect(decoded).to eq(params)
+          end
+        end
+
+        context "when longer and more complex" do
+          let(:params) do
+            {
+              :ids => {
+                "0"  => "407304",
+                "1"  => "407305",
+                "2"  => "407306",
+                "3"  => "407307",
+                "4"  => "407308",
+                "5"  => "407309",
+                "6"  => "407310",
+                "7"  => "407311",
+                "8"  => "407312",
+                "9"  => "407313",
+                "10" => "327012"
+              }
+            }
+          end
+
+          it "decodes ensuring arrays maintain their original order" do
+            expect(decoded[:ids]).to eq(["407304", "407305", "407306", "407307", "407308", "407309", "407310", "407311", "407312", "407313", "327012"])
+          end
+        end
+
+        context "when nested" do
+          let(:params) do
+            { :array => { '0' => 0, '1' => { '0' => 'sub0', '1' => 'sub1' } } }
+          end
+
+          it "decodes" do
+            expect(decoded[:array]).to include(0)
+            expect(decoded[:array].find{|e| e.is_a?(Array)}).to(
+              match_array(['sub0', 'sub1'])
+            )
+          end
+
+          it "modifies" do
+            expect(decoded).to eq(params)
+          end
+        end
+      end
+
+      context "when not encoded" do
+        let(:params) { {:a => :a} }
+
+        it "doesn't modify" do
+          expect(decoded).to be(params)
+        end
+      end
+    end
+
+    context "when no hash" do
+      let(:params) { "a" }
+
+      it "returns self" do
+        expect(decoded).to be(params)
+      end
+    end
+  end
+
+  describe "#encoded?" do
+    let(:encoded) { klass.send(:encoded?, params) }
+
+    context "when there is only one key" do
+      context "and its 0" do
+        let(:params){ {'0' => 1} }
+        it 'returns true' do
+          expect(encoded).to be_truthy
+        end
+      end
+      context "and its not 0" do
+        let(:params){ {'some-key' => 1}}
+        it 'returns false' do
+          expect(encoded).to be_falsey
+        end
+      end
+    end
+
+    context "when keys are ascending numbers starting with zero" do
+      let(:params) { Hash[12.times.map {|i| [i, (i+65).chr]}] }
+
+      it "returns true" do
+        expect(encoded).to be_truthy
+      end
+    end
+
+    context "when keys are not ascending numbers starting with zero" do
+      let(:params) { {:a => 1} }
+
+      it "returns false" do
+        expect(encoded).to be_falsey
+      end
+    end
+  end
+
+  describe "#convert" do
+    let(:converted) { klass.send(:convert, params) }
+
+    context "when encoded" do
+      let(:params) { {'0' => :a, '1' => :b} }
+
+      it "returns values" do
+        expect(converted).to match_array([:a, :b])
+      end
+    end
+
+    context "when not encoded" do
+      let(:params) { {:a => :a} }
+
+      it "returns unmodified" do
+        expect(converted).to be(params)
+      end
+    end
+  end
+end

data/spec/rack/typhoeus/middleware/params_decoder_spec.rb

@@ -0,0 +1,31 @@
+require 'spec_helper'
+
+describe "Rack::Typhoeus::Middleware::ParamsDecoder" do
+
+  before(:all) do
+    require "rack/typhoeus"
+  end
+
+  let(:app) do
+    double
+  end
+
+  let(:env) do
+    double
+  end
+
+  let(:klass) do
+    Rack::Typhoeus::Middleware::ParamsDecoder
+  end
+
+  describe "#call" do
+  end
+
+  context "when requesting" do
+    let(:response) { Typhoeus.get("localhost:3001", :params => {:x => [:a]}) }
+
+    it "transforms parameters" do
+      expect(response.body).to include("query_hash\":{\"x\":[\"a\"]}")
+    end
+  end
+end

data/spec/spec_helper.rb

@@ -0,0 +1,29 @@
+$LOAD_PATH.unshift(File.dirname(__FILE__))
+$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), "..", "lib"))
+
+require "bundler"
+Bundler.setup
+require "typhoeus"
+require "rspec"
+
+Dir[File.join(File.dirname(__FILE__), "support/**/*.rb")].each { |f| require f }
+
+RSpec.configure do |config|
+  config.order = :rand
+
+  config.before(:suite) do
+    LocalhostServer.new(TESTSERVER.new, 3001)
+  end
+
+  config.after do
+    Typhoeus::Pool.clear
+    Typhoeus::Expectation.clear
+    Typhoeus.before.clear
+    Typhoeus.on_complete.clear
+    Typhoeus.on_success.clear
+    Typhoeus.on_failure.clear
+    Typhoeus::Config.verbose = false
+    Typhoeus::Config.block_connection = false
+    Typhoeus::Config.memoize = false
+  end
+end

data/spec/support/localhost_server.rb

@@ -0,0 +1,94 @@
+require 'rack'
+require 'rack/handler/webrick'
+require 'net/http'
+
+# The code for this is inspired by Capybara's server:
+#   http://github.com/jnicklas/capybara/blob/0.3.9/lib/capybara/server.rb
+class LocalhostServer
+  READY_MESSAGE = "Server ready"
+
+  class Identify
+    def initialize(app)
+      @app = app
+    end
+
+    def call(env)
+      if env["PATH_INFO"] == "/__identify__"
+        [200, {}, [LocalhostServer::READY_MESSAGE]]
+      else
+        @app.call(env)
+      end
+    end
+  end
+
+  attr_reader :port
+
+  def initialize(rack_app, port = nil)
+    @port = port || find_available_port
+    @rack_app = rack_app
+    concurrently { boot }
+    wait_until(10, "Boot failed.") { booted? }
+  end
+
+  private
+
+  def find_available_port
+    server = TCPServer.new('127.0.0.1', 0)
+    server.addr[1]
+  ensure
+    server.close if server
+  end
+
+  def boot
+    # Use WEBrick since it's part of the ruby standard library and is available on all ruby interpreters.
+    options = { :Port => port }
+    options.merge!(:AccessLog => [], :Logger => WEBrick::BasicLog.new(StringIO.new)) unless ENV['VERBOSE_SERVER']
+    Rack::Handler::WEBrick.run(Identify.new(@rack_app), options)
+  end
+
+  def booted?
+    res = ::Net::HTTP.get_response("localhost", '/__identify__', port)
+    if res.is_a?(::Net::HTTPSuccess) or res.is_a?(::Net::HTTPRedirection)
+      return res.body == READY_MESSAGE
+    end
+  rescue Errno::ECONNREFUSED, Errno::EBADF
+    return false
+  end
+
+  def concurrently
+    if should_use_subprocess?
+      pid = Process.fork do
+        trap(:INT) { ::Rack::Handler::WEBrick.shutdown }
+        yield
+        exit # manually exit; otherwise this sub-process will re-run the specs that haven't run yet.
+      end
+
+      at_exit do
+        Process.kill('INT', pid)
+        begin
+          Process.wait(pid)
+        rescue Errno::ECHILD
+          # ignore this error...I think it means the child process has already exited.
+        end
+      end
+    else
+      Thread.new { yield }
+    end
+  end
+
+  def should_use_subprocess?
+    # !ENV['THREADED']
+    false
+  end
+
+  def wait_until(timeout, error_message, &block)
+    start_time = Time.now
+
+    while true
+      return if yield
+      raise TimeoutError.new(error_message) if (Time.now - start_time) > timeout
+      sleep(0.05)
+    end
+  end
+end
+

data/spec/support/memory_cache.rb

@@ -0,0 +1,15 @@
+class MemoryCache
+  attr_reader :memory
+
+  def initialize
+    @memory = {}
+  end
+
+  def get(request)
+    memory[request]
+  end
+
+  def set(request, response)
+    memory[request] = response
+  end
+end

data/spec/support/server.rb

@@ -0,0 +1,116 @@
+#!/usr/bin/env ruby
+require 'json'
+require 'zlib'
+require 'sinatra/base'
+require 'rack/typhoeus'
+
+TESTSERVER = Sinatra.new do
+  set :logging, false
+  use Rack::Typhoeus::Middleware::ParamsDecoder
+
+  fail_count = 0
+
+  post '/file' do
+    {
+      'content-type' => params[:file][:type],
+      'filename' => params[:file][:filename],
+      'content' => params[:file][:tempfile].read,
+      'request-content-type' => request.env['CONTENT_TYPE']
+    }.to_json
+  end
+
+  get '/multiple-headers' do
+    [200, { 'Set-Cookie' => %w[ foo bar ], 'Content-Type' => 'text/plain' }, ['']]
+  end
+
+  get '/cookies-test' do
+    [200, { 'Set-Cookie' => %w(foo=bar bar=foo), 'Content-Type' => 'text/plain' }, ['']]
+  end
+
+  get '/cookies-test2' do
+    [200, { 'Set-Cookie' => %w(foo2=bar bar2=foo), 'Content-Type' => 'text/plain' }, ['']]
+  end
+
+  get '/fail/:number' do
+    if fail_count >= params[:number].to_i
+      "ok"
+    else
+      fail_count += 1
+      error 500, "oh noes!"
+    end
+  end
+
+  get '/fail_forever' do
+    error 500, "oh noes!"
+  end
+
+  get '/redirect' do
+    redirect '/'
+  end
+
+  get '/bad_redirect' do
+    redirect '/bad_redirect'
+  end
+
+  get '/auth_basic/:username/:password' do
+    @auth ||=  Rack::Auth::Basic::Request.new(request.env)
+    # Check that we've got a basic auth, and that it's credentials match the ones
+    # provided in the request
+    if @auth.provided? && @auth.basic? && @auth.credentials == [ params[:username], params[:password] ]
+      # auth is valid - confirm it
+      true
+    else
+      # invalid auth - request the authentication
+      response['WWW-Authenticate'] = %(Basic realm="Testing HTTP Auth")
+      throw(:halt, [401, "Not authorized\n"])
+    end
+  end
+
+  get '/auth_ntlm' do
+    # we're just checking for the existence if NTLM auth header here. It's validation
+    # is too troublesome and really doesn't bother is much, it's up to libcurl to make
+    # it valid
+    response['WWW-Authenticate'] = 'NTLM'
+    is_ntlm_auth = /^NTLM/ =~ request.env['HTTP_AUTHORIZATION']
+    true if is_ntlm_auth
+    throw(:halt, [401, "Not authorized\n"]) if !is_ntlm_auth
+  end
+
+  get '/gzipped' do
+    req_env = request.env.to_json
+    z = Zlib::Deflate.new
+    gzipped_env = z.deflate(req_env, Zlib::FINISH)
+    z.close
+    response['Content-Encoding'] = 'gzip'
+    gzipped_env
+  end
+
+  get '/**' do
+    sleep params["delay"].to_i if params.has_key?("delay")
+    request.env.merge!(:body => request.body.read).to_json
+  end
+
+  head '/**' do
+    sleep params["delay"].to_i if params.has_key?("delay")
+  end
+
+  put '/**' do
+    request.env.merge!(:body => request.body.read).to_json
+  end
+
+  post '/**' do
+    request.env.merge!(:body => request.body.read).to_json
+  end
+
+  delete '/**' do
+    request.env.merge!(:body => request.body.read).to_json
+  end
+
+  patch '/**' do
+    request.env.merge!(:body => request.body.read).to_json
+  end
+
+  options '/**' do
+    request.env.merge!(:body => request.body.read).to_json
+  end
+end