em-http-request 1.0.0.beta.3 → 1.0.0.beta.4

data/Gemfile

@@ -1,3 +1,14 @@
 source "http://rubygems.org"
 
-gemspec
+gemspec
+
+group :benchmark do
+  gem 'httparty'
+  gem 'rest-client'
+  gem 'tach', '0.0.8'
+  gem 'typhoeus'
+  gem 'sinatra'
+  gem 'streamly_ffi'
+  gem 'curb'
+  gem 'excon'
+end

data/README.md

@@ -11,8 +11,8 @@ Async (EventMachine) HTTP client, with support for:
 - HTTP proxy and SOCKS5 support
 - Basic Auth & OAuth
 - Connection-level & Global middleware support
-- Ryan Dahl's HTTP parser via [http_parser.rb](https://github.com/tmm1/http_parser.rb)
-- Works wherever EventMachine runs
+- HTTP parser via [http_parser.rb](https://github.com/tmm1/http_parser.rb)
+- Works wherever EventMachine runs: Rubinius, JRuby, MRI
 
 ## Getting started
 

data/Rakefile

@@ -6,4 +6,5 @@ require 'rspec/core/rake_task'
 desc "Run all RSpec tests"
 RSpec::Core::RakeTask.new(:spec)
 
-task :default => :spec
+task :default => :spec
+task :test => [:spec]

data/benchmarks/clients.rb

@@ -0,0 +1,156 @@
+$: << './benchmarks'
+require 'server'
+
+require 'excon'
+require 'httparty'
+require 'net/http'
+require 'open-uri'
+require 'rest_client'
+require 'tach'
+require 'typhoeus'
+
+url = 'http://127.0.0.1/10k.html'
+
+with_server do
+  Tach.meter(100) do
+
+    tach('curb (persistent)') do |n|
+      curb = Curl::Easy.new
+
+      n.times do
+        curb.url = url
+        curb.http_get
+        curb.body_str
+      end
+    end
+
+    tach('em-http-request') do |n|
+      EventMachine.run {
+        count = 0
+        error = 0
+
+        n.times do
+          http = EventMachine::HttpRequest.new(url).get
+
+          http.callback {
+            count += 1
+            if count == n
+              p [count, error]
+              EM.stop
+            end
+          }
+
+          http.errback {
+            count += 1
+            error += 1
+            if count == n
+              p [count, error]
+              EM.stop
+            end
+          }
+        end
+      }
+    end
+
+    tach('em-http-request (persistent)') do |n|
+      EventMachine.run {
+        count = 0
+        error = 0
+
+        conn = EventMachine::HttpRequest.new(url)
+
+        n.times do
+          http = conn.get :keepalive => true
+          http.callback {
+            count += 1
+            if count == n
+              p [count, error]
+              EM.stop
+            end
+          }
+
+          http.errback {
+            count += 1
+            error += 1
+            if count == n
+              p [count, error]
+              EM.stop
+            end
+          }
+        end
+      }
+    end
+
+    tach('Excon') do
+      Excon.get(url).body
+    end
+
+    excon = Excon.new(url)
+    tach('Excon (persistent)') do
+      excon.request(:method => 'get').body
+    end
+
+    tach('HTTParty') do
+      HTTParty.get(url).body
+    end
+
+    uri = Addressable::URI.parse(url)
+    tach('Net::HTTP') do
+      Net::HTTP.start(uri.host, uri.port) {|http| http.get(uri.path).body }
+    end
+
+    uri = Addressable::URI.parse(url)
+    Net::HTTP.start(uri.host, uri.port) do |http|
+      tach('Net::HTTP (persistent)') do
+        http.get(uri.path).body
+      end
+    end
+
+    tach('open-uri') do
+      open(url).read
+    end
+
+    tach('RestClient') do
+      RestClient.get(url)
+    end
+
+    streamly = StreamlyFFI::Connection.new
+    tach('StreamlyFFI (persistent)') do
+      streamly.get(url)
+    end
+
+    tach('Typhoeus') do
+      Typhoeus::Request.get(url).body
+    end
+
+  end
+end
+
+
+# +------------------------------+----------+
+# | tach                         | total    |
+# +------------------------------+----------+
+# | em-http-request (persistent) | 0.016779 |
+# +------------------------------+----------+
+# | Excon (persistent)           | 0.019606 |
+# +------------------------------+----------+
+# | curb (persistent)            | 0.022034 |
+# +------------------------------+----------+
+# | Typhoeus                     | 0.027276 |
+# +------------------------------+----------+
+# | Excon                        | 0.034482 |
+# +------------------------------+----------+
+# | StreamlyFFI (persistent)     | 0.036474 |
+# +------------------------------+----------+
+# | em-http-request              | 0.041866 |
+# +------------------------------+----------+
+# | Net::HTTP (persistent)       | 0.098379 |
+# +------------------------------+----------+
+# | Net::HTTP                    | 0.103786 |
+# +------------------------------+----------+
+# | RestClient                   | 0.111841 |
+# +------------------------------+----------+
+# | HTTParty                     | 0.118632 |
+# +------------------------------+----------+
+# | open-uri                     | 0.170172 |
+# +------------------------------+----------+

data/benchmarks/em-excon.rb

@@ -0,0 +1,87 @@
+$: << './benchmarks'
+require 'server'
+
+url = 'http://127.0.0.1/10k.html'
+
+with_server do
+  Tach.meter(100) do
+
+    excon = Excon.new(url)
+    tach('Excon (persistent)') do
+      excon.request(:method => 'get').body
+    end
+
+    tach('Excon') do
+      Excon.get(url).body
+    end
+
+    tach('em-http-request') do |n|
+      EventMachine.run {
+        count = 0
+        error = 0
+        n.times do
+          EM.next_tick do
+            http = EventMachine::HttpRequest.new(url, :connect_timeout => 1).get
+
+            http.callback {
+              count += 1
+              if count == n
+                p [count, error]
+                EM.stop
+              end
+            }
+
+            http.errback {
+              count += 1
+              error += 1
+              if count == n
+                p [count, error]
+                EM.stop
+              end
+            }
+          end
+        end
+      }
+    end
+
+    tach('em-http-request (persistent)') do |n|
+      EventMachine.run {
+        count = 0
+        error = 0
+        conn = EventMachine::HttpRequest.new(url)
+
+        n.times do
+          http = conn.get :keepalive => true
+          http.callback {
+            count += 1
+            if count == n
+              p [count, error]
+              EM.stop
+            end
+          }
+
+          http.errback {
+            count += 1
+            error += 1
+            if count == n
+              p [count, error]
+              EM.stop
+            end
+          }
+        end
+      }
+    end
+  end
+end
+
+# +------------------------------+----------+
+# | tach                         | total    |
+# +------------------------------+----------+
+# | em-http-request (persistent) | 0.018133 |
+# +------------------------------+----------+
+# | Excon (persistent)           | 0.023975 |
+# +------------------------------+----------+
+# | Excon                        | 0.032877 |
+# +------------------------------+----------+
+# | em-http-request              | 0.042891 |
+# +------------------------------+----------+

data/benchmarks/em-profile.txt

@@ -0,0 +1,65 @@
+Total: 143 samples
+      67  46.9%  46.9%      109  76.2% Kernel#gem_original_require
+      17  11.9%  58.7%       17  11.9% garbage_collector
+      12   8.4%  67.1%       12   8.4% EventMachine.connect_server
+      11   7.7%  74.8%       11   7.7% Dir.[]
+      11   7.7%  82.5%       11   7.7% File.file?
+       4   2.8%  85.3%        4   2.8% Kernel#eval
+       3   2.1%  87.4%       16  11.2% Enumerable#find
+       3   2.1%  89.5%        3   2.1% Regexp#initialize
+       2   1.4%  90.9%       16  11.2% EventMachine.run_machine
+       2   1.4%  92.3%        2   1.4% Regexp#match
+       1   0.7%  93.0%        1   0.7% FFI::DynamicLibrary.open
+       1   0.7%  93.7%        1   0.7% Hash#values
+       1   0.7%  94.4%        7   4.9% Kernel#load
+       1   0.7%  95.1%        3   2.1% MIME::Type#initialize
+       1   0.7%  95.8%        1   0.7% MatchData#captures
+       1   0.7%  96.5%       13   9.1% Module#bind_connect
+       1   0.7%  97.2%       17  11.9% Module#searcher
+       1   0.7%  97.9%        1   0.7% String#downcase
+       1   0.7%  98.6%        1   0.7% String#scan
+       1   0.7%  99.3%        1   0.7% String#strip
+       1   0.7% 100.0%        1   0.7% TCPSocket#initialize
+       0   0.0% 100.0%        1   0.7% Array#map
+       0   0.0% 100.0%       14   9.8% Array#reverse_each
+       0   0.0% 100.0%       14   9.8% Class#from_gems_in
+       0   0.0% 100.0%       14   9.8% Class#from_installed_gems
+       0   0.0% 100.0%       14   9.8% Class#load
+       0   0.0% 100.0%        1   0.7% Class#parse
+       0   0.0% 100.0%        1   0.7% Class#simplified
+       0   0.0% 100.0%        6   4.2% Enumerable#each_with_index
+       0   0.0% 100.0%        1   0.7% Excon::Connection#connect
+       0   0.0% 100.0%        1   0.7% Excon::Connection#request
+       0   0.0% 100.0%        1   0.7% Excon::Connection#socket
+       0   0.0% 100.0%        1   0.7% FFI::Library#ffi_lib
+       0   0.0% 100.0%       10   7.0% Gem::GemPathSearcher#find
+       0   0.0% 100.0%        1   0.7% Gem::GemPathSearcher#find_active
+       0   0.0% 100.0%       16  11.2% Gem::GemPathSearcher#init_gemspecs
+       0   0.0% 100.0%       16  11.2% Gem::GemPathSearcher#initialize
+       0   0.0% 100.0%       11   7.7% Gem::GemPathSearcher#matching_file?
+       0   0.0% 100.0%       11   7.7% Gem::GemPathSearcher#matching_files
+       0   0.0% 100.0%       14   9.8% Gem::SourceIndex#load_gems_in
+       0   0.0% 100.0%       14   9.8% Gem::SourceIndex#refresh!
+       0   0.0% 100.0%        1   0.7% Gem::SourceIndex#search
+       0   0.0% 100.0%        1   0.7% HttpOptions#initialize
+       0   0.0% 100.0%        1   0.7% HttpOptions#set_uri
+       0   0.0% 100.0%       14   9.8% Integer#times
+       0   0.0% 100.0%        1   0.7% Kernel#loop
+       0   0.0% 100.0%      109  76.2% Kernel#require
+       0   0.0% 100.0%        1   0.7% Module#activate
+       0   0.0% 100.0%       13   9.1% Module#connect
+       0   0.0% 100.0%        1   0.7% Module#get
+       0   0.0% 100.0%        2   1.4% Module#load_full_rubygems_library
+       0   0.0% 100.0%        5   3.5% Module#loaded_path?
+       0   0.0% 100.0%       16  11.2% Module#meter
+       0   0.0% 100.0%       16  11.2% Module#run
+       0   0.0% 100.0%       16  11.2% Module#source_index
+       0   0.0% 100.0%       30  21.0% Module#try_activate
+       0   0.0% 100.0%       17  11.9% Object#with_server
+       0   0.0% 100.0%        2   1.4% Regexp.union
+       0   0.0% 100.0%        1   0.7% TCPSocket.open
+       0   0.0% 100.0%       16  11.2% Tach::Meter#initialize
+       0   0.0% 100.0%       16  11.2% Tach::Meter#run_tach
+       0   0.0% 100.0%        1   0.7% URI::Parser#initialize
+       0   0.0% 100.0%        1   0.7% URI::Parser#initialize_regexp
+       0   0.0% 100.0%        7   4.9% YAML::EngineManager#yamler=

data/benchmarks/server.rb

@@ -0,0 +1,48 @@
+require 'excon'
+require 'httparty'
+require 'net/http'
+require 'open-uri'
+require 'rest_client'
+require 'tach'
+require 'typhoeus'
+require 'sinatra/base'
+require 'streamly_ffi'
+require 'curb'
+
+require File.join(File.expand_path(File.dirname(__FILE__)), '..', 'lib', 'em-http')
+
+module Benchmark
+  class Server < Sinatra::Base
+
+    def self.run
+      Rack::Handler::WEBrick.run(
+        Benchmark::Server.new,
+        :Port => 9292,
+        :AccessLog => [],
+        :Logger => WEBrick::Log.new(nil, WEBrick::Log::ERROR)
+      )
+    end
+
+    get '/data/:amount' do |amount|
+      'x' * amount.to_i
+    end
+
+  end
+end
+
+def with_server(&block)
+  pid = Process.fork do
+    # Benchmark::Server.run
+  end
+  loop do
+    sleep(1)
+    begin
+      # Excon.get('http://localhost:9292/api/foo')
+      break
+    rescue
+    end
+  end
+  yield
+ensure
+  Process.kill(9, pid)
+end

data/em-http-request.gemspec

@@ -14,7 +14,7 @@ Gem::Specification.new do |s|
   s.description = s.summary
   s.rubyforge_project = "em-http-request"
 
-  s.add_dependency "eventmachine"
+  s.add_dependency "eventmachine", ">= 1.0.0.beta.3"
   s.add_dependency "addressable", ">= 2.2.3"
   s.add_dependency "http_parser.rb", ">= 0.5.1"
   s.add_dependency "em-socksify"
@@ -23,6 +23,7 @@ Gem::Specification.new do |s|
   s.add_development_dependency "rake"
   s.add_development_dependency "rack"
   s.add_development_dependency "yajl-ruby"
+  s.add_development_dependency "cookiejar"
   s.add_development_dependency "mongrel", "~> 1.2.0.pre2"
 
   s.files         = `git ls-files`.split("\n")

data/examples/.gitignore

@@ -0,0 +1 @@
+twitter_auth.rb

data/examples/fibered-http.rb

@@ -9,17 +9,21 @@ def async_fetch(url)
   f = Fiber.current
   http = EventMachine::HttpRequest.new(url).get :timeout => 10
 
-  http.callback { f.resume(http) }
-  http.errback  { f.resume(http) }
-
-  return Fiber.yield
+  if http.error.empty?
+    http.callback { f.resume(http) }
+    http.errback  { f.resume(http) }
+
+    Fiber.yield
+  else
+    http
+  end
 end
 
 EventMachine.run do
   Fiber.new{
 
     puts "Setting up HTTP request #1"
-    data = async_fetch('http://www.google.com/')
+    data = async_fetch('http://www.google.moo/')
     puts "Fetched page #1: #{data.response_header.status}"
 
     puts "Setting up HTTP request #2"
@@ -36,4 +40,4 @@ puts "Done"
 #  Fetched page #1: 302
 #  Setting up HTTP request #2
 #  Fetched page #2: 200
-#  Done
+#  Done