em-http-request 1.0.2 → 1.0.3

data/.gitignore

@@ -6,3 +6,4 @@ Gemfile.lock
 misc
 gems/
 vendor/
+.idea

data/benchmarks/clients.rb

@@ -9,7 +9,7 @@ require 'rest_client'
 require 'tach'
 require 'typhoeus'
 
-url = 'http://127.0.0.1/10k.html'
+url = 'http://127.0.0.1:9292/data/10000'
 
 with_server do
   Tach.meter(100) do
@@ -119,38 +119,52 @@ with_server do
       streamly.get(url)
     end
 
-    tach('Typhoeus') do
-      Typhoeus::Request.get(url).body
+    tach('Typhoeus') do |n|
+        hydra = Typhoeus::Hydra.new( max_concurrency: 8 )
+        hydra.disable_memoization
+        count = 0
+        error = 0
+        n.times {
+            req = Typhoeus::Request.new( url )
+            req.on_complete do |res|
+                count += 1
+                error += 1 if !res.success?
+                p [count, error] if count == n
+
+            end
+            hydra.queue( req )
+        }
+        hydra.run
     end
 
   end
 end
 
 
-# +------------------------------+----------+
-# | tach                         | total    |
-# +------------------------------+----------+
-# | em-http-request (persistent) | 0.016779 |
-# +------------------------------+----------+
-# | Excon (persistent)           | 0.019606 |
-# +------------------------------+----------+
-# | curb (persistent)            | 0.022034 |
-# +------------------------------+----------+
-# | Typhoeus                     | 0.027276 |
-# +------------------------------+----------+
-# | Excon                        | 0.034482 |
-# +------------------------------+----------+
-# | StreamlyFFI (persistent)     | 0.036474 |
-# +------------------------------+----------+
-# | em-http-request              | 0.041866 |
-# +------------------------------+----------+
-# | Net::HTTP (persistent)       | 0.098379 |
-# +------------------------------+----------+
-# | Net::HTTP                    | 0.103786 |
-# +------------------------------+----------+
-# | RestClient                   | 0.111841 |
-# +------------------------------+----------+
-# | HTTParty                     | 0.118632 |
-# +------------------------------+----------+
-# | open-uri                     | 0.170172 |
-# +------------------------------+----------+
+#+------------------------------+-----------+
+#| tach                         | total     |
+#+------------------------------+-----------+
+#| em-http-request (persistent) | 0.145512  |
+#+------------------------------+-----------+
+#| Excon                        | 0.181564  |
+#+------------------------------+-----------+
+#| RestClient                   | 0.253127  |
+#+------------------------------+-----------+
+#| Net::HTTP                    | 0.294412  |
+#+------------------------------+-----------+
+#| HTTParty                     | 0.305397  |
+#+------------------------------+-----------+
+#| open-uri                     | 0.307007  |
+#+------------------------------+-----------+
+#| Net::HTTP (persistent)       | 0.313716  |
+#+------------------------------+-----------+
+#| Typhoeus                     | 0.514725  |
+#+------------------------------+-----------+
+#| curb (persistent)            | 3.981700  |
+#+------------------------------+-----------+
+#| StreamlyFFI (persistent)     | 3.989063  |
+#+------------------------------+-----------+
+#| Excon (persistent)           | 4.018761  |
+#+------------------------------+-----------+
+#| em-http-request              | 15.025291 |
+#+------------------------------+-----------+

data/benchmarks/server.rb

@@ -32,12 +32,12 @@ end
 
 def with_server(&block)
   pid = Process.fork do
-    # Benchmark::Server.run
+     Benchmark::Server.run
   end
   loop do
     sleep(1)
     begin
-      # Excon.get('http://localhost:9292/api/foo')
+      #Excon.get('http://localhost:9292/api/foo')
       break
     rescue
     end
@@ -45,4 +45,4 @@ def with_server(&block)
   yield
 ensure
   Process.kill(9, pid)
-end
+end

data/em-http-request.gemspec

@@ -1,33 +1,33 @@
-# -*- encoding: utf-8 -*-
-$:.push File.expand_path("../lib", __FILE__)
-require "em-http/version"
-
-Gem::Specification.new do |s|
-  s.name        = "em-http-request"
-  s.version     = EventMachine::HttpRequest::VERSION
-
-  s.platform    = Gem::Platform::RUBY
-  s.authors     = ["Ilya Grigorik"]
-  s.email       = ["ilya@igvita.com"]
-  s.homepage    = "http://github.com/igrigorik/em-http-request"
-  s.summary     = "EventMachine based, async HTTP Request client"
-  s.description = s.summary
-  s.rubyforge_project = "em-http-request"
-
-  s.add_dependency "eventmachine", ">= 1.0.0.beta.4"
-  s.add_dependency "addressable", ">= 2.2.3"
-  s.add_dependency "http_parser.rb", ">= 0.5.3"
-  s.add_dependency "em-socksify"
-  s.add_dependency "cookiejar"
-
-  s.add_development_dependency "rspec"
-  s.add_development_dependency "rake"
-  s.add_development_dependency "rack"
-  s.add_development_dependency "yajl-ruby"
-  s.add_development_dependency "mongrel", "~> 1.2.0.pre2"
-
-  s.files         = `git ls-files`.split("\n")
-  s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
-  s.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
-  s.require_paths = ["lib"]
-end
+# -*- encoding: utf-8 -*-
+$:.push File.expand_path("../lib", __FILE__)
+require "em-http/version"
+
+Gem::Specification.new do |s|
+  s.name        = "em-http-request"
+  s.version     = EventMachine::HttpRequest::VERSION
+
+  s.platform    = Gem::Platform::RUBY
+  s.authors     = ["Ilya Grigorik"]
+  s.email       = ["ilya@igvita.com"]
+  s.homepage    = "http://github.com/igrigorik/em-http-request"
+  s.summary     = "EventMachine based, async HTTP Request client"
+  s.description = s.summary
+  s.rubyforge_project = "em-http-request"
+
+  s.add_dependency "eventmachine", ">= 1.0.0.beta.4"
+  s.add_dependency "addressable", ">= 2.2.3"
+  s.add_dependency "http_parser.rb", ">= 0.5.3"
+  s.add_dependency "em-socksify"
+  s.add_dependency "cookiejar"
+
+  s.add_development_dependency "rspec"
+  s.add_development_dependency "rake"
+  s.add_development_dependency "rack"
+  s.add_development_dependency "yajl-ruby"
+  s.add_development_dependency "mongrel", "~> 1.2.0.pre2"
+
+  s.files         = `git ls-files`.split("\n")
+  s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
+  s.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+  s.require_paths = ["lib"]
+end

data/lib/em-http/client.rb

@@ -133,9 +133,10 @@ module EventMachine
 
     def build_request
       head    = @req.headers ? munge_header_keys(@req.headers) : {}
-
-      if @req.http_proxy?
-        head['proxy-authorization'] = @req.proxy[:authorization] if @req.proxy[:authorization]
+      
+      if @conn.connopts.http_proxy?
+        proxy = @conn.connopts.proxy
+        head['proxy-authorization'] = proxy[:authorization] if proxy[:authorization]
       end
 
       # Set the cookie header if provided
@@ -217,6 +218,7 @@ module EventMachine
     end
 
     def parse_response_header(header, version, status)
+      @response_header.raw = header
       header.each do |key, val|
         @response_header[key.upcase.gsub('-','_')] = val
       end

data/lib/em-http/decoders.rb

@@ -10,7 +10,7 @@ module EventMachine::HttpDecoders
 
   class << self
     def accepted_encodings
-      DECODERS.inject([]) { |r,d| r + d.encoding_names }
+      DECODERS.inject([]) { |r, d| r + d.encoding_names }
     end
 
     def decoder_for_encoding(encoding)
@@ -44,7 +44,7 @@ module EventMachine::HttpDecoders
       decompressed = finalize
       receive_decompressed decompressed
     end
-    
+
     private
 
     def receive_decompressed(decompressed)
@@ -91,30 +91,52 @@ module EventMachine::HttpDecoders
     end
   end
 
-  ##
-  # Oneshot decompressor, due to lack of a streaming Gzip reader
-  # implementation. We may steal code from Zliby to improve this.
-  #
-  # For now, do not put `gzip' or `compressed' in your accept-encoding
-  # header if you expect much data through the :on_response interface.
   class GZip < Base
     def self.encoding_names
       %w(gzip compressed)
     end
 
     def decompress(compressed)
-      @buf ||= ''
-      @buf += compressed
-      nil
+      @buf ||= LazyStringIO.new
+      @buf << compressed
+
+      # Zlib::GzipReader loads input in 2048 byte chunks
+      if @buf.size > 2048
+        @gzip ||= Zlib::GzipReader.new @buf
+        @gzip.readline
+      end
     end
 
     def finalize
       begin
-        Zlib::GzipReader.new(StringIO.new(@buf.to_s)).read
+        @gzip ||= Zlib::GzipReader.new @buf
+        @gzip.read
       rescue Zlib::Error
         raise DecoderError
       end
     end
+
+    class LazyStringIO
+      def initialize(string="")
+        @stream = string
+      end
+
+      def <<(string)
+        @stream << string
+      end
+
+      def read(length=nil, buffer=nil)
+        buffer ||= ""
+        length ||= 0
+        buffer << @stream[0..(length-1)]
+        @stream = @stream[length..-1]
+        buffer
+      end
+
+      def size
+        @stream.size
+      end
+    end
   end
 
   DECODERS = [Deflate, GZip]

data/lib/em-http/http_client_options.rb

@@ -1,5 +1,5 @@
 class HttpClientOptions
-  attr_reader :uri, :method, :host, :port, :proxy
+  attr_reader :uri, :method, :host, :port
   attr_reader :headers, :file, :body, :query, :path
   attr_reader :keepalive, :pass_cookies, :decoding
 
@@ -25,7 +25,6 @@ class HttpClientOptions
   end
 
   def follow_redirect?; @followed < @redirects; end
-  def http_proxy?; @proxy && [nil, :http].include?(@proxy[:type]); end
   def ssl?; @uri.scheme == "https" || @uri.port == 443; end
   def no_body?; @method == "HEAD"; end
 

data/lib/em-http/http_connection.rb

@@ -1,11 +1,13 @@
 module EventMachine
 
   module HTTPMethods
-    def get    options = {}, &blk;  setup_request(:get,   options, &blk); end
-    def head   options = {}, &blk;  setup_request(:head,  options, &blk); end
-    def delete options = {}, &blk;  setup_request(:delete,options, &blk); end
-    def put    options = {}, &blk;  setup_request(:put,   options, &blk); end
-    def post   options = {}, &blk;  setup_request(:post,  options, &blk); end
+    def get      options = {}, &blk;  setup_request(:get,     options, &blk); end
+    def head     options = {}, &blk;  setup_request(:head,    options, &blk); end
+    def delete   options = {}, &blk;  setup_request(:delete,  options, &blk); end
+    def put      options = {}, &blk;  setup_request(:put,     options, &blk); end
+    def post     options = {}, &blk;  setup_request(:post,    options, &blk); end
+    def patch    options = {}, &blk;  setup_request(:patch,   options, &blk); end
+    def options  options = {}, &blk;  setup_request(:options, options, &blk); end
   end
 
   class HttpStubConnection < Connection
@@ -180,10 +182,15 @@ module EventMachine
             @conn.reconnect(r.req.host, r.req.port)
           end
 
+          @conn.pending_connect_timeout = @connopts.connect_timeout
+          @conn.comm_inactivity_timeout = @connopts.inactivity_timeout
           @conn.callback { r.connection_completed }
         rescue EventMachine::ConnectionError => e
           @clients.pop.close(e.message)
         end
+      else
+        @deferred = true
+        @conn.close_connection
       end
     end
     alias :close :unbind

data/lib/em-http/http_connection_options.rb

@@ -11,7 +11,10 @@ class HttpConnectionOptions
 
     if bind = options[:bind]
       @bind = bind[:host] || '0.0.0.0'
-      @bind_port = bind[:port] || 0
+
+      # Eventmachine will open a UNIX socket if bind :port
+      # is explicitly set to nil
+      @bind_port = bind[:port]
     end
 
     uri = uri.kind_of?(Addressable::URI) ? uri : Addressable::URI::parse(uri.to_s)
@@ -25,4 +28,6 @@ class HttpConnectionOptions
       @port = uri.port
     end
   end
+
+  def http_proxy?; @proxy && [nil, :http].include?(@proxy[:type]); end
 end

data/lib/em-http/http_header.rb

@@ -11,6 +11,9 @@ module EventMachine
     # The status code (as a string!)
     attr_accessor :http_status
 
+    # Raw headers
+    attr_accessor :raw
+
     # E-Tag
     def etag
       self[HttpClient::ETAG]

data/lib/em-http/version.rb

@@ -1,5 +1,5 @@
 module EventMachine
   class HttpRequest
-    VERSION = "1.0.2"
+    VERSION = "1.0.3"
   end
 end

data/spec/client_fiber_spec.rb

@@ -1,23 +1,23 @@
-require 'helper'
-require 'fiber'
-
-describe EventMachine::HttpRequest do
-  context "with fibers" do
-
-    it "should be transparent to connection errors" do
-      EventMachine.run do
-        Fiber.new do
-          f = Fiber.current
-          fired = false
-          http = EventMachine::HttpRequest.new('http://non-existing.domain/', :connection_timeout => 0.1).get
-          http.callback { failed(http) }
-          http.errback { f.resume :errback }
-
-          Fiber.yield.should == :errback
-          EM.stop
-        end.resume
-      end
-    end
-
-  end
-end
+require 'helper'
+require 'fiber'
+
+describe EventMachine::HttpRequest do
+  context "with fibers" do
+
+    it "should be transparent to connection errors" do
+      EventMachine.run do
+        Fiber.new do
+          f = Fiber.current
+          fired = false
+          http = EventMachine::HttpRequest.new('http://non-existing.domain/', :connection_timeout => 0.1).get
+          http.callback { failed(http) }
+          http.errback { f.resume :errback }
+
+          Fiber.yield.should == :errback
+          EM.stop
+        end.resume
+      end
+    end
+
+  end
+end

data/spec/client_spec.rb

@@ -188,6 +188,19 @@ describe EventMachine::HttpRequest do
     }
   end
 
+  it "should perform successful PATCH" do
+    EventMachine.run {
+      http = EventMachine::HttpRequest.new('http://127.0.0.1:8090/').patch :body => "data"
+
+      http.errback { failed(http) }
+      http.callback {
+        http.response_header.status.should == 200
+        http.response.should match(/data/)
+        EventMachine.stop
+      }
+    }
+  end
+
   it "should escape body on POST" do
     EventMachine.run {
       http = EventMachine::HttpRequest.new('http://127.0.0.1:8090/').post :body => {:stuff => 'string&string'}
@@ -214,7 +227,7 @@ describe EventMachine::HttpRequest do
       }
     }
   end
-  
+
   it "should set content-length to 0 on posts with empty bodies" do
     EventMachine.run {
       http = EventMachine::HttpRequest.new('http://127.0.0.1:8090/echo_content_length_from_header').post
@@ -343,6 +356,20 @@ describe EventMachine::HttpRequest do
     }
   end
 
+  it "should return raw headers in a hash" do
+    EventMachine.run {
+      http = EventMachine::HttpRequest.new('http://127.0.0.1:8090/echo_headers').get
+
+      http.errback { failed(http) }
+      http.callback {
+        http.response_header.status.should == 200
+        http.response_header.raw['Set-Cookie'].should match('test=yes')
+        http.response_header.raw['X-Forward-Host'].should match('proxy.local')
+        EventMachine.stop
+      }
+    }
+  end
+
   it "should detect deflate encoding" do
     EventMachine.run {
 
@@ -362,18 +389,40 @@ describe EventMachine::HttpRequest do
   it "should detect gzip encoding" do
     EventMachine.run {
 
-      http = EventMachine::HttpRequest.new('http://127.0.0.1:8090/gzip').get :head => {
-      "accept-encoding" => "gzip, compressed"
-    }
+      http = EventMachine::HttpRequest.new('http://127.0.0.1:8090/gzip').get :head => {"accept-encoding" => "gzip, compressed"}
 
-    http.errback { failed(http) }
-    http.callback {
-      http.response_header.status.should == 200
-      http.response_header["CONTENT_ENCODING"].should == "gzip"
-      http.response.should == "compressed"
+      http.errback { failed(http) }
+      http.callback {
+        http.response_header.status.should == 200
+        http.response_header["CONTENT_ENCODING"].should == "gzip"
+        http.response.should == "compressed"
 
-      EventMachine.stop
+        EventMachine.stop
+      }
     }
+  end
+
+  it "should stream gzip responses" do
+    expected_response = Zlib::GzipReader.open(File.dirname(__FILE__) + "/fixtures/gzip-sample.gz") { |f| f.read }
+    actual_response = ''
+
+    EventMachine.run {
+
+      http = EventMachine::HttpRequest.new('http://127.0.0.1:8090/gzip-large').get :head => {"accept-encoding" => "gzip, compressed"}
+
+      http.errback { failed(http) }
+      http.callback {
+        http.response_header.status.should == 200
+        http.response_header["CONTENT_ENCODING"].should == "gzip"
+        http.response.should == ''
+
+        actual_response.should == expected_response
+
+        EventMachine.stop
+      }
+      http.stream do |chunk|
+        actual_response << chunk
+      end
     }
   end
 
@@ -633,6 +682,9 @@ describe EventMachine::HttpRequest do
   end
 
   it "should get the body without Content-Length" do
+    pending "blocked on new http_parser.rb release"
+    # https://github.com/igrigorik/em-http-request/issues/168
+
     EventMachine.run {
       @s = StubServer.new("HTTP/1.1 200 OK\r\n\r\nFoo")
 
@@ -709,6 +761,24 @@ describe EventMachine::HttpRequest do
     }
   end
 
+  it "should reconnect if connection was closed between requests" do
+    EventMachine.run {
+      conn = EM::HttpRequest.new('http://127.0.0.1:8090/', :inactivity_timeout => 0.5)
+      req = conn.get :keepalive => true
+
+      req.callback do
+        EM.add_timer(1) do
+          req = conn.get
+
+          req.callback do
+            req.response_header.status.should == 200
+            EventMachine.stop
+          end
+        end
+      end
+    }
+  end
+
   it 'should handle malformed Content-Type header repetitions' do
     EventMachine.run {
       response =<<-HTTP.gsub(/^ +/, '').strip