em-http-request 1.1.2 → 1.1.7

data/lib/em-http/http_encoding.rb

@@ -46,12 +46,19 @@ module EventMachine
       end
     end
 
-    def encode_request(method, uri, query, proxy)
+    def encode_request(method, uri, query, connopts)
       query = encode_query(uri, query)
 
       # Non CONNECT proxies require that you provide the full request
       # uri in request header, as opposed to a relative path.
-      query = uri.join(query) if proxy
+      # Don't modify the header with CONNECT proxies. It's unneeded and will
+      # cause 400 Bad Request errors with many standard setups.
+      if connopts.proxy && !connopts.connect_proxy?
+        query = uri.join(query)
+        # Drop the userinfo, it's been converted to a header and won't be
+        # accepted by the proxy
+        query.userinfo = nil
+      end
 
       HTTP_REQUEST_HEADER % [method.to_s.upcase, query]
     end
@@ -112,7 +119,7 @@ module EventMachine
     #     String - custom auth string (OAuth, etc)
     def encode_auth(k,v)
       if v.is_a? Array
-        FIELD_ENCODING % [k, ["Basic", Base64.encode64(v.join(":")).split.join].join(" ")]
+        FIELD_ENCODING % [k, ["Basic", Base64.strict_encode64(v.join(":")).split.join].join(" ")]
       else
         encode_field(k,v)
       end

data/lib/em-http/http_header.rb

@@ -2,13 +2,13 @@ module EventMachine
   # A simple hash is returned for each request made by HttpClient with the
   # headers that were given by the server for that request.
   class HttpResponseHeader < Hash
-    # The reason returned in the http response ("OK","File not found",etc.)
+    # The reason returned in the http response (string - e.g. "OK")
     attr_accessor :http_reason
 
-    # The HTTP version returned.
+    # The HTTP version returned (string - e.g. "1.1")
     attr_accessor :http_version
 
-    # The status code (as a string!)
+    # The status code (integer - e.g. 200)
     attr_accessor :http_status
 
     # Raw headers
@@ -23,9 +23,9 @@ module EventMachine
       self[HttpClient::LAST_MODIFIED]
     end
 
-    # HTTP response status as an integer
+    # HTTP response status
     def status
-      @status ||= Integer(http_status) rescue 0
+      Integer(http_status) rescue 0
     end
 
     # Length of content as an integer, or nil if chunked/unspecified

data/lib/em-http/middleware/json_response.rb

@@ -7,7 +7,7 @@ module EventMachine
         begin
           body = MultiJson.load(resp.response)
           resp.response = body
-        rescue Exception => e
+        rescue => e
         end
       end
     end

data/lib/em-http/version.rb

@@ -1,5 +1,5 @@
 module EventMachine
   class HttpRequest
-    VERSION = "1.1.2"
+    VERSION = "1.1.7"
   end
 end

data/lib/em/io_streamer.rb

@@ -0,0 +1,49 @@
+require 'em/streamer'
+
+# similar to EventMachine::FileStreamer, but for any IO object
+module EventMachine
+  class IOStreamer
+    include Deferrable
+    CHUNK_SIZE = 16384
+
+    # @param [EventMachine::Connection] connection
+    # @param [IO] io Data source
+    # @param [Integer] Data size
+    #
+    # @option opts [Boolean] :http_chunks (false) Use HTTP 1.1 style chunked-encoding semantics.
+    def initialize(connection, io, opts = {})
+      @connection = connection
+      @io = io
+      @http_chunks = opts[:http_chunks]
+
+      @buff = String.new
+      @io.binmode if @io.respond_to?(:binmode)
+      stream_one_chunk
+    end
+
+  private
+
+    # Used internally to stream one chunk at a time over multiple reactor ticks
+    # @private
+    def stream_one_chunk
+      loop do
+        if @io.eof?
+          @connection.send_data "0\r\n\r\n" if @http_chunks
+          succeed
+          break
+        end
+
+        if @connection.respond_to?(:get_outbound_data_size) && (@connection.get_outbound_data_size > FileStreamer::BackpressureLevel)
+          EventMachine::next_tick { stream_one_chunk }
+          break
+        end
+
+        if @io.read(CHUNK_SIZE, @buff)
+          @connection.send_data("#{@buff.length.to_s(16)}\r\n") if @http_chunks
+          @connection.send_data(@buff)
+          @connection.send_data("\r\n") if @http_chunks
+        end
+      end
+    end
+  end
+end

data/spec/client_spec.rb

@@ -51,7 +51,7 @@ describe EventMachine::HttpRequest do
     EventMachine.run {
       lambda {
       EventMachine::HttpRequest.new('random?text').get
-    }.should raise_error
+    }.should raise_error(Addressable::URI::InvalidURIError)
 
     EM.stop
     }
@@ -256,6 +256,19 @@ describe EventMachine::HttpRequest do
     }
   end
 
+  xit "should support expect-continue header" do
+    EventMachine.run {
+      http = EventMachine::HttpRequest.new('http://127.0.0.1:8090').post :body => "data", :head => { 'expect' => '100-continue' }
+
+      http.errback { failed(http) }
+      http.callback {
+        http.response_header.status.should == 200
+        http.response.should == "data"
+        EventMachine.stop
+      }
+    }
+  end
+
   it "should perform successful GET with custom header" do
     EventMachine.run {
       http = EventMachine::HttpRequest.new('http://127.0.0.1:8090/').get :head => {'if-none-match' => 'evar!'}
@@ -386,7 +399,7 @@ describe EventMachine::HttpRequest do
     }
   end
 
-  it "should detect gzip encoding" do
+  it "should auto-detect and decode gzip encoding" do
     EventMachine.run {
 
       http = EventMachine::HttpRequest.new('http://127.0.0.1:8090/gzip').get :head => {"accept-encoding" => "gzip, compressed"}
@@ -446,6 +459,36 @@ describe EventMachine::HttpRequest do
     }
   end
 
+  it "should default to requesting compressed response" do
+    EventMachine.run {
+
+      http = EventMachine::HttpRequest.new('http://127.0.0.1:8090/echo_accept_encoding').get
+
+      http.errback { failed(http) }
+      http.callback {
+        http.response_header.status.should == 200
+        http.response.should == "gzip, compressed"
+
+        EventMachine.stop
+      }
+    }
+  end
+
+  it "should default to requesting compressed response" do
+    EventMachine.run {
+
+      http = EventMachine::HttpRequest.new('http://127.0.0.1:8090/echo_accept_encoding').get :compressed => false
+
+      http.errback { failed(http) }
+      http.callback {
+        http.response_header.status.should == 200
+        http.response.should == ""
+
+        EventMachine.stop
+      }
+    }
+  end
+
   it "should timeout after 0.1 seconds of inactivity" do
     EventMachine.run {
       t = Time.now.to_i
@@ -759,6 +802,28 @@ describe EventMachine::HttpRequest do
     }
   end
 
+  it "streams POST request from disk via Pathname" do
+    EventMachine.run {
+      http = EventMachine::HttpRequest.new('http://127.0.0.1:8090/').post :body => Pathname.new('spec/fixtures/google.ca')
+      http.errback { failed(http) }
+      http.callback {
+        http.response.should match('google')
+        EventMachine.stop
+      }
+    }
+  end
+
+  it "streams POST request from IO object" do
+    EventMachine.run {
+      http = EventMachine::HttpRequest.new('http://127.0.0.1:8090/').post :body => StringIO.new(File.read('spec/fixtures/google.ca'))
+      http.errback { failed(http) }
+      http.callback {
+        http.response.should match('google')
+        EventMachine.stop
+      }
+    }
+  end
+
   it "should reconnect if connection was closed between requests" do
     EventMachine.run {
       conn = EM::HttpRequest.new('http://127.0.0.1:8090/')
@@ -848,6 +913,33 @@ describe EventMachine::HttpRequest do
     }
   end
 
+  it "should close connection on invalid HTTP response" do
+    EventMachine.run {
+      response =<<-HTTP.gsub(/^ +/, '').strip
+        HTTP/1.1 403 Forbidden
+        Content-Type: text/plain
+        Content-Length: 13
+
+        Access Denied
+
+        HTTP/1.1 403 Forbidden
+        Content-Type: text/plain
+        Content-Length: 13
+
+        Access Denied
+      HTTP
+
+      @s = StubServer.new(response)
+      lambda {
+        conn = EventMachine::HttpRequest.new('http://127.0.0.1:8081/')
+        req = conn.get
+        req.errback { failed(http) }
+        req.callback { EM.stop }
+      }.should_not raise_error
+
+    }
+  end
+
   context "User-Agent" do
     it 'should default to "EventMachine HttpClient"' do
       EventMachine.run {
@@ -885,4 +977,24 @@ describe EventMachine::HttpRequest do
       }
     end
   end
+
+  context "IPv6" do
+    it "should perform successful GET" do
+      EventMachine.run {
+        @s = StubServer.new({
+          response: "HTTP/1.1 200 OK\r\n\r\nHello IPv6",
+          port: 8091,
+          host: '::1',
+        })
+        http = EventMachine::HttpRequest.new('http://[::1]:8091/').get
+
+        http.errback { failed(http) }
+        http.callback {
+          http.response_header.status.should == 200
+          http.response.should match(/Hello IPv6/)
+          EventMachine.stop
+        }
+      }
+    end
+  end
 end

data/spec/dns_spec.rb

@@ -7,7 +7,7 @@ describe EventMachine::HttpRequest do
       http = EventMachine::HttpRequest.new('http://127.0.0.1:8090/redirect/badhost', :connect_timeout => 0.1).get :redirects => 1
       http.callback { failed(http) }
       http.errback {
-        http.error.should match('unable to resolve server address')
+        http.error.should match(/unable to resolve (server |)address/)
         EventMachine.stop
       }
     }
@@ -31,7 +31,7 @@ describe EventMachine::HttpRequest do
       http = EventMachine::HttpRequest.new('http://somethinglocal/', :connect_timeout => 0.1).get
       http.callback { failed(http) }
       http.errback {
-        http.error.should match(/unable to resolve server address/)
+        http.error.should match(/unable to resolve (server |)address/)
         http.response_header.status.should == 0
         EventMachine.stop
       }

data/spec/external_spec.rb

@@ -80,11 +80,11 @@ requires_connection do
     end
 
     it "should detect deflate encoding" do
-      pending "need an endpoint which supports deflate.. MSN is no longer"
+      # pending "need an endpoint which supports deflate.. MSN is no longer"
       EventMachine.run {
 
         options = {:head => {"accept-encoding" => "deflate"}, :redirects => 5}
-        http = EventMachine::HttpRequest.new('http://www.msn.com').get options
+        http = EventMachine::HttpRequest.new('http://www.libpng.org/').get options
 
         http.errback { failed(http) }
         http.callback {
@@ -135,7 +135,7 @@ requires_connection do
 
       it "should work with keep-alive servers" do
         EventMachine.run {
-          http = EventMachine::HttpRequest.new('http://mexicodiario.com/touch.public.json.php').get :keepalive => true
+          http = EventMachine::HttpRequest.new('https://github.com/igrigorik/em-http-request').get :keepalive => true
 
           http.errback { failed(http) }
           http.callback {

data/spec/gzip_spec.rb

@@ -56,6 +56,29 @@ describe EventMachine::HttpDecoders::GZip do
     decompressed.size.should eq(32907)
   end
 
+  it "should not care how many chunks the file is split up into" do
+    examples = [
+      ["\x1F", "\x8B", "\b", "\x00", "\x00", "\x00", "\x00", "\x00", "\x00", "\x00", "\xF3", "\xCB", "/", "Q", "p\xCB/\xCDK\x01\x00M\x8Ct\xB1\t\x00\x00\x00"],
+      ["\x1F", "\x8B", "\b", "\x00", "\x00", "\x00", "\x00", "\x00\x00\x00\xF3\xCB/Qp\xCB/\xCDK\x01\x00M\x8Ct\xB1\t\x00\x00\x00"]
+    ]
+
+    examples.each do |example|
+      decompressed = ""
+
+      gz = EventMachine::HttpDecoders::GZip.new do |data|
+        decompressed << data
+      end
+
+      example.each do |bytes|
+        gz << bytes
+      end
+
+      gz.finalize!
+
+      decompressed.should eq("Not Found")
+    end
+  end
+
   it "should fail with a DecoderError if not a gzip file" do
     not_a_gzip = ["1f8c08089668a650000"].pack("H*")
     header = EventMachine::HttpDecoders::GZipHeader.new

data/spec/helper.rb

@@ -1,3 +1,4 @@
+require 'spec_helper'
 require 'rubygems'
 require 'bundler/setup'
 

data/spec/http_proxy_spec.rb

@@ -1,90 +1,268 @@
 require 'helper'
 
+shared_examples "*_PROXY var (through proxy)" do
+  it "should use HTTP proxy" do
+    EventMachine.run {
+      http = EventMachine::HttpRequest.new("#{proxy_test_scheme}://127.0.0.1:8090/?q=test").get
+
+      http.errback { failed(http) }
+      http.callback {
+        http.response_header.status.should == 200
+        http.response_header.should_not include("X_PROXY_AUTH")
+        http.response.should match('test')
+        EventMachine.stop
+      }
+    }
+  end
+end
+
+shared_examples "*_PROXY var (testing var)" do
+  subject { HttpConnectionOptions.new("#{proxy_test_scheme}://example.com", {}) }
+  it { expect(subject.proxy_from_env).to eq({ :host => "127.0.0.1", :port => 8083, :type => :http }) }
+  it { expect(subject.host).to eq "127.0.0.1" }
+  it { expect(subject.port).to be 8083 }
+  it do
+    case proxy_test_scheme.to_sym
+    when :http
+      expect(subject.http_proxy?).to be_truthy
+    when :https
+      expect(subject.connect_proxy?).to be_truthy
+    end
+  end
+end
+
 describe EventMachine::HttpRequest do
 
   context "connections via" do
-    let(:proxy) { {:proxy => { :host => '127.0.0.1', :port => 8083 }} }
-    let(:authenticated_proxy) { {:proxy => { :host => '127.0.0.1', :port => 8083, :authorization => ["user", "name"] } } }
-
-    it "should use HTTP proxy" do
-      EventMachine.run {
-        http = EventMachine::HttpRequest.new('http://127.0.0.1:8090/?q=test', proxy).get
-
-        http.errback { failed(http) }
-        http.callback {
-          http.response_header.status.should == 200
-          http.response_header.should_not include("X_PROXY_AUTH")
-          http.response.should match('test')
-          EventMachine.stop
+    context "without *_PROXY env" do
+      let(:proxy) { {:proxy => { :host => '127.0.0.1', :port => 8083 }} }
+      let(:authenticated_proxy) { {:proxy => { :host => '127.0.0.1', :port => 8083, :authorization => ["user", "name"] } } }
+
+      it "should use HTTP proxy" do
+        EventMachine.run {
+          http = EventMachine::HttpRequest.new('http://127.0.0.1:8090/?q=test', proxy).get
+
+          http.errback { failed(http) }
+          http.callback {
+            http.response_header.status.should == 200
+            http.response_header.should_not include("X_PROXY_AUTH")
+            http.response.should match('test')
+            EventMachine.stop
+          }
         }
-      }
-    end
+      end
 
-    it "should use HTTP proxy with authentication" do
-      EventMachine.run {
-        http = EventMachine::HttpRequest.new('http://127.0.0.1:8090/proxyauth?q=test', authenticated_proxy).get
+      it "should use HTTP proxy with authentication" do
+        EventMachine.run {
+          http = EventMachine::HttpRequest.new('http://127.0.0.1:8090/proxyauth?q=test', authenticated_proxy).get
 
-        http.errback { failed(http) }
-        http.callback {
-          http.response_header.status.should == 200
-          http.response_header['X_PROXY_AUTH'].should == "Proxy-Authorization: Basic dXNlcjpuYW1l"
-          http.response.should match('test')
-          EventMachine.stop
+          http.errback { failed(http) }
+          http.callback {
+            http.response_header.status.should == 200
+            http.response_header['X_PROXY_AUTH'].should == "Proxy-Authorization: Basic dXNlcjpuYW1l"
+            http.response.should match('test')
+            EventMachine.stop
+          }
         }
-      }
-    end
+      end
 
-    it "should send absolute URIs to the proxy server" do
-      EventMachine.run {
+      it "should send absolute URIs to the proxy server" do
+        EventMachine.run {
 
-        http = EventMachine::HttpRequest.new('http://127.0.0.1:8090/?q=test', proxy).get
+          http = EventMachine::HttpRequest.new('http://127.0.0.1:8090/?q=test', proxy).get
 
-        http.errback { failed(http) }
-        http.callback {
-          http.response_header.status.should == 200
+          http.errback { failed(http) }
+          http.callback {
+            http.response_header.status.should == 200
 
-          # The test proxy server gives the requested uri back in this header
-          http.response_header['X_THE_REQUESTED_URI'].should == 'http://127.0.0.1:8090/?q=test'
-          http.response_header['X_THE_REQUESTED_URI'].should_not == '/?q=test'
-          http.response.should match('test')
-          EventMachine.stop
+            # The test proxy server gives the requested uri back in this header
+            http.response_header['X_THE_REQUESTED_URI'].should == 'http://127.0.0.1:8090/?q=test'
+            http.response_header['X_THE_REQUESTED_URI'].should_not == '/?q=test'
+            http.response.should match('test')
+            EventMachine.stop
+          }
         }
-      }
-    end
+      end
+
+      it "should strip basic auth from before the host in URI sent to proxy" do
+        EventMachine.run {
 
-    it "should include query parameters specified in the options" do
-      EventMachine.run {
-        http = EventMachine::HttpRequest.new('http://127.0.0.1:8090/', proxy).get :query => { 'q' => 'test' }
+          http = EventMachine::HttpRequest.new('http://user:pass@127.0.0.1:8090/echo_authorization_header', proxy).get
 
-        http.errback { failed(http) }
-        http.callback {
-          http.response_header.status.should == 200
-          http.response.should match('test')
-          EventMachine.stop
+          http.errback { failed(http) }
+          http.callback {
+            http.response_header.status.should == 200
+            # The test proxy server gives the requested uri back in this header
+            http.response_header['X_THE_REQUESTED_URI'].should == 'http://127.0.0.1:8090/echo_authorization_header'
+            # Ensure the basic auth was converted to a header correctly
+            http.response.should match('authorization:Basic dXNlcjpwYXNz')
+            EventMachine.stop
+          }
         }
-      }
-    end
+      end
 
-    it "should use HTTP proxy while redirecting" do
-      EventMachine.run {
-        http = EventMachine::HttpRequest.new('http://127.0.0.1:8090/redirect', proxy).get :redirects => 1
+      it "should include query parameters specified in the options" do
+        EventMachine.run {
+          http = EventMachine::HttpRequest.new('http://127.0.0.1:8090/', proxy).get :query => { 'q' => 'test' }
 
-        http.errback { failed(http) }
-        http.callback {
-          http.response_header.status.should == 200
+          http.errback { failed(http) }
+          http.callback {
+            http.response_header.status.should == 200
+            http.response.should match('test')
+            EventMachine.stop
+          }
+        }
+      end
+
+      it "should use HTTP proxy while redirecting" do
+        EventMachine.run {
+          http = EventMachine::HttpRequest.new('http://127.0.0.1:8090/redirect', proxy).get :redirects => 1
 
-          http.response_header['X_THE_REQUESTED_URI'].should == 'http://127.0.0.1:8090/gzip'
-          http.response_header['X_THE_REQUESTED_URI'].should_not == '/redirect'
+          http.errback { failed(http) }
+          http.callback {
+            http.response_header.status.should == 200
 
-          http.response_header["CONTENT_ENCODING"].should == "gzip"
-          http.response.should == "compressed"
-          http.last_effective_url.to_s.should == 'http://127.0.0.1:8090/gzip'
-          http.redirects.should == 1
+            http.response_header['X_THE_REQUESTED_URI'].should == 'http://127.0.0.1:8090/gzip'
+            http.response_header['X_THE_REQUESTED_URI'].should_not == '/redirect'
 
-          EventMachine.stop
+            http.response_header["CONTENT_ENCODING"].should == "gzip"
+            http.response.should == "compressed"
+            http.last_effective_url.to_s.should == 'http://127.0.0.1:8090/gzip'
+            http.redirects.should == 1
+
+            EventMachine.stop
+          }
         }
-      }
+      end
+    end
+
+    context "when parsing *_PROXY var (through proxy)s" do
+      context "with $HTTP_PROXY env" do
+        let(:proxy_test_scheme) { :http }
+
+        before(:all) do
+          PROXY_ENV_VARS.each {|k| ENV.delete k }
+          ENV['HTTP_PROXY'] = 'http://127.0.0.1:8083'
+        end
+
+        include_examples "*_PROXY var (through proxy)"
+      end
+
+      context "with $http_proxy env" do
+        let(:proxy_test_scheme) { :http }
+
+        before(:all) do
+          PROXY_ENV_VARS.each {|k| ENV.delete k }
+          ENV['http_proxy'] = 'http://127.0.0.1:8083'
+        end
+
+        include_examples "*_PROXY var (through proxy)"
+      end
+
+      ## TODO: Use a Mongrel HTTP server that can handle SSL:
+      context "with $HTTPS_PROXY env", skip: "Mongrel isn't configured to handle HTTPS, currently" do
+        let(:proxy_test_scheme) { :https }
+
+        before(:all) do
+          PROXY_ENV_VARS.each {|k| ENV.delete k }
+          ENV['HTTPS_PROXY'] = 'http://127.0.0.1:8083'
+        end
+
+        include_examples "*_PROXY var (through proxy)"
+      end
+
+      ## TODO: Use a Mongrel HTTP server that can handle SSL:
+      context "with $https_proxy env", skip: "Mongrel isn't configured to handle HTTPS, currently" do
+        let(:proxy_test_scheme) { :https }
+
+        before(:all) do
+          PROXY_ENV_VARS.each {|k| ENV.delete k }
+          ENV['https_proxy'] = 'http://127.0.0.1:8083'
+        end
+
+        include_examples "*_PROXY var (through proxy)"
+      end
+
+      context "with $ALL_PROXY env" do
+        let(:proxy_test_scheme) { :http }
+
+        before(:all) do
+          PROXY_ENV_VARS.each {|k| ENV.delete k }
+          ENV['ALL_PROXY'] = 'http://127.0.0.1:8083'
+        end
+
+        include_examples "*_PROXY var (through proxy)"
+      end
     end
   end
 
+  context "when parsing *_PROXY vars" do
+    context "without a *_PROXY var" do
+      before(:all) do
+        PROXY_ENV_VARS.each {|k| ENV.delete k }
+      end
+
+      subject { HttpConnectionOptions.new("http://example.com", {}) }
+      it { expect(subject.proxy_from_env).to be_nil }
+      it { expect(subject.host).to eq "example.com" }
+      it { expect(subject.port).to be 80 }
+      it { expect(subject.http_proxy?).to be_falsey }
+      it { expect(subject.connect_proxy?).to be_falsey }
+    end
+
+    context "with $HTTP_PROXY env" do
+      let(:proxy_test_scheme) { :http }
+
+      before(:each) do
+        PROXY_ENV_VARS.each {|k| ENV.delete k }
+        ENV['HTTP_PROXY'] = 'http://127.0.0.1:8083'
+      end
+
+      include_examples "*_PROXY var (testing var)"
+    end
+
+    context "with $http_proxy env" do
+      let(:proxy_test_scheme) { :http }
+
+      before(:each) do
+        PROXY_ENV_VARS.each {|k| ENV.delete k }
+        ENV['http_proxy'] = 'http://127.0.0.1:8083'
+      end
+
+      include_examples "*_PROXY var (testing var)"
+    end
+
+    context "with $HTTPS_PROXY env" do
+      let(:proxy_test_scheme) { :https }
+
+      before(:each) do
+        PROXY_ENV_VARS.each {|k| ENV.delete k }
+        ENV['HTTPS_PROXY'] = 'http://127.0.0.1:8083'
+      end
+
+      include_examples "*_PROXY var (testing var)"
+    end
+
+    context "with $https_proxy env" do
+      let(:proxy_test_scheme) { :https }
+
+      before(:each) do
+        PROXY_ENV_VARS.each {|k| ENV.delete k }
+        ENV['https_proxy'] = 'http://127.0.0.1:8083'
+      end
+
+      include_examples "*_PROXY var (testing var)"
+    end
+
+    context "with $ALL_PROXY env" do
+      let(:proxy_test_scheme) { :https }
+
+      before(:each) do
+        PROXY_ENV_VARS.each {|k| ENV.delete k }
+        ENV['ALL_PROXY'] = 'http://127.0.0.1:8083'
+      end
+
+      include_examples "*_PROXY var (testing var)"
+    end
+  end
 end