em-http-request 1.0.3 → 1.1.0

data/Changelog.md

@@ -1,5 +1,9 @@
 # Changelog
 
+## master
+
+- User-Agent header is now removed if set to nil.
+
 ## 1.0.0.beta.1 / 2011-02-20 - The big rewrite
 
 - Switched parser from Ragel to http_parser.rb

data/README.md

@@ -36,7 +36,7 @@ Several higher-order Ruby projects have incorporated em-http and other Ruby HTTP
 - [EM-Synchrony](https://github.com/igrigorik/em-synchrony) - Collection of convenience classes and primitives to help untangle evented code (Ruby 1.9 + Fibers).
 - [Rack-Client](https://github.com/halorgium/rack-client) - Use Rack API for server, test, and client side. Supports Rack middleware!
     - [Example in action](https://gist.github.com/802391)
-- [Faraday](https://github.com/technoweenie/faraday) - Modular HTTP client library using middleware heavily inspired by Rack.
+- [Faraday](https://github.com/lostisland/faraday) - Modular HTTP client library using middleware heavily inspired by Rack.
     - [Example in action](https://gist.github.com/802395)
 
 ## Testing
@@ -59,4 +59,4 @@ Several higher-order Ruby projects have incorporated em-http and other Ruby HTTP
 
 ### License
 
-(MIT License) - Copyright (c) 2011 Ilya Grigorik
+(MIT License) - Copyright (c) 2011 Ilya Grigorik

data/em-http-request.gemspec

@@ -14,10 +14,10 @@ Gem::Specification.new do |s|
   s.description = s.summary
   s.rubyforge_project = "em-http-request"
 
-  s.add_dependency "eventmachine", ">= 1.0.0.beta.4"
-  s.add_dependency "addressable", ">= 2.2.3"
-  s.add_dependency "http_parser.rb", ">= 0.5.3"
-  s.add_dependency "em-socksify"
+  s.add_dependency "eventmachine", ">= 1.0.3"
+  s.add_dependency "addressable", ">= 2.3.4"
+  s.add_dependency "http_parser.rb", ">= 0.6.0.beta.2"
+  s.add_dependency "em-socksify", ">= 0.3"
   s.add_dependency "cookiejar"
 
   s.add_development_dependency "rspec"

data/examples/digest_auth/client.rb

@@ -0,0 +1,25 @@
+$: << 'lib' << '../../lib'
+
+require 'em-http'
+require 'em-http/middleware/digest_auth'
+
+digest_config = {
+  :username => 'digest_username',
+  :password => 'digest_password'
+}
+
+EM.run do
+
+  conn_handshake = EM::HttpRequest.new('http://localhost:3000')
+  http_handshake = conn_handshake.get
+
+  http_handshake.callback do
+    conn = EM::HttpRequest.new('http://localhost:3000')
+    conn.use EM::Middleware::DigestAuth, http_handshake.response_header['WWW_AUTHENTICATE'], digest_config
+    http = conn.get
+    http.callback do
+      puts http.response
+      EM.stop
+    end
+  end
+end

data/examples/digest_auth/server.rb

@@ -0,0 +1,28 @@
+require 'webrick'
+
+include WEBrick
+
+config = { :Realm => 'DigestAuth_REALM' }
+
+htdigest = WEBrick::HTTPAuth::Htdigest.new 'my_password_file'
+htdigest.set_passwd config[:Realm], 'digest_username', 'digest_password'
+htdigest.flush
+
+config[:UserDB] = htdigest
+
+digest_auth = WEBrick::HTTPAuth::DigestAuth.new config
+
+class TestServlet < HTTPServlet::AbstractServlet
+  def do_GET(req, res)
+    @options[0][:authenticator].authenticate req, res
+    res.body = "You are authenticated to see the super secret data\n"
+  end
+end
+
+s = HTTPServer.new(:Port => 3000)
+s.mount('/', TestServlet, {:authenticator => digest_auth})
+trap("INT") do
+  File.delete('my_password_file')
+  s.shutdown
+end
+s.start

data/examples/fibered-http.rb

@@ -9,7 +9,7 @@ require 'fiber'
 
 def async_fetch(url)
   f = Fiber.current
-  http = EventMachine::HttpRequest.new(url).get :timeout => 10
+  http = EventMachine::HttpRequest.new(url, :connect_timeout => 10, :inactivity_timeout => 20).get
 
   http.callback { f.resume(http) }
   http.errback  { f.resume(http) }

data/lib/em-http/client.rb

@@ -80,7 +80,7 @@ module EventMachine
     end
 
     def redirect?
-      @response_header.location && @req.follow_redirect?
+      @response_header.redirection? && @req.follow_redirect?
     end
 
     def unbind(reason = nil)
@@ -101,6 +101,7 @@ module EventMachine
               @cookies.clear
               @cookies = @cookiejar.get(@response_header.location).map(&:to_s) if @req.pass_cookies
               @req.set_uri(@response_header.location)
+
               @conn.redirect(self)
             else
               succeed(self)
@@ -114,7 +115,7 @@ module EventMachine
         end
 
       else
-        on_error(reason)
+        on_error(reason || 'connection closed by server')
       end
     end
 
@@ -133,7 +134,7 @@ module EventMachine
 
     def build_request
       head    = @req.headers ? munge_header_keys(@req.headers) : {}
-      
+
       if @conn.connopts.http_proxy?
         proxy = @conn.connopts.proxy
         head['proxy-authorization'] = proxy[:authorization] if proxy[:authorization]
@@ -154,7 +155,11 @@ module EventMachine
       head['host'] ||= encode_host
 
       # Set the User-Agent if it hasn't been specified
-      head['user-agent'] ||= "EventMachine HttpClient"
+      if !head.key?('user-agent')
+        head['user-agent'] = "EventMachine HttpClient"
+      elsif head['user-agent'].nil?
+        head.delete('user-agent')
+      end
 
       # Set the auth from the URI if given
       head['Authorization'] = @req.uri.userinfo.split(/:/, 2) if @req.uri.userinfo
@@ -180,7 +185,7 @@ module EventMachine
       end
 
       # Set content-type header if missing and body is a Ruby hash
-      if not head['content-type'] and @req.body.is_a? Hash
+      if !head['content-type'] and @req.body.is_a? Hash
         head['content-type'] = 'application/x-www-form-urlencoded'
       end
 
@@ -246,15 +251,17 @@ module EventMachine
       if @response_header.location
         begin
           location = Addressable::URI.parse(@response_header.location)
+          location.path = "/" if location.path.empty?
 
           if location.relative?
             location = @req.uri.join(location)
-            @response_header[LOCATION] = location.to_s
           else
             # if redirect is to an absolute url, check for correct URI structure
             raise if location.host.nil?
           end
 
+          @response_header[LOCATION] = location.to_s
+
         rescue
           on_error "Location header format error"
           return

data/lib/em-http/decoders.rb

@@ -91,52 +91,161 @@ module EventMachine::HttpDecoders
     end
   end
 
-  class GZip < Base
-    def self.encoding_names
-      %w(gzip compressed)
+  ##
+  # Partial implementation of RFC 1952 to extract the deflate stream from a gzip file
+  class GZipHeader
+    def initialize
+      @state = :begin
+      @data = ""
+      @pos = 0
     end
 
-    def decompress(compressed)
-      @buf ||= LazyStringIO.new
-      @buf << compressed
+    def finished?
+      @state == :finish
+    end
 
-      # Zlib::GzipReader loads input in 2048 byte chunks
-      if @buf.size > 2048
-        @gzip ||= Zlib::GzipReader.new @buf
-        @gzip.readline
+    def read(n, buffer)
+      if (@pos + n) <= @data.size
+        buffer << @data[@pos..(@pos + n - 1)]
+        @pos += n
+        return true
+      else
+        return false
       end
     end
 
-    def finalize
-      begin
-        @gzip ||= Zlib::GzipReader.new @buf
-        @gzip.read
-      rescue Zlib::Error
-        raise DecoderError
+    def readbyte
+      if (@pos + 1) <= @data.size
+        @pos += 1
+        @data.getbyte(@pos - 1)
       end
     end
 
-    class LazyStringIO
-      def initialize(string="")
-        @stream = string
+    def eof?
+      @pos >= @data.size
+    end
+
+    def extract_stream(compressed)
+      @data << compressed
+      pos = @pos
+
+      while !eof? && !finished?
+        buffer = ""
+
+        case @state
+        when :begin
+          break if !read(10, buffer)
+
+          if buffer.getbyte(0) != 0x1f || buffer.getbyte(1) != 0x8b
+            raise DecoderError.new("magic header not found")
+          end
+
+          if buffer.getbyte(2) != 0x08
+            raise DecoderError.new("unknown compression method")
+          end
+
+          @flags = buffer.getbyte(3)
+          if (@flags & 0xe0).nonzero?
+            raise DecoderError.new("unknown header flags set")
+          end
+
+          # We don't care about these values, I'm leaving the code for reference
+          # @time = buffer[4..7].unpack("V")[0] # little-endian uint32
+          # @extra_flags = buffer.getbyte(8)
+          # @os = buffer.getbyte(9)
+
+          @state = :extra_length
+
+        when :extra_length
+          if (@flags & 0x04).nonzero?
+            break if !read(2, buffer)
+            @extra_length = buffer.unpack("v")[0] # little-endian uint16
+            @state = :extra
+          else
+            @state = :extra
+          end
+
+        when :extra
+          if (@flags & 0x04).nonzero?
+            break if read(@extra_length, buffer)
+            @state = :name
+          else
+            @state = :name
+          end
+
+        when :name
+          if (@flags & 0x08).nonzero?
+            while !(buffer = readbyte).nil?
+              if buffer == 0
+                @state = :comment
+                break
+              end
+            end
+          else
+            @state = :comment
+          end
+
+        when :comment
+          if (@flags & 0x10).nonzero?
+            while !(buffer = readbyte).nil?
+              if buffer == 0
+                @state = :hcrc
+                break
+              end
+            end
+          else
+            @state = :hcrc
+          end
+
+        when :hcrc
+          if (@flags & 0x02).nonzero?
+            break if !read(2, buffer)
+            @state = :finish
+          else
+            @state = :finish
+          end
+        end
       end
 
-      def <<(string)
-        @stream << string
+      if finished?
+        compressed[(@pos - pos)..-1]
+      else
+        ""
       end
+    end
+  end
 
-      def read(length=nil, buffer=nil)
-        buffer ||= ""
-        length ||= 0
-        buffer << @stream[0..(length-1)]
-        @stream = @stream[length..-1]
-        buffer
+  class GZip < Base
+    def self.encoding_names
+      %w(gzip compressed)
+    end
+
+    def decompress(compressed)
+      @header ||= GZipHeader.new
+      if !@header.finished?
+        compressed = @header.extract_stream(compressed)
       end
 
-      def size
-        @stream.size
+      @zstream ||= Zlib::Inflate.new(-Zlib::MAX_WBITS)
+      @zstream.inflate(compressed)
+    rescue Zlib::Error
+      raise DecoderError
+    end
+
+    def finalize
+      if @zstream
+        if !@zstream.finished?
+          r = @zstream.finish
+        end
+        @zstream.close
+        r
+      else
+        nil
       end
+    rescue Zlib::Error
+      raise DecoderError
     end
+
   end
 
   DECODERS = [Deflate, GZip]

data/lib/em-http/http_client_options.rb

@@ -11,9 +11,9 @@ class HttpClientOptions
     @followed  = options[:followed]  ||= 0     # keep track of number of followed requests
 
     @method   = method.to_s.upcase
-    @headers  = options[:head]  || {}
+    @headers  = options[:head] || {}
     @query    = options[:query]
-    @path     = options[:path]
+
 
     @file     = options[:file]
     @body     = options[:body]
@@ -21,19 +21,20 @@ class HttpClientOptions
     @pass_cookies = options.fetch(:pass_cookies, true)  # pass cookies between redirects
     @decoding     = options.fetch(:decoding, true)      # auto-decode compressed response
 
-    set_uri(uri)
+    set_uri(uri, options[:path])
   end
 
   def follow_redirect?; @followed < @redirects; end
   def ssl?; @uri.scheme == "https" || @uri.port == 443; end
   def no_body?; @method == "HEAD"; end
 
-  def set_uri(uri)
+  def set_uri(uri, path = nil)
     uri = uri.kind_of?(Addressable::URI) ? uri : Addressable::URI::parse(uri.to_s)
+    uri.path = path if path
     uri.path = '/' if uri.path.empty?
-    uri.path = @path if @path
 
     @uri = uri
+    @path = uri.path
 
     # Make sure the ports are set as Addressable::URI doesn't
     # set the port if it isn't there

data/lib/em-http/http_connection.rb

@@ -35,6 +35,7 @@ module EventMachine
   class HttpConnection
     include HTTPMethods
     include Socksify
+    include Connectify
 
     attr_reader :deferred
     attr_accessor :error, :connopts, :uri, :conn
@@ -51,7 +52,9 @@ module EventMachine
 
     def activate_connection(client)
       begin
-        EventMachine.bind_connect(@connopts.bind, @connopts.bind_port, @connopts.host, @connopts.port, HttpStubConnection) do |conn|
+        EventMachine.bind_connect(@connopts.bind, @connopts.bind_port,
+                                  @connopts.host, @connopts.port,
+                                  HttpStubConnection) do |conn|
           post_init
 
           @deferred = false
@@ -120,7 +123,7 @@ module EventMachine
       end
 
       @p.on_message_complete = proc do
-        if not client.continue?
+        if !client.continue?
           c = @clients.shift
           c.state = :finished
           c.on_request_complete
@@ -148,8 +151,10 @@ module EventMachine
     def connection_completed
       @peer = @conn.get_peername
 
-      if @connopts.proxy && @connopts.proxy[:type] == :socks5
+      if @connopts.socks_proxy?
         socksify(client.req.uri.host, client.req.uri.port, *@connopts.proxy[:authorization]) { start }
+      elsif @connopts.connect_proxy?
+        connectify(client.req.uri.host, client.req.uri.port, *@connopts.proxy[:authorization]) { start }
       else
         start
       end
@@ -164,7 +169,7 @@ module EventMachine
       @pending.push client
     end
 
-    def unbind(reason)
+    def unbind(reason = nil)
       @clients.map { |c| c.unbind(reason) }
 
       if r = @pending.shift