em-http-request 1.0.0 → 1.0.1

data/.gitignore

@@ -4,3 +4,5 @@ Makefile
 mkmf.log
 Gemfile.lock
 misc
+gems/
+vendor/

data/README.md

@@ -46,9 +46,11 @@ Several higher-order Ruby projects have incorporated em-http and other Ruby HTTP
 
 ## Other libraries & applications using EM-HTTP
 
+- [VMWare CloudFoundry](https://github.com/cloudfoundry) - The open platform-as-a-service project
+- [PubSubHubbub](https://github.com/igrigorik/PubSubHubbub) - Asynchronous PubSubHubbub ruby client
+- [em-net-http](https://github.com/jfairbairn/em-net-http) - Monkeypatching Net::HTTP to play ball with EventMachine
 - [chirpstream](https://github.com/joshbuddy/chirpstream) - EM client for Twitters Chirpstream API
 - [rsolr-async](https://github.com/mwmitchell/rsolr-async) - An asynchronus connection adapter for RSolr
-- [PubSubHubbub](https://github.com/igrigorik/PubSubHubbub) - Asynchronous PubSubHubbub ruby client
 - [Firering](https://github.com/EmmanuelOga/firering) - Eventmachine powered Campfire API
 - [RDaneel](https://github.com/hasmanydevelopers/RDaneel) - Ruby crawler which respects robots.txt
 - [em-eventsource](https://github.com/AF83/em-eventsource) - EventSource client for EventMachine
@@ -57,4 +59,4 @@ Several higher-order Ruby projects have incorporated em-http and other Ruby HTTP
 
 ### License
 
-(MIT License) - Copyright (c) 2011 Ilya Grigorik
+(MIT License) - Copyright (c) 2011 Ilya Grigorik

data/em-http-request.gemspec

@@ -1,33 +1,33 @@
-# -*- encoding: utf-8 -*-
-$:.push File.expand_path("../lib", __FILE__)
-require "em-http/version"
-
-Gem::Specification.new do |s|
-  s.name        = "em-http-request"
-  s.version     = EventMachine::HttpRequest::VERSION
-
-  s.platform    = Gem::Platform::RUBY
-  s.authors     = ["Ilya Grigorik"]
-  s.email       = ["ilya@igvita.com"]
-  s.homepage    = "http://github.com/igrigorik/em-http-request"
-  s.summary     = "EventMachine based, async HTTP Request client"
-  s.description = s.summary
-  s.rubyforge_project = "em-http-request"
-
-  s.add_dependency "eventmachine", ">= 1.0.0.beta.3"
-  s.add_dependency "addressable", ">= 2.2.3"
-  s.add_dependency "http_parser.rb", ">= 0.5.2"
-  s.add_dependency "em-socksify"
-
-  s.add_development_dependency "rspec"
-  s.add_development_dependency "rake"
-  s.add_development_dependency "rack"
-  s.add_development_dependency "yajl-ruby"
-  s.add_development_dependency "cookiejar"
-  s.add_development_dependency "mongrel", "~> 1.2.0.pre2"
-
-  s.files         = `git ls-files`.split("\n")
-  s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
-  s.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
-  s.require_paths = ["lib"]
-end
+# -*- encoding: utf-8 -*-
+$:.push File.expand_path("../lib", __FILE__)
+require "em-http/version"
+
+Gem::Specification.new do |s|
+  s.name        = "em-http-request"
+  s.version     = EventMachine::HttpRequest::VERSION
+
+  s.platform    = Gem::Platform::RUBY
+  s.authors     = ["Ilya Grigorik"]
+  s.email       = ["ilya@igvita.com"]
+  s.homepage    = "http://github.com/igrigorik/em-http-request"
+  s.summary     = "EventMachine based, async HTTP Request client"
+  s.description = s.summary
+  s.rubyforge_project = "em-http-request"
+
+  s.add_dependency "eventmachine", ">= 1.0.0.beta.4"
+  s.add_dependency "addressable", ">= 2.2.3"
+  s.add_dependency "http_parser.rb", ">= 0.5.3"
+  s.add_dependency "em-socksify"
+  s.add_dependency "cookiejar"
+
+  s.add_development_dependency "rspec"
+  s.add_development_dependency "rake"
+  s.add_development_dependency "rack"
+  s.add_development_dependency "yajl-ruby"
+  s.add_development_dependency "mongrel", "~> 1.2.0.pre2"
+
+  s.files         = `git ls-files`.split("\n")
+  s.test_files    = `git ls-files -- {test,spec,features}/*`.split("\n")
+  s.executables   = `git ls-files -- bin/*`.split("\n").map{ |f| File.basename(f) }
+  s.require_paths = ["lib"]
+end

data/examples/fibered-http.rb

@@ -1,47 +1,51 @@
-$: << 'lib' << '../lib'
-
-require 'eventmachine'
-require 'em-http'
-require 'fiber'
-
-# Using Fibers in Ruby 1.9 to simulate blocking IO / IO scheduling
-# while using the async EventMachine API's
-
-def async_fetch(url)
-  f = Fiber.current
-  http = EventMachine::HttpRequest.new(url).get :timeout => 10
-
-  http.callback { f.resume(http) }
-  http.errback  { f.resume(http) }
-
-  Fiber.yield
-
-  if http.error
-    p [:HTTP_ERROR, http.error]
-  end
-
-  http
-end
-
-EventMachine.run do
-  Fiber.new{
-
-    puts "Setting up HTTP request #1"
-    data = async_fetch('http://0.0.0.0/')    
-    puts "Fetched page #1: #{data.response_header.status}"
-
-    puts "Setting up HTTP request #2"
-    data = async_fetch('http://www.yahoo.com/')
-    puts "Fetched page #2: #{data.response_header.status}"
-
-    EventMachine.stop
-  }.resume
-end
-
-puts "Done"
-
-#  Setting up HTTP request #1
-#  Fetched page #1: 302
-#  Setting up HTTP request #2
-#  Fetched page #2: 200
-#  Done
+$: << 'lib' << '../lib'
+
+require 'eventmachine'
+require 'em-http'
+require 'fiber'
+
+# Using Fibers in Ruby 1.9 to simulate blocking IO / IO scheduling
+# while using the async EventMachine API's
+
+def async_fetch(url)
+  f = Fiber.current
+  http = EventMachine::HttpRequest.new(url).get :timeout => 10
+
+  http.callback { f.resume(http) }
+  http.errback  { f.resume(http) }
+
+  Fiber.yield
+
+  if http.error
+    p [:HTTP_ERROR, http.error]
+  end
+
+  http
+end
+
+EventMachine.run do
+  Fiber.new{
+
+    puts "Setting up HTTP request #1"
+    data = async_fetch('http://0.0.0.0/')    
+    puts "Fetched page #1: #{data.response_header.status}"
+
+    puts "Setting up HTTP request #2"
+    data = async_fetch('http://www.yahoo.com/')
+    puts "Fetched page #2: #{data.response_header.status}"
+    
+    puts "Setting up HTTP request #3"
+    data = async_fetch('http://non-existing.domain/')
+    puts "Fetched page #3: #{data.response_header.status}"
+
+    EventMachine.stop
+  }.resume
+end
+
+puts "Done"
+
+#  Setting up HTTP request #1
+#  Fetched page #1: 302
+#  Setting up HTTP request #2
+#  Fetched page #2: 200
+#  Done

data/lib/em-http/client.rb

@@ -1,275 +1,307 @@
-module EventMachine
-
-  class HttpClient
-    include Deferrable
-    include HttpEncoding
-    include HttpStatus
-
-    TRANSFER_ENCODING="TRANSFER_ENCODING"
-    CONTENT_ENCODING="CONTENT_ENCODING"
-    CONTENT_LENGTH="CONTENT_LENGTH"
-    CONTENT_TYPE="CONTENT_TYPE"
-    LAST_MODIFIED="LAST_MODIFIED"
-    KEEP_ALIVE="CONNECTION"
-    SET_COOKIE="SET_COOKIE"
-    LOCATION="LOCATION"
-    HOST="HOST"
-    ETAG="ETAG"
-
-    CRLF="\r\n"
-
-    attr_accessor :state, :response
-    attr_reader   :response_header, :error, :content_charset, :req, :cookies
-
-    def initialize(conn, options)
-      @conn = conn
-      @req  = options
-
-      @stream = nil
-      @headers = nil
-      @cookies = []
-
-      reset!
-    end
-
-    def reset!
-      @response_header = HttpResponseHeader.new
-      @state = :response_header
-
-      @response = ''
-      @error = nil
-      @content_decoder = nil
-      @content_charset = nil
-    end
-
-    def last_effective_url; @req.uri; end
-    def redirects; @req.followed; end
-    def peer; @conn.peer; end
-
-    def connection_completed
-      @state = :response_header
-
-      head, body = build_request, @req.body
-      @conn.middleware.each do |m|
-        head, body = m.request(self, head, body) if m.respond_to?(:request)
-      end
-
-      send_request(head, body)
-    end
-
-    def on_request_complete
-      begin
-        @content_decoder.finalize! if @content_decoder
-      rescue HttpDecoders::DecoderError
-        on_error "Content-decoder error"
-      end
-
-      unbind
-    end
-
-    def continue?
-      @response_header.status == 100 && (@req.method == 'POST' || @req.method == 'PUT')
-    end
-
-    def finished?
-      @state == :finished || (@state == :body && @response_header.content_length.nil?)
-    end
-
-    def redirect?
-      @response_header.location && @req.follow_redirect?
-    end
-
-    def unbind(reason = nil)
-      if finished?
-        if redirect?
-
-          begin
-            @conn.middleware.each do |m|
-              m.response(self) if m.respond_to?(:response)
-            end
-
-            # one of the injected middlewares could have changed
-            # our redirect settings, check if we still want to
-            # follow the location header
-            if redirect?
-              @req.followed += 1
-              @req.set_uri(@response_header.location)
-              @conn.redirect(self)
-            else
-              succeed(self)
-            end
-
-          rescue Exception => e
-            on_error(e.message)
-          end
-        else
-          succeed(self)
-        end
-
-      else
-        on_error(reason)
-      end
-    end
-
-    def on_error(msg = nil)
-      @error = msg
-      fail(self)
-    end
-    alias :close :on_error
-
-    def stream(&blk); @stream = blk; end
-    def headers(&blk); @headers = blk; end
-
-    def normalize_body(body)
-      body.is_a?(Hash) ? form_encode_body(body) : body
-    end
-
-    def build_request
-      head    = @req.headers ? munge_header_keys(@req.headers) : {}
-      proxy   = @req.proxy
-
-      if @req.http_proxy?
-        head['proxy-authorization'] = @req.proxy[:authorization] if @req.proxy[:authorization]
-      end
-
-      # Set the cookie header if provided
-      if cookie = head['cookie']
-        @cookies << encode_cookie(cookie) if cookie
-      end
-      head['cookie'] = @cookies.compact.uniq.join("; ").squeeze(";") unless @cookies.empty?
-
-      # Set connection close unless keepalive
-      if !@req.keepalive
-        head['connection'] = 'close'
-      end
-
-      # Set the Host header if it hasn't been specified already
-      head['host'] ||= encode_host
-
-      # Set the User-Agent if it hasn't been specified
-      head['user-agent'] ||= "EventMachine HttpClient"
-
-      head
-    end
-
-    def send_request(head, body)
-      body    = normalize_body(body)
-      file    = @req.file
-      query   = @req.query
-
-      # Set the Content-Length if file is given
-      head['content-length'] = File.size(file) if file
-
-      # Set the Content-Length if body is given
-      head['content-length'] =  body.bytesize if body
-
-      # Set content-type header if missing and body is a Ruby hash
-      if not head['content-type'] and @req.body.is_a? Hash
-        head['content-type'] = 'application/x-www-form-urlencoded'
-      end
-
-      request_header ||= encode_request(@req.method, @req.uri, query, @conn.connopts.proxy)
-      request_header << encode_headers(head)
-      request_header << CRLF
-      @conn.send_data request_header
-
-      if body
-        @conn.send_data body
-      elsif @req.file
-        @conn.stream_file_data @req.file, :http_chunks => false
-      end
-    end
-
-    def on_body_data(data)
-      if @content_decoder
-        begin
-          @content_decoder << data
-        rescue HttpDecoders::DecoderError
-          on_error "Content-decoder error"
-        end
-      else
-        on_decoded_body_data(data)
-      end
-    end
-
-    def on_decoded_body_data(data)
-      data.force_encoding @content_charset if @content_charset
-      if @stream
-        @stream.call(data)
-      else
-        @response << data
-      end
-    end
-
-    def parse_response_header(header, version, status)
-      header.each do |key, val|
-        @response_header[key.upcase.gsub('-','_')] = val
-      end
-
-      @response_header.http_version = version.join('.')
-      @response_header.http_status  = status
-      @response_header.http_reason  = CODE[status] || 'unknown'
-
-      # invoke headers callback after full parse
-      # if one is specified by the user
-      @headers.call(@response_header) if @headers
-
-      unless @response_header.http_status and @response_header.http_reason
-        @state = :invalid
-        on_error "no HTTP response"
-        return
-      end
-
-      # add set-cookie's to cookie list
-      @cookies << @response_header.cookie if @response_header.cookie && @req.pass_cookies
-
-      # correct location header - some servers will incorrectly give a relative URI
-      if @response_header.location
-        begin
-          location = Addressable::URI.parse(@response_header.location)
-
-          if location.relative?
-            location = @req.uri.join(location)
-            @response_header[LOCATION] = location.to_s
-          else
-            # if redirect is to an absolute url, check for correct URI structure
-            raise if location.host.nil?
-          end
-
-        rescue
-          on_error "Location header format error"
-          return
-        end
-      end
-
-      # Fire callbacks immediately after recieving header requests
-      # if the request method is HEAD. In case of a redirect, terminate
-      # current connection and reinitialize the process.
-      if @req.method == "HEAD"
-        @state = :finished
-        return
-      end
-
-      if @response_header.chunked_encoding?
-        @state = :chunk_header
-      elsif @response_header.content_length
-        @state = :body
-      else
-        @state = :body
-      end
-
-      if @req.decoding && decoder_class = HttpDecoders.decoder_for_encoding(response_header[CONTENT_ENCODING])
-        begin
-          @content_decoder = decoder_class.new do |s| on_decoded_body_data(s) end
-        rescue HttpDecoders::DecoderError
-          on_error "Content-decoder error"
-        end
-      end
-
-      if String.method_defined?(:force_encoding) && /;\s*charset=\s*(.+?)\s*(;|$)/.match(response_header[CONTENT_TYPE])
-        @content_charset = Encoding.find($1.gsub(/^\"|\"$/, '')) rescue Encoding.default_external
-      end
-    end
-
-  end
-end
+require 'cookiejar'
+
+module EventMachine
+
+
+  class HttpClient
+    include Deferrable
+    include HttpEncoding
+    include HttpStatus
+
+    TRANSFER_ENCODING="TRANSFER_ENCODING"
+    CONTENT_ENCODING="CONTENT_ENCODING"
+    CONTENT_LENGTH="CONTENT_LENGTH"
+    CONTENT_TYPE="CONTENT_TYPE"
+    LAST_MODIFIED="LAST_MODIFIED"
+    KEEP_ALIVE="CONNECTION"
+    SET_COOKIE="SET_COOKIE"
+    LOCATION="LOCATION"
+    HOST="HOST"
+    ETAG="ETAG"
+
+    CRLF="\r\n"
+
+    attr_accessor :state, :response
+    attr_reader   :response_header, :error, :content_charset, :req, :cookies
+
+    def initialize(conn, options)
+      @conn = conn
+      @req  = options
+
+      @stream    = nil
+      @headers   = nil
+      @cookies   = []
+      @cookiejar = CookieJar.new
+
+      reset!
+    end
+
+    def reset!
+      @response_header = HttpResponseHeader.new
+      @state = :response_header
+
+      @response = ''
+      @error = nil
+      @content_decoder = nil
+      @content_charset = nil
+    end
+
+    def last_effective_url; @req.uri; end
+    def redirects; @req.followed; end
+    def peer; @conn.peer; end
+
+    def connection_completed
+      @state = :response_header
+
+      head, body = build_request, @req.body
+      @conn.middleware.each do |m|
+        head, body = m.request(self, head, body) if m.respond_to?(:request)
+      end
+
+      send_request(head, body)
+    end
+
+    def on_request_complete
+      begin
+        @content_decoder.finalize! if @content_decoder
+      rescue HttpDecoders::DecoderError
+        on_error "Content-decoder error"
+      end
+
+      unbind
+    end
+
+    def continue?
+      @response_header.status == 100 && (@req.method == 'POST' || @req.method == 'PUT')
+    end
+
+    def finished?
+      @state == :finished || (@state == :body && @response_header.content_length.nil?)
+    end
+
+    def redirect?
+      @response_header.location && @req.follow_redirect?
+    end
+
+    def unbind(reason = nil)
+      if finished?
+        if redirect?
+
+          begin
+            @conn.middleware.each do |m|
+              m.response(self) if m.respond_to?(:response)
+            end
+
+            # one of the injected middlewares could have changed
+            # our redirect settings, check if we still want to
+            # follow the location header
+            if redirect?
+              @req.followed += 1
+
+              @cookies.clear
+              @cookies = @cookiejar.get(@response_header.location).map(&:to_s) if @req.pass_cookies
+              @req.set_uri(@response_header.location)
+              @conn.redirect(self)
+            else
+              succeed(self)
+            end
+
+          rescue Exception => e
+            on_error(e.message)
+          end
+        else
+          succeed(self)
+        end
+
+      else
+        on_error(reason)
+      end
+    end
+
+    def on_error(msg = nil)
+      @error = msg
+      fail(self)
+    end
+    alias :close :on_error
+
+    def stream(&blk); @stream = blk; end
+    def headers(&blk); @headers = blk; end
+
+    def normalize_body(body)
+      body.is_a?(Hash) ? form_encode_body(body) : body
+    end
+
+    def build_request
+      head    = @req.headers ? munge_header_keys(@req.headers) : {}
+      proxy   = @req.proxy
+
+      if @req.http_proxy?
+        head['proxy-authorization'] = @req.proxy[:authorization] if @req.proxy[:authorization]
+      end
+
+      # Set the cookie header if provided
+      if cookie = head['cookie']
+        @cookies << encode_cookie(cookie) if cookie
+      end
+      head['cookie'] = @cookies.compact.uniq.join("; ").squeeze(";") unless @cookies.empty?
+
+      # Set connection close unless keepalive
+      if !@req.keepalive
+        head['connection'] = 'close'
+      end
+
+      # Set the Host header if it hasn't been specified already
+      head['host'] ||= encode_host
+
+      # Set the User-Agent if it hasn't been specified
+      head['user-agent'] ||= "EventMachine HttpClient"
+
+      head
+    end
+
+    def send_request(head, body)
+      body    = normalize_body(body)
+      file    = @req.file
+      query   = @req.query
+
+      # Set the Content-Length if file is given
+      head['content-length'] = File.size(file) if file
+
+      # Set the Content-Length if body is given,
+      # or we're doing an empty post or put
+      if body
+        head['content-length'] = body.bytesize 
+      elsif @req.method == 'POST' or @req.method == 'PUT'
+        # wont happen if body is set and we already set content-length above
+        head['content-length'] = 0
+      end
+
+      # Set content-type header if missing and body is a Ruby hash
+      if not head['content-type'] and @req.body.is_a? Hash
+        head['content-type'] = 'application/x-www-form-urlencoded'
+      end
+
+      request_header ||= encode_request(@req.method, @req.uri, query, @conn.connopts.proxy)
+      request_header << encode_headers(head)
+      request_header << CRLF
+      @conn.send_data request_header
+
+      if body
+        @conn.send_data body
+      elsif @req.file
+        @conn.stream_file_data @req.file, :http_chunks => false
+      end
+    end
+
+    def on_body_data(data)
+      if @content_decoder
+        begin
+          @content_decoder << data
+        rescue HttpDecoders::DecoderError
+          on_error "Content-decoder error"
+        end
+      else
+        on_decoded_body_data(data)
+      end
+    end
+
+    def on_decoded_body_data(data)
+      data.force_encoding @content_charset if @content_charset
+      if @stream
+        @stream.call(data)
+      else
+        @response << data
+      end
+    end
+
+    def parse_response_header(header, version, status)
+      header.each do |key, val|
+        @response_header[key.upcase.gsub('-','_')] = val
+      end
+
+      @response_header.http_version = version.join('.')
+      @response_header.http_status  = status
+      @response_header.http_reason  = CODE[status] || 'unknown'
+
+      # invoke headers callback after full parse
+      # if one is specified by the user
+      @headers.call(@response_header) if @headers
+
+      unless @response_header.http_status and @response_header.http_reason
+        @state = :invalid
+        on_error "no HTTP response"
+        return
+      end
+
+      # add set-cookie's to cookie list
+      if @response_header.cookie && @req.pass_cookies
+        [@response_header.cookie].flatten.each {|cookie| @cookiejar.set(cookie, @req.uri)}
+      end
+
+      # correct location header - some servers will incorrectly give a relative URI
+      if @response_header.location
+        begin
+          location = Addressable::URI.parse(@response_header.location)
+
+          if location.relative?
+            location = @req.uri.join(location)
+            @response_header[LOCATION] = location.to_s
+          else
+            # if redirect is to an absolute url, check for correct URI structure
+            raise if location.host.nil?
+          end
+
+        rescue
+          on_error "Location header format error"
+          return
+        end
+      end
+
+      # Fire callbacks immediately after recieving header requests
+      # if the request method is HEAD. In case of a redirect, terminate
+      # current connection and reinitialize the process.
+      if @req.method == "HEAD"
+        @state = :finished
+        return
+      end
+
+      if @response_header.chunked_encoding?
+        @state = :chunk_header
+      elsif @response_header.content_length
+        @state = :body
+      else
+        @state = :body
+      end
+
+      if @req.decoding && decoder_class = HttpDecoders.decoder_for_encoding(response_header[CONTENT_ENCODING])
+        begin
+          @content_decoder = decoder_class.new do |s| on_decoded_body_data(s) end
+        rescue HttpDecoders::DecoderError
+          on_error "Content-decoder error"
+        end
+      end
+
+      # handle malformed header - Content-Type repetitions.
+      content_type = [response_header[CONTENT_TYPE]].flatten.first
+
+      if String.method_defined?(:force_encoding) && /;\s*charset=\s*(.+?)\s*(;|$)/.match(content_type)
+        @content_charset = Encoding.find($1.gsub(/^\"|\"$/, '')) rescue Encoding.default_external
+      end
+    end
+
+    class CookieJar
+      def initialize
+        @jar = ::CookieJar::Jar.new
+      end
+
+      def set string, uri
+        @jar.set_cookie(uri, string) rescue nil # drop invalid cookies
+      end
+
+      def get uri
+        uri = URI.parse(uri) rescue nil
+        uri ? @jar.get_cookies(uri) : []
+      end
+    end # CookieJar
+  end
+end