em-http-request 1.0.0.beta.3 → 1.0.0.beta.4

data/examples/oauth-tweet.rb

@@ -1,50 +1,35 @@
-# Courtesy of Darcy Laycock:
-# http://gist.github.com/265261
-#
-
-require 'rubygems'
+$: << 'lib' << '../lib'
 
 require 'em-http'
-require 'oauth'
-
-# At a minimum, require 'oauth/request_proxy/em_http_request'
-# for this example, we'll use Net::HTTP like support.
-require 'oauth/client/em_http'
+require 'em-http/middleware/oauth'
+require 'em-http/middleware/json_response'
 
-# You need two things: an oauth consumer and an access token.
-# You need to generate an access token, I suggest looking elsewhere how to do that or wait for a full tutorial.
-# For a consumer key / consumer secret, signup for an app at:
-# http://twitter.com/apps/new
+require 'pp'
 
-# Edit in your details.
-CONSUMER_KEY = ""
-CONSUMER_SECRET = ""
-ACCESS_TOKEN = ""
-ACCESS_TOKEN_SECRET = ""
-
-def twitter_oauth_consumer
-  @twitter_oauth_consumer ||= OAuth::Consumer.new(CONSUMER_KEY, CONSUMER_SECRET, :site => "http://twitter.com")
-end
-
-def twitter_oauth_access_token
-  @twitter_oauth_access_token ||= OAuth::AccessToken.new(twitter_oauth_consumer, ACCESS_TOKEN, ACCESS_TOKEN_SECRET)
-end
+OAuthConfig = {
+  :consumer_key     => '',
+  :consumer_secret  => '',
+  :access_token     => '',
+  :access_token_secret => ''
+}
 
 EM.run do
+  # automatically parse the JSON response into a Ruby object
+  EventMachine::HttpRequest.use EventMachine::Middleware::JSONResponse
 
-  request = EventMachine::HttpRequest.new('http://twitter.com/statuses/update.json')
-  http = request.post(:body => {'status' => 'Hello Twitter from em-http-request with OAuth'}, :head => {"Content-Type" => "application/x-www-form-urlencoded"}) do |client|
-    twitter_oauth_consumer.sign!(client, twitter_oauth_access_token)
-  end
+  # sign the request with OAuth credentials
+  conn = EventMachine::HttpRequest.new('http://api.twitter.com/1/statuses/home_timeline.json')
+  conn.use EventMachine::Middleware::OAuth, OAuthConfig
 
+  http = conn.get
   http.callback do
-    puts "Response: #{http.response} (Code: #{http.response_header.status})"
-    EM.stop_event_loop
+    pp http.response
+    EM.stop
   end
 
   http.errback do
-    puts "Failed to post"
-    EM.stop_event_loop
+    puts "Failed retrieving user stream."
+    pp http.response
+    EM.stop
   end
-
-end
+end

data/lib/em-http.rb

@@ -10,8 +10,9 @@ require 'em-http/core_ext/bytesize'
 require 'em-http/http_connection'
 require 'em-http/http_header'
 require 'em-http/http_encoding'
-require 'em-http/http_options'
+require 'em-http/http_client_options'
+require 'em-http/http_connection_options'
 require 'em-http/client'
 require 'em-http/multi'
 require 'em-http/request'
-require 'em-http/decoders'
+require 'em-http/decoders'

data/lib/em-http/client.rb

@@ -1,4 +1,5 @@
 module EventMachine
+
   class HttpClient
     include Deferrable
     include HttpEncoding
@@ -17,17 +18,15 @@ module EventMachine
     CRLF="\r\n"
 
     attr_accessor :state, :response
-    attr_reader   :response_header, :error, :content_charset, :req
+    attr_reader   :response_header, :error, :content_charset, :req, :cookies
 
-    def initialize(conn, req, options)
+    def initialize(conn, options)
       @conn = conn
-
-      @req = req
-      @method = req.method
-      @options = options
+      @req  = options
 
       @stream = nil
       @headers = nil
+      @cookies = []
 
       reset!
     end
@@ -43,14 +42,15 @@ module EventMachine
     end
 
     def last_effective_url; @req.uri; end
-    def redirects; @req.options[:followed]; end
+    def redirects; @req.followed; end
+    def peer; @conn.peer; end
 
     def connection_completed
       @state = :response_header
 
-      head, body = build_request, @options[:body]
+      head, body = build_request, @req.body
       @conn.middleware.each do |m|
-        head, body = m.request(head, body) if m.respond_to?(:request)
+        head, body = m.request(self, head, body) if m.respond_to?(:request)
       end
 
       send_request(head, body)
@@ -66,6 +66,10 @@ module EventMachine
       unbind
     end
 
+    def continue?
+      @response_header.status == 100 && (@req.method == 'POST' || @req.method == 'PUT')
+    end
+
     def finished?
       @state == :finished || (@state == :body && @response_header.content_length.nil?)
     end
@@ -77,8 +81,26 @@ module EventMachine
     def unbind
       if finished?
         if redirect?
-          @req.options[:followed] += 1
-          @conn.redirect(self, @response_header.location)
+
+          begin
+            @conn.middleware.each do |m|
+              m.response(self) if m.respond_to?(:response)
+            end
+
+            # one of the injected middlewares could have changed
+            # our redirect settings, check if we still want to
+            # follow the location header
+            if redirect?
+              @req.followed += 1
+              @req.set_uri(@response_header.location)
+              @conn.redirect(self)
+            else
+              succeed(self)
+            end
+
+          rescue Exception => e
+            on_error(e.message)
+          end
         else
           succeed(self)
         end
@@ -101,32 +123,22 @@ module EventMachine
       body.is_a?(Hash) ? form_encode_body(body) : body
     end
 
-    def proxy?; !@options[:proxy].nil?; end
-    def http_proxy?; proxy? && [nil, :http].include?(@options[:proxy][:type]); end
-
-    def ssl?; @req.uri.scheme == "https" || @req.uri.port == 443; end
-
-    def continue?
-      @response_header.status == 100 && (@method == 'POST' || @method == 'PUT')
-    end
-
     def build_request
-      head    = @options[:head] ? munge_header_keys(@options[:head]) : {}
-      proxy   = @options[:proxy]
+      head    = @req.headers ? munge_header_keys(@req.headers) : {}
+      proxy   = @req.proxy
 
-      if http_proxy?
-        # initialize headers for the http proxy
-        head = proxy[:head] ? munge_header_keys(proxy[:head]) : {}
-        head['proxy-authorization'] = proxy[:authorization] if proxy[:authorization]
+      if @req.http_proxy?
+        head['proxy-authorization'] = @req.proxy[:authorization] if @req.proxy[:authorization]
       end
 
       # Set the cookie header if provided
-      if cookie = head.delete('cookie')
-        head['cookie'] = encode_cookie(cookie)
+      if cookie = head['cookie']
+        @cookies << encode_cookie(cookie) if cookie
       end
+      head['cookie'] = @cookies.compact.uniq.join("; ").squeeze(";") unless @cookies.empty?
 
       # Set connection close unless keepalive
-      unless @options[:keepalive]
+      if !@req.keepalive
         head['connection'] = 'close'
       end
 
@@ -141,8 +153,8 @@ module EventMachine
 
     def send_request(head, body)
       body    = normalize_body(body)
-      file    = @options[:file]
-      query   = @options[:query]
+      file    = @req.file
+      query   = @req.query
 
       # Set the Content-Length if file is given
       head['content-length'] = File.size(file) if file
@@ -151,19 +163,19 @@ module EventMachine
       head['content-length'] =  body.bytesize if body
 
       # Set content-type header if missing and body is a Ruby hash
-      if not head['content-type'] and @options[:body].is_a? Hash
+      if not head['content-type'] and @req.body.is_a? Hash
         head['content-type'] = 'application/x-www-form-urlencoded'
       end
 
-      request_header ||= encode_request(@method, @req.uri, query, @conn.opts.proxy)
+      request_header ||= encode_request(@req.method, @req.uri, query, @conn.connopts.proxy)
       request_header << encode_headers(head)
       request_header << CRLF
       @conn.send_data request_header
 
       if body
         @conn.send_data body
-      elsif @options[:file]
-        @conn.stream_file_data @options[:file], :http_chunks => false
+      elsif @req.file
+        @conn.stream_file_data @req.file, :http_chunks => false
       end
     end
 
@@ -207,6 +219,9 @@ module EventMachine
         return
       end
 
+      # add set-cookie's to cookie list
+      @cookies << @response_header.cookie if @response_header.cookie && @req.pass_cookies
+
       # correct location header - some servers will incorrectly give a relative URI
       if @response_header.location
         begin
@@ -229,7 +244,7 @@ module EventMachine
       # Fire callbacks immediately after recieving header requests
       # if the request method is HEAD. In case of a redirect, terminate
       # current connection and reinitialize the process.
-      if @method == "HEAD"
+      if @req.method == "HEAD"
         @state = :finished
         return
       end
@@ -242,7 +257,7 @@ module EventMachine
         @state = :body
       end
 
-      if decoder_class = HttpDecoders.decoder_for_encoding(response_header[CONTENT_ENCODING])
+      if @req.decoding && decoder_class = HttpDecoders.decoder_for_encoding(response_header[CONTENT_ENCODING])
         begin
           @content_decoder = decoder_class.new do |s| on_decoded_body_data(s) end
         rescue HttpDecoders::DecoderError

data/lib/em-http/decoders.rb

@@ -71,7 +71,7 @@ module EventMachine::HttpDecoders
   class Deflate < Base
     def decompress(compressed)
       begin
-        @zstream ||= Zlib::Inflate.new(nil)
+        @zstream ||= Zlib::Inflate.new(-Zlib::MAX_WBITS)
         @zstream.inflate(compressed)
       rescue Zlib::Error
         raise DecoderError

data/lib/em-http/http_client_options.rb

@@ -0,0 +1,56 @@
+class HttpClientOptions
+  attr_reader :uri, :method, :host, :port, :proxy
+  attr_reader :headers, :file, :body, :query, :path
+  attr_reader :keepalive, :pass_cookies, :decoding
+
+  attr_accessor :followed, :redirects
+
+  def initialize(uri, options, method)
+    @keepalive = options[:keepalive] || false  # default to single request per connection
+    @redirects = options[:redirects] ||= 0     # default number of redirects to follow
+    @followed  = options[:followed]  ||= 0     # keep track of number of followed requests
+
+    @method   = method.to_s.upcase
+    @headers  = options[:head]  || {}
+    @proxy    = options[:proxy] || {}
+    @query    = options[:query]
+    @path     = options[:path]
+
+    @file     = options[:file]
+    @body     = options[:body]
+
+    @pass_cookies = options.fetch(:pass_cookies, true)  # pass cookies between redirects
+    @decoding     = options.fetch(:decoding, true)      # auto-decode compressed response
+
+    set_uri(uri)
+  end
+
+  def follow_redirect?; @followed < @redirects; end
+  def http_proxy?; @proxy && [nil, :http].include?(@proxy[:type]); end
+  def ssl?; @uri.scheme == "https" || @uri.port == 443; end
+
+  def set_uri(uri)
+    uri = uri.kind_of?(Addressable::URI) ? uri : Addressable::URI::parse(uri.to_s)
+    uri.path = '/' if uri.path.empty?
+    uri.path = @path if @path
+
+    @uri = uri
+
+    # Make sure the ports are set as Addressable::URI doesn't
+    # set the port if it isn't there
+    if @uri.scheme == "https"
+      @uri.port ||= 443
+    else
+      @uri.port ||= 80
+    end
+
+    if !@proxy.empty?
+      @host = @proxy[:host]
+      @port = @proxy[:port]
+    else
+      @host = @uri.host
+      @port = @uri.port
+    end
+
+  end
+end

data/lib/em-http/http_connection.rb

@@ -8,40 +8,90 @@ module EventMachine
     def post   options = {}, &blk;  setup_request(:post,  options, &blk); end
   end
 
-  class FailedConnection
-    include HTTPMethods
+  class HttpStubConnection < Connection
     include Deferrable
+    attr_reader :parent
 
-    attr_accessor :error, :opts
+    def parent=(p)
+      @parent = p
+      @parent.conn = self
+    end
 
-    def initialize(req)
-      @opts = req
+    def receive_data(data)
+      @parent.receive_data data
     end
 
-    def setup_request(method, options)
-      c = HttpClient.new(self, HttpOptions.new(@opts.uri, options, method), options)
-      c.close(@error)
-      c
+    def connection_completed
+      @parent.connection_completed
+    end
+
+    def unbind
+      @parent.unbind
     end
   end
 
-  class HttpConnection < Connection
+  class HttpConnection
     include HTTPMethods
-    include Deferrable
     include Socksify
 
-    attr_accessor :error, :opts
+    attr_reader :deferred
+    attr_accessor :error, :connopts, :uri, :conn
 
-    def setup_request(method, options = {})
-      c = HttpClient.new(self, HttpOptions.new(@opts.uri, options, method), options)
-      callback { c.connection_completed }
+    def initialize
+      @deferred = true
+      @middleware = []
+    end
+
+    def conn=(c)
+      @conn = c
+      @deferred = false
+    end
+
+    def activate_connection(client)
+      begin
+        EventMachine.connect(@connopts.host, @connopts.port, HttpStubConnection) do |conn|
+          post_init
+
+          @deferred = false
+          @conn = conn
+
+          conn.parent = self
+          conn.pending_connect_timeout = @connopts.connect_timeout
+          conn.comm_inactivity_timeout = @connopts.inactivity_timeout
+        end
+
+        finalize_request(client)
+      rescue EventMachine::ConnectionError => e
+        #
+        # Currently, this can only fire on initial connection setup
+        # since #connect is a synchronous method. Hence, rescue the
+        # exception, and return a failed deferred which will immediately
+        # fail any client request.
+        #
+        # Once there is async-DNS, then we'll iterate over the outstanding
+        # client requests and fail them in order.
+        #
+        # Net outcome: failed connection will invoke the same ConnectionError
+        # message on the connection deferred, and on the client deferred.
+        #
+        client.close(e.message)
+      end
+    end
+
+    def setup_request(method, options = {}, c = nil)
+      c ||= HttpClient.new(self, HttpClientOptions.new(@uri, options, method))
+      @deferred ? activate_connection(c) : finalize_request(c)
+      c
+    end
+
+    def finalize_request(c)
+      @conn.callback { c.connection_completed }
 
       middleware.each do |m|
         c.callback &m.method(:response) if m.respond_to?(:response)
       end
 
       @clients.push c
-      c
     end
 
     def middleware
@@ -52,19 +102,17 @@ module EventMachine
       @clients = []
       @pending = []
 
-      @middleware = []
-
       @p = Http::Parser.new
       @p.on_headers_complete = proc do |h|
-        @clients.first.parse_response_header(h, @p.http_version, @p.status_code)
+        client.parse_response_header(h, @p.http_version, @p.status_code)
       end
 
       @p.on_body = proc do |b|
-        @clients.first.on_body_data(b)
+        client.on_body_data(b)
       end
 
       @p.on_message_complete = proc do
-        if not @clients.first.continue?
+        if not client.continue?
           c = @clients.shift
           c.state = :finished
           c.on_request_complete
@@ -72,8 +120,12 @@ module EventMachine
       end
     end
 
-    def use(klass)
-      @middleware << klass
+    def use(klass, *args, &block)
+      @middleware << klass.new(*args, &block)
+    end
+
+    def peer
+      Socket.unpack_sockaddr_in(@peer)[1] rescue nil
     end
 
     def receive_data(data)
@@ -81,35 +133,31 @@ module EventMachine
         @p << data
       rescue HTTP::Parser::Error => e
         c = @clients.shift
-        c.on_error(e.message)
+        c.nil? ? unbind : c.on_error(e.message)
       end
     end
 
     def connection_completed
-      if @opts.proxy && @opts.proxy[:type] == :socks5
-        socksify(client.req.uri.host, client.req.uri.port, *@opts.proxy[:authorization]) { start }
+      @peer = @conn.get_peername
 
+      if @connopts.proxy && @connopts.proxy[:type] == :socks5
+        socksify(client.req.uri.host, client.req.uri.port, *@connopts.proxy[:authorization]) { start }
       else
         start
       end
     end
 
     def start
-      ssl = @opts.options[:tls] || @opts.options[:ssl] || {}
-      start_tls(ssl) if client && client.ssl?
-
-      succeed
+      @conn.start_tls(@connopts.tls) if client && client.req.ssl?
+      @conn.succeed
     end
 
-    def redirect(client, location)
-      client.req.set_uri(location)
+    def redirect(client)
       @pending.push client
-    rescue Exception => e
-      client.on_error(e.message)
     end
 
     def unbind
-      @clients.map {|c| c.unbind }
+      @clients.map { |c| c.unbind }
 
       if r = @pending.shift
         @clients.push r
@@ -118,19 +166,27 @@ module EventMachine
         @p.reset!
 
         begin
-          set_deferred_status :unknown
-          reconnect(r.req.host, r.req.port)
-          callback { r.connection_completed }
+          @conn.set_deferred_status :unknown
+          @conn.reconnect(r.req.host, r.req.port)
+          @conn.callback { r.connection_completed }
         rescue EventMachine::ConnectionError => e
           @clients.pop.close(e.message)
         end
       end
     end
 
+    def send_data(data)
+      @conn.send_data data
+    end
+
+    def stream_file_data(filename, args = {})
+      @conn.stream_file_data filename, args
+    end
+
     private
 
-      def client
-        @clients.first
-      end
+    def client
+      @clients.first
+    end
   end
 end