astro-em-http-request 0.2.3 → 0.2.6

data/README.rdoc

@@ -3,8 +3,10 @@
 EventMachine based HTTP Request interface. Supports streaming response processing, uses Ragel HTTP parser. 
 - Simple interface for single & parallel requests via deferred callbacks
 - Automatic gzip & deflate decoding
-- Basic-Auth support
-- Custom timeouts 
+- Basic-Auth & OAuth support
+- Custom timeouts
+- Proxy support (with SSL Tunneling)
+- Bi-directional communication with web-socket services
 
 Screencast / Demo of using EM-HTTP-Request: 
 - http://everburning.com/news/eventmachine-screencast-em-http-request/
@@ -35,6 +37,7 @@ Screencast / Demo of using EM-HTTP-Request:
   }
 
 == Multi request example
+Fire and wait for multiple requess to complete via the MultiRequest interface.
 
   EventMachine.run {
     multi = EventMachine::MultiRequest.new
@@ -52,6 +55,7 @@ Screencast / Demo of using EM-HTTP-Request:
   }
 
 == Basic-Auth example
+Full basic author support. For OAuth, check examples/oauth-tweet.rb file.
 
   EventMachine.run {
     http = EventMachine::HttpRequest.new('http://www.website.com/').get :head => {'authorization' => ['user', 'pass']}
@@ -59,13 +63,12 @@ Screencast / Demo of using EM-HTTP-Request:
     http.errback { failed }
     http.callback {
       p http.response_header
-
       EventMachine.stop
     }
   }
 
-== POST example
 
+== POST example
   EventMachine.run {
     http1 = EventMachine::HttpRequest.new('http://www.website.com/').post :body => {"key1" => 1, "key2" => [2,3]}
     http2 = EventMachine::HttpRequest.new('http://www.website.com/').post :body => "some data"
@@ -74,9 +77,44 @@ Screencast / Demo of using EM-HTTP-Request:
   }
 
 == Streaming body processing
+Allows you to consume an HTTP stream of content in real-time. Each time a new piece of conent is pushed
+to the client, it is passed to the stream callback for you to operate on.
+
   EventMachine.run {
     http = EventMachine::HttpRequest.new('http://www.website.com/').get
     http.stream { |chunk| print chunk }
 
     # ...
   }
+
+== Proxy example
+Full transparent proxy support with support for SSL tunneling.
+
+  EventMachine.run {
+    http = EventMachine::HttpRequest.new('http://www.website.com/').get :proxy => {
+      :host => 'www.myproxy.com',
+      :port => 8080,
+      :authorization => ['username', 'password'] # authorization is optional
+  }
+
+== WebSocket example
+Bi-directional communication with WebSockets: simply pass in a ws:// resource and the client will
+negotiate the connection upgrade for you. On successfull handshake the callback is invoked, and
+any incoming messages will be passed to the stream callback. The client can also send data to the
+server at will by calling the "send" method!
+- http://www.igvita.com/2009/12/22/ruby-websockets-tcp-for-the-browser/
+
+  EventMachine.run {
+    http = EventMachine::HttpRequest.new("ws://yourservice.com/websocket").get :timeout => 0
+ 
+    http.errback { puts "oops" }
+    http.callback {
+      puts "WebSocket connected!"
+      http.send("Hello client")
+    }
+
+    http.stream { |msg|
+      puts "Recieved: #{msg}"
+      http.send "Pong: #{msg}"
+    }
+  }

data/Rakefile

@@ -35,7 +35,7 @@ task :ragel do
 end
 
 task :spec do
-  sh 'spec -c test/test_*.rb'
+	sh 'spec spec/*_spec.rb'
 end
 
 def make(makedir)
@@ -91,11 +91,12 @@ begin
     gemspec.description = gemspec.summary
     gemspec.email = "ilya@igvita.com"
     gemspec.homepage = "http://github.com/igrigorik/em-http-request"
-    gemspec.authors = ["Ilya Grigorik", "Stephan Maka"]
+    gemspec.authors = ["Ilya Grigorik", "Stephan Maka", "Julien Genestoux"]
     gemspec.extensions = ["ext/buffer/extconf.rb" , "ext/http11_client/extconf.rb"]
     gemspec.add_dependency('eventmachine', '>= 0.12.2')
     gemspec.add_dependency('addressable', '>= 2.0.0')
     gemspec.rubyforge_project = "em-http-request"
+    gemspec.files = FileList[`git ls-files`.split]
   end
   
   Jeweler::GemcutterTasks.new

data/VERSION

@@ -1 +1 @@
-0.2.3
+0.2.6

data/examples/oauth-tweet.rb

@@ -0,0 +1,50 @@
+# Courtesy of Darcy Laycock:
+# http://gist.github.com/265261
+#
+
+require 'rubygems'
+
+require 'em-http'
+require 'oauth'
+
+# At a minimum, require 'oauth/request_proxy/em_http_request'
+# for this example, we'll use Net::HTTP like support.
+require 'oauth/client/em_http'
+
+# You need two things: an oauth consumer and an access token.
+# You need to generate an access token, I suggest looking elsewhere how to do that or wait for a full tutorial.
+# For a consumer key / consumer secret, signup for an app at:
+# http://twitter.com/apps/new
+
+# Edit in your details.
+CONSUMER_KEY = ""
+CONSUMER_SECRET = ""
+ACCESS_TOKEN = ""
+ACCESS_TOKEN_SECRET = ""
+
+def twitter_oauth_consumer
+  @twitter_oauth_consumer ||= OAuth::Consumer.new(CONSUMER_KEY, CONSUMER_SECRET, :site => "http://twitter.com")
+end
+
+def twitter_oauth_access_token
+  @twitter_oauth_access_token ||= OAuth::AccessToken.new(twitter_oauth_consumer, ACCESS_TOKEN, ACCESS_TOKEN_SECRET)
+end
+
+EM.run do
+
+  request = EventMachine::HttpRequest.new('http://twitter.com/statuses/update.json')
+  http = request.post(:body => {'status' => 'Hello Twitter from em-http-request with OAuth'}, :head => {"Content-Type" => "application/x-www-form-urlencoded"}) do |client|
+    twitter_oauth_consumer.sign!(client, twitter_oauth_access_token)
+  end
+
+  http.callback do
+    puts "Response: #{http.response} (Code: #{http.response_header.status})"
+    EM.stop_event_loop
+  end
+
+  http.errback do
+    puts "Failed to post"
+    EM.stop_event_loop
+  end
+
+end

data/ext/http11_client/http11_client.c

@@ -21,12 +21,27 @@ static VALUE eHttpClientParserError;
 #define  id_chunk_size rb_intern("@http_chunk_size")
 #define  id_last_chunk rb_intern("@last_chunk")
 
+#ifndef RHASH_TBL
+/* rb_hash_lookup() is only in Ruby 1.8.7 */
+static VALUE rb_hash_lookup(VALUE hash, VALUE key)
+{
+  VALUE val;
+
+  if (!st_lookup(RHASH(hash)->tbl, key, &val)) {
+    return Qnil; /* without Hash#default */
+  }
+
+  return val;
+}
+#endif
+
 void client_http_field(void *data, const char *field, size_t flen, const char *value, size_t vlen)
 {
   char *ch, *end;
   VALUE req = (VALUE)data;
   VALUE v = Qnil;
   VALUE f = Qnil;
+  VALUE el = Qnil;
 
   v = rb_str_new(value, vlen);
   f = rb_str_new(field, flen);
@@ -41,7 +56,18 @@ void client_http_field(void *data, const char *field, size_t flen, const char *v
     }
   }
 
-  rb_hash_aset(req, f, v);
+  el = rb_hash_lookup(req, f);
+  switch(TYPE(el)) {
+    case T_ARRAY:
+      rb_ary_push(el, v);
+      break;
+    case T_STRING:
+      rb_hash_aset(req, f, rb_ary_new3(2, el, v));
+      break;
+    default:
+      rb_hash_aset(req, f, v);
+      break;
+  }
 }
 
 void client_reason_phrase(void *data, const char *at, size_t length)

data/lib/em-http.rb

@@ -3,10 +3,8 @@
 # You can redistribute this under the terms of the Ruby license
 # See file LICENSE for details
 #++
-
-require 'rubygems'
-require 'eventmachine'
-
+                          
+require 'eventmachine'    
 
 require File.dirname(__FILE__) + '/http11_client'
 require File.dirname(__FILE__) + '/em_buffer'
@@ -15,4 +13,4 @@ require File.dirname(__FILE__) + '/em-http/core_ext/hash'
 require File.dirname(__FILE__) + '/em-http/client'
 require File.dirname(__FILE__) + '/em-http/multi'
 require File.dirname(__FILE__) + '/em-http/request'
-require File.dirname(__FILE__) + '/em-http/decoders'
+require File.dirname(__FILE__) + '/em-http/decoders'

data/lib/em-http/client.rb

@@ -39,7 +39,7 @@ module EventMachine
     # Length of content as an integer, or nil if chunked/unspecified
     def content_length
       @content_length ||= ((s = self[HttpClient::CONTENT_LENGTH]) &&
-                           (s =~ /^(\d+)$/)) ? $1.to_i : nil
+          (s =~ /^(\d+)$/)) ? $1.to_i : nil
     end
 
     # Cookie header from the server
@@ -80,7 +80,6 @@ module EventMachine
   module HttpEncoding
     HTTP_REQUEST_HEADER="%s %s HTTP/1.1\r\n"
     FIELD_ENCODING = "%s: %s\r\n"
-    BASIC_AUTH_ENCODING = "%s: Basic %s\r\n"
 
     # Escapes a URI.
     def escape(s)
@@ -108,18 +107,18 @@ module EventMachine
       @uri.host + (@uri.port != 80 ? ":#{@uri.port}" : "")
     end
 
-    def encode_request(method, path, query)
-      HTTP_REQUEST_HEADER % [method.to_s.upcase, encode_query(path, query)]
+    def encode_request(method, path, query, uri_query)
+      HTTP_REQUEST_HEADER % [method.to_s.upcase, encode_query(path, query, uri_query)]
     end
 
-    def encode_query(path, query)
+    def encode_query(path, query, uri_query)
       encoded_query = if query.kind_of?(Hash)
         query.map { |k, v| encode_param(k, v) }.join('&')
       else
         query.to_s
       end
-      if !@uri.query.to_s.empty?
-        encoded_query = [encoded_query, @uri.query].reject {|part| part.empty?}.join("&")
+      if !uri_query.to_s.empty?
+        encoded_query = [encoded_query, uri_query].reject {|part| part.empty?}.join("&")
       end
       return path if encoded_query.to_s.empty?
       "#{path}?#{encoded_query}"
@@ -141,18 +140,25 @@ module EventMachine
     end
 
     # Encode basic auth in an HTTP header
-    def encode_basic_auth(k,v)
-      BASIC_AUTH_ENCODING % [k, Base64.encode64(v.join(":")).chomp]
+    # In: Array ([user, pass]) - for basic auth
+    #     String - custom auth string (OAuth, etc)
+    def encode_auth(k,v)
+      if v.is_a? Array
+        FIELD_ENCODING % [k, ["Basic", Base64.encode64(v.join(":")).chomp].join(" ")]
+      else
+        encode_field(k,v)
+      end      
     end
 
     def encode_headers(head)
       head.inject('') do |result, (key, value)|
         # Munge keys from foo-bar-baz to Foo-Bar-Baz
-        key = key.split('-').map { |k| k.capitalize }.join('-')
-        unless key == "Authorization"
-          result << encode_field(key, value)
+        key = key.split('-').map { |k| k.to_s.capitalize }.join('-')
+        result << case key
+        when 'Authorization', 'Proxy-authorization'          
+          encode_auth(key, value)
         else
-          result << encode_basic_auth(key, value)
+          encode_field(key, value)
         end
       end
     end
@@ -185,31 +191,43 @@ module EventMachine
     def post_init
       @parser = HttpClientParser.new
       @data = EventMachine::Buffer.new
-      @response_header = HttpResponseHeader.new
       @chunk_header = HttpChunkHeader.new
-
-      @state = :response_header
+      @response_header = HttpResponseHeader.new
       @parser_nbytes = 0
       @response = ''
       @errors = ''
       @content_decoder = nil
       @stream = nil
+      @state = :response_header
+      @bytes_received = 0
+      @options = {}
     end
 
     # start HTTP request once we establish connection to host
-    def connection_completed
-      if @options[:max_connection_duration] 
-        EM.add_timer(@options[:max_connection_duration]) {
-          @aborted = true
-          on_error("Max Connection Duration Exceeded (#{@options[:max_connection_duration]}s.)")
-        }
-      end
-      ssl = @options[:tls] || @options[:ssl] || {}
-      @bytes_received = 0
-      start_tls(ssl) if @uri.scheme == "https" #or @uri.port == 443 # A user might not want https even on port 443.
-
-      send_request_header
-      send_request_body
+    def connection_completed              
+      # if connecting to proxy, then first negotiate the connection
+      # to intermediate server and wait for 200 response 
+      if @options[:proxy] and @state == :response_header 
+        @state = :response_proxy
+        send_request_header
+        
+        # if connecting via proxy, then state will be :proxy_connected,
+        # indicating successful tunnel. from here, initiate normal http
+        # exchange
+      else
+        if @options[:max_connection_duration] 
+          EM.add_timer(@options[:max_connection_duration]) {
+            @aborted = true
+            on_error("Max Connection Duration Exceeded (#{@options[:max_connection_duration]}s.)")
+          }
+        end
+        @state = :response_header
+                      
+        ssl = @options[:tls] || @options[:ssl] || {}
+        start_tls(ssl) if @uri.scheme == "https" #or @uri.port == 443 # A user might not want https even on port 443.
+        send_request_header
+        send_request_body
+      end
     end
     
     # request is done, invoke the callback
@@ -236,43 +254,87 @@ module EventMachine
       @stream = blk
     end
 
+    # raw data push from the client (WebSocket) should
+    # only be invoked after handshake, otherwise it will
+    # inject data into the header exchange
+    #
+    # frames need to start with 0x00-0x7f byte and end with
+    # an 0xFF byte. Per spec, we can also set the first
+    # byte to a value betweent 0x80 and 0xFF, followed by
+    # a leading length indicator
+    def send(data)
+      if @state == :websocket
+        send_data("\x00#{data}\xff")
+      end
+    end
+
     def normalize_body
-      if @options[:body].is_a? Hash
-        @options[:body].to_params
-      else
-        @options[:body]
+      @normalized_body ||= begin
+        if @options[:body].is_a? Hash
+          @options[:body].to_params
+        else
+          @options[:body]
+        end
+      end
+    end
+    
+    def normalize_uri
+      @normalized_uri ||= begin
+        uri = @uri.dup
+        encoded_query = encode_query(@uri.path, @options[:query], @uri.query)
+        path, query = encoded_query.split("?", 2)
+        uri.query = query unless encoded_query.empty?
+        uri.path  = path
+        uri
       end
     end
 
+    def websocket?; @uri.scheme == 'ws'; end
+    
     def send_request_header
       query   = @options[:query]
       head    = @options[:head] ? munge_header_keys(@options[:head]) : {}
       body    = normalize_body
+      request_header = nil
+
+      if @state == :response_proxy
+        proxy = @options[:proxy]
+
+        # initialize headers to establish the HTTP tunnel
+        head = proxy[:head] ? munge_header_keys(proxy[:head]) : {}
+        head['proxy-authorization'] = proxy[:authorization] if proxy[:authorization]
+        request_header = HTTP_REQUEST_HEADER % ['CONNECT', "#{@uri.host}:#{@uri.port}"]
+        
+      elsif websocket?
+        head['upgrade'] = 'WebSocket'
+        head['connection'] = 'Upgrade'
+        head['origin'] = @options[:origin] || @uri.host
+        
+      else
+        # Set the Content-Length if body is given
+        head['content-length'] =  body.length if body
+
+        # Set the cookie header if provided
+        if cookie = head.delete('cookie')
+          head['cookie'] = encode_cookie(cookie)
+        end
+
+        # Set content-type header if missing and body is a Ruby hash
+        if not head['content-type'] and options[:body].is_a? Hash
+          head['content-type'] = "application/x-www-form-urlencoded"
+        end
+      end
 
       # Set the Host header if it hasn't been specified already
       head['host'] ||= encode_host
 
-      # Set the Content-Length if body is given
-      head['content-length'] =  body.length if body
-
       # Set the User-Agent if it hasn't been specified
       head['user-agent'] ||= "EventMachine HttpClient"
 
-      # Set the cookie header if provided
-      if cookie = head.delete('cookie')
-        head['cookie'] = encode_cookie(cookie)
-      end
-
-      # Set content-type header if missing and body is a Ruby hash
-      if not head['content-type'] and options[:body].is_a? Hash
-        head['content-type'] = "application/x-www-form-urlencoded"
-      end
-
-      # Build the request
-      request_header = encode_request(@method, @uri.path, query)
+      # Build the request headers
+      request_header ||= encode_request(@method, @uri.path, query, @uri.query)
       request_header << encode_headers(head)
       request_header << CRLF
-
       send_data request_header
     end
 
@@ -333,6 +395,8 @@ module EventMachine
 
     def dispatch
       while case @state
+        when :response_proxy
+          parse_response_proxy
         when :response_header
           parse_response_header
         when :chunk_header
@@ -345,6 +409,8 @@ module EventMachine
           process_response_footer
         when :body
           process_body
+        when :websocket
+          process_websocket
         when :finished, :invalid
           break
         else raise RuntimeError, "invalid state: #{@state}"
@@ -370,6 +436,28 @@ module EventMachine
 
       true
     end
+    
+    # TODO: refactor with parse_response_header
+    def parse_response_proxy
+      return false unless parse_header(@response_header)
+
+      unless @response_header.http_status and @response_header.http_reason
+        @state = :invalid
+        on_error "no HTTP response"
+        return false
+      end
+          
+      # when a successfull tunnel is established, the proxy responds with a
+      # 200 response code. from here, the tunnel is transparent.
+      if @response_header.http_status.to_i == 200
+        @response_header = HttpResponseHeader.new
+        connection_completed
+      else
+        @state = :invalid
+        on_error "proxy not accessible"
+        return false
+      end
+    end
 
     def parse_response_header
       return false unless parse_header(@response_header)
@@ -394,17 +482,25 @@ module EventMachine
         end
       end
 
-      # shortcircuit on HEAD requests 
+      # shortcircuit on HEAD requests
       if @method == "HEAD"
         @state = :finished
         on_request_complete
       end
 
-      if @response_header.chunked_encoding?
+      if websocket?
+        if @response_header.status == 101
+          @state = :websocket
+          succeed
+        else
+          fail "websocket handshake failed"
+        end
+       
+      elsif @response_header.chunked_encoding?
         @state = :chunk_header
       elsif @response_header.content_length
         @state = :body
-        @bytes_remaining = @response_header.content_length 
+        @bytes_remaining = @response_header.content_length
       else
         @state = :body
         @bytes_remaining = nil
@@ -519,7 +615,23 @@ module EventMachine
       false
     end
 
-  
+    def process_websocket
+      return false if @data.empty?
+
+      # slice the message out of the buffer and pass in
+      # for processing, and buffer data otherwise
+      buffer = @data.read
+      while msg = buffer.slice!(/\000([^\377]*)\377/)
+        msg.gsub!(/^\x00|\xff$/, '')
+        @stream.call(msg)
+      end
+
+      # store remainder if message boundary has not yet
+      # been recieved
+      @data << buffer if not buffer.empty?
+
+      false
+    end
   end
 
 end