em-http-request 0.2.4 → 0.2.5

data/.gitignore

@@ -0,0 +1,4 @@
+*.bundle
+*.o
+Makefile
+mkmf.log

data/README.rdoc

@@ -59,13 +59,23 @@ Screencast / Demo of using EM-HTTP-Request:
     http.errback { failed }
     http.callback {
       p http.response_header
+      EventMachine.stop
+    }
+  }
+
+== OAuth example
 
+  EventMachine.run {
+    http = EventMachine::HttpRequest.new('http://www.website.com/').get :head => {'authorization' => 'OAuth oauth_nonce=...'}
+    http.errback { failed }
+    http.callback {
+      p http.response_header
       EventMachine.stop
     }
   }
 
-== POST example
 
+== POST example
   EventMachine.run {
     http1 = EventMachine::HttpRequest.new('http://www.website.com/').post :body => {"key1" => 1, "key2" => [2,3]}
     http2 = EventMachine::HttpRequest.new('http://www.website.com/').post :body => "some data"
@@ -80,3 +90,12 @@ Screencast / Demo of using EM-HTTP-Request:
 
     # ...
   }
+
+== Proxy example
+  EventMachine.run {
+    http1 = EventMachine::HttpRequest.new('http://www.website.com/').get :proxy => {
+      :host => 'www.myproxy.com',
+      :port => 8080,
+      :authorization => ['username', 'password'] # authorization is optional
+    }
+

data/Rakefile

@@ -97,7 +97,7 @@ begin
     gemspec.add_dependency('eventmachine', '>= 0.12.9')
     gemspec.add_dependency('addressable', '>= 2.0.0')
     gemspec.rubyforge_project = "em-http-request"
-    gemspec.files = FileList["[A-Z]*", "{lib,test,ext}/**/*"]
+    gemspec.files = FileList[`git ls-files`.split]
   end
   
   Jeweler::GemcutterTasks.new

data/VERSION

@@ -1 +1 @@
-0.2.4
+0.2.5

data/lib/em-http/client.rb

@@ -39,7 +39,7 @@ module EventMachine
     # Length of content as an integer, or nil if chunked/unspecified
     def content_length
       @content_length ||= ((s = self[HttpClient::CONTENT_LENGTH]) &&
-                           (s =~ /^(\d+)$/)) ? $1.to_i : nil
+          (s =~ /^(\d+)$/)) ? $1.to_i : nil
     end
 
     # Cookie header from the server
@@ -80,7 +80,6 @@ module EventMachine
   module HttpEncoding
     HTTP_REQUEST_HEADER="%s %s HTTP/1.1\r\n"
     FIELD_ENCODING = "%s: %s\r\n"
-    BASIC_AUTH_ENCODING = "%s: Basic %s\r\n"
 
     # Escapes a URI.
     def escape(s)
@@ -141,18 +140,25 @@ module EventMachine
     end
 
     # Encode basic auth in an HTTP header
-    def encode_basic_auth(k,v)
-      BASIC_AUTH_ENCODING % [k, Base64.encode64(v.join(":")).chomp]
+    # In: Array ([user, pass]) - for basic auth
+    #     String - custom auth string (OAuth, etc)
+    def encode_auth(k,v)
+      if v.is_a? Array
+        FIELD_ENCODING % [k, ["Basic", Base64.encode64(v.join(":")).chomp].join(" ")]
+      else
+        encode_field(k,v)
+      end      
     end
 
     def encode_headers(head)
       head.inject('') do |result, (key, value)|
         # Munge keys from foo-bar-baz to Foo-Bar-Baz
-        key = key.split('-').map { |k| k.capitalize }.join('-')
-        unless key == "Authorization"
-          result << encode_field(key, value)
+        key = key.split('-').map { |k| k.to_s.capitalize }.join('-')
+        result << case key
+        when 'Authorization', 'Proxy-authorization'          
+          encode_auth(key, value)
         else
-          result << encode_basic_auth(key, value)
+          encode_field(key, value)
         end
       end
     end
@@ -185,24 +191,35 @@ module EventMachine
     def post_init
       @parser = HttpClientParser.new
       @data = EventMachine::Buffer.new
-      @response_header = HttpResponseHeader.new
       @chunk_header = HttpChunkHeader.new
-
-      @state = :response_header
+      @response_header = HttpResponseHeader.new
       @parser_nbytes = 0
       @response = ''
       @errors = ''
       @content_decoder = nil
       @stream = nil
+      @state = :response_header
     end
 
     # start HTTP request once we establish connection to host
-    def connection_completed
-      ssl = @options[:tls] || @options[:ssl] || {}
-      start_tls(ssl) if @uri.scheme == "https" or @uri.port == 443
-
-      send_request_header
-      send_request_body
+    def connection_completed              
+      # if connecting to proxy, then first negotiate the connection
+      # to intermediate server and wait for 200 response 
+      if @options[:proxy] and @state == :response_header 
+        @state = :response_proxy
+        send_request_header
+        
+        # if connecting via proxy, then state will be :proxy_connected,
+        # indicating successful tunnel. from here, initiate normal http
+        # exchange
+      else  
+        @state = :response_header
+                      
+        ssl = @options[:tls] || @options[:ssl] || {}
+        start_tls(ssl) if @uri.scheme == "https" or @uri.port == 443
+        send_request_header
+        send_request_body
+      end
     end
     
     # request is done, invoke the callback
@@ -236,36 +253,46 @@ module EventMachine
         @options[:body]
       end
     end
-
+                  
     def send_request_header
       query   = @options[:query]
       head    = @options[:head] ? munge_header_keys(@options[:head]) : {}
       body    = normalize_body
+      request_header = nil
 
-      # Set the Host header if it hasn't been specified already
-      head['host'] ||= encode_host
+      if @state == :response_proxy
+        proxy = @options[:proxy]
 
-      # Set the Content-Length if body is given
-      head['content-length'] =  body.length if body
+        # initialize headers to establish the HTTP tunnel
+        head = proxy[:head] ? munge_header_keys(proxy[:head]) : {}
+        head['proxy-authorization'] = proxy[:authorization] if proxy[:authorization]
+        request_header = HTTP_REQUEST_HEADER % ['CONNECT', "#{@uri.host}:#{@uri.port}"]
 
-      # Set the User-Agent if it hasn't been specified
-      head['user-agent'] ||= "EventMachine HttpClient"
+      else
+        # Set the Content-Length if body is given
+        head['content-length'] =  body.length if body
 
-      # Set the cookie header if provided
-      if cookie = head.delete('cookie')
-        head['cookie'] = encode_cookie(cookie)
-      end
+        # Set the cookie header if provided
+        if cookie = head.delete('cookie')
+          head['cookie'] = encode_cookie(cookie)
+        end
 
-      # Set content-type header if missing and body is a Ruby hash
-      if not head['content-type'] and options[:body].is_a? Hash
-        head['content-type'] = "application/x-www-form-urlencoded"
+        # Set content-type header if missing and body is a Ruby hash
+        if not head['content-type'] and options[:body].is_a? Hash
+          head['content-type'] = "application/x-www-form-urlencoded"
+        end
       end
 
-      # Build the request
-      request_header = encode_request(@method, @uri.path, query, @uri.query)
+       # Set the Host header if it hasn't been specified already
+      head['host'] ||= encode_host
+
+      # Set the User-Agent if it hasn't been specified
+      head['user-agent'] ||= "EventMachine HttpClient"
+
+      # Build the request headers
+      request_header ||= encode_request(@method, @uri.path, query, @uri.query)
       request_header << encode_headers(head)
       request_header << CRLF
-
       send_data request_header
     end
 
@@ -303,7 +330,7 @@ module EventMachine
 
     def unbind
       if @state == :finished || (@state == :body && @bytes_remaining.nil?)
-        succeed(self) 
+        succeed(self)
       else
         fail(self)
       end
@@ -316,6 +343,8 @@ module EventMachine
 
     def dispatch
       while case @state
+        when :response_proxy
+          parse_response_proxy
         when :response_header
           parse_response_header
         when :chunk_header
@@ -354,6 +383,28 @@ module EventMachine
 
       true
     end
+    
+    # TODO: refactor with parse_response_header
+    def parse_response_proxy
+      return false unless parse_header(@response_header)
+
+      unless @response_header.http_status and @response_header.http_reason
+        @state = :invalid
+        on_error "no HTTP response"
+        return false
+      end
+          
+      # when a successfull tunnel is established, the proxy responds with a
+      # 200 response code. from here, the tunnel is transparent.
+      if @response_header.http_status.to_i == 200
+        @response_header = HttpResponseHeader.new
+        connection_completed
+      else
+        @state = :invalid
+        on_error "proxy not accessible"
+        return false
+      end
+    end
 
     def parse_response_header
       return false unless parse_header(@response_header)
@@ -378,7 +429,7 @@ module EventMachine
         end
       end
 
-      # shortcircuit on HEAD requests 
+      # shortcircuit on HEAD requests
       if @method == "HEAD"
         @state = :finished
         on_request_complete
@@ -388,7 +439,7 @@ module EventMachine
         @state = :chunk_header
       elsif @response_header.content_length
         @state = :body
-        @bytes_remaining = @response_header.content_length 
+        @bytes_remaining = @response_header.content_length
       else
         @state = :body
         @bytes_remaining = nil

data/lib/em-http/request.rb

@@ -9,7 +9,6 @@ module EventMachine
   #
   # == Example
   #
-  #
   #  EventMachine.run {
   #    http = EventMachine::HttpRequest.new('http://127.0.0.1/').get :query => {'keyname' => 'value'}
   #
@@ -58,22 +57,31 @@ module EventMachine
 
     def setup_request(method, options)
       raise ArgumentError, "invalid request path" unless /^\// === @uri.path
-
       @options = options
+      
+      if proxy = options[:proxy]
+        @host_to_connect = proxy[:host]
+        @port_to_connect = proxy[:port]
+      else
+        @host_to_connect = @uri.host
+        @port_to_connect = @uri.port
+      end                                      
+      
+      # default connect & inactivity timeouts        
+      @options[:timeout] = 10 if not @options[:timeout]  
 
-      # default connect & inactivity timeouts
-      @options[:timeout] = 5 if not @options[:timeout]
-
-      # Make sure the port is set as Addressable::URI doesn't set the
-      # port if it isn't there.
+      # Make sure the ports are set as Addressable::URI doesn't
+      # set the port if it isn't there
       @uri.port ||= 80
+      @port_to_connect ||= 80
+      
       @method = method.to_s.upcase
       send_request
     end
     
     def send_request
       begin
-       EventMachine.connect(@uri.host, @uri.port, EventMachine::HttpClient) { |c|
+       EventMachine.connect(@host_to_connect, @port_to_connect, EventMachine::HttpClient) { |c|
           c.uri = @uri
           c.method = @method
           c.options = @options

data/test/stallion.rb

@@ -127,15 +127,19 @@ Stallion.saddle :spec do |stable|
       stable.response.status = 304
 
     elsif stable.request.env["HTTP_AUTHORIZATION"]
-      auth = "Basic %s" % Base64.encode64(['user', 'pass'].join(':')).chomp
-
-      if auth == stable.request.env["HTTP_AUTHORIZATION"]
+      if stable.request.path_info == '/oauth_auth'
         stable.response.status = 200
-        stable.response.write 'success'
+        stable.response.write stable.request.env["HTTP_AUTHORIZATION"]      
       else
-        stable.response.status = 401
-      end
+        auth = "Basic %s" % Base64.encode64(['user', 'pass'].join(':')).chomp
 
+        if auth == stable.request.env["HTTP_AUTHORIZATION"]
+          stable.response.status = 200
+          stable.response.write 'success'
+        else
+          stable.response.status = 401
+        end
+      end
     elsif stable.request.path_info == '/relative-location'
       stable.response.status = 301
       stable.response["Location"] = '/forwarded'
@@ -155,4 +159,61 @@ Thread.new do
   end
 end
 
+#
+# HTTP Proxy server
+#
+Thread.new do
+  server = TCPServer.new('127.0.0.1', 8082)
+  loop do
+    session = server.accept
+    request = ""
+    while (data = session.gets) != "\r\n"
+      request << data
+    end
+    parts = request.split("\r\n")
+    method, destination, http_version = parts.first.split(' ')
+    if method == 'CONNECT'
+      target_host, target_port = destination.split(':')
+      client = TCPSocket.open(target_host, target_port)
+      session.write "HTTP/1.1 200 Connection established\r\nProxy-agent: Whatever\r\n\r\n"
+      session.flush
+
+      content_length = -1
+      verb = ""
+      req = ""
+
+      while data = session.gets
+        if request = data.match(/(\w+).*HTTP\/1\.1/)
+          verb = request[1]
+        end
+
+        if post = data.match(/Content-Length: (\d+)/)
+          content_length = post[1].to_i
+        end
+
+        req += data
+
+        # read POST data
+        if data == "\r\n" and verb == "POST"
+          req += session.read(content_length)
+        end
+
+        if data == "\r\n"
+          client.write req
+          client.flush
+          client.close_write
+          break
+        end
+      end
+      
+      while data = client.gets
+        session.write data
+      end
+      session.flush
+      client.close
+    end
+    session.close
+  end
+end
+
 sleep(1)

data/test/test_multi.rb

@@ -11,7 +11,7 @@ describe EventMachine::MultiRequest do
       
       # add multiple requests to the multi-handler
       multi.add(EventMachine::HttpRequest.new('http://127.0.0.1:8080/').get(:query => {:q => 'test'}))
-      multi.add(EventMachine::HttpRequest.new('http://0.0.0.0:8081/').get(:timeout => 1))
+      multi.add(EventMachine::HttpRequest.new('http://0.0.0.0:8083/').get(:timeout => 1))
       
       multi.callback  {
         # verify successfull request

data/test/test_request.rb

@@ -223,7 +223,7 @@ describe EventMachine::HttpRequest do
       http = EventMachine::HttpRequest.new('http://digg.com/').get
 
       http.errback { failed }
-      http.callback {
+      http.callback { 
         http.response_header.status.should == 200
         EventMachine.stop
       }
@@ -242,6 +242,20 @@ describe EventMachine::HttpRequest do
       }
     }
   end
+  
+  it "should send proper OAuth auth header" do 
+    EventMachine.run {
+      oauth_header = 'OAuth oauth_nonce="oqwgSYFUD87MHmJJDv7bQqOF2EPnVus7Wkqj5duNByU", b=c, d=e'
+      http = EventMachine::HttpRequest.new('http://127.0.0.1:8080/oauth_auth').get :head => {'authorization' => oauth_header}
+
+      http.errback { failed }
+      http.callback {
+        http.response_header.status.should == 200
+        http.response.should == oauth_header
+        EventMachine.stop
+      }
+    } 
+  end
 
   it "should work with keep-alive servers" do
     EventMachine.run {
@@ -472,5 +486,38 @@ describe EventMachine::HttpRequest do
           EventMachine.stop
         }
       }
+  end   
+  
+  context "connections via proxy" do
+    
+    it "should work with proxy servers" do
+      EventMachine.run {
+
+        http = EventMachine::HttpRequest.new('http://127.0.0.1:8080/').get :proxy => {:host => '127.0.0.1', :port => 8082}
+
+        http.errback { p http.inspect; failed }
+        http.callback { 
+          http.response_header.status.should == 200
+          http.response.should == 'Hello, World!'
+          EventMachine.stop
+        }
+      }
+    end 
+
+    it "should proxy POST data" do
+      EventMachine.run {
+
+         http = EventMachine::HttpRequest.new('http://127.0.0.1:8080/').post({
+           :body => "data", :proxy => {:host => '127.0.0.1', :port => 8082}})
+
+         http.errback { failed }
+         http.callback { 
+           http.response_header.status.should == 200
+           http.response.should match(/data/)
+           EventMachine.stop
+         }
+       }
+     end
+     
   end
-end
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification 
 name: em-http-request
 version: !ruby/object:Gem::Version 
-  version: 0.2.4
+  version: 0.2.5
 platform: ruby
 authors: 
 - Ilya Grigorik
@@ -9,7 +9,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2009-12-01 00:00:00 -05:00
+date: 2009-12-12 00:00:00 -05:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency 
@@ -43,16 +43,17 @@ extra_rdoc_files:
 - LICENSE
 - README.rdoc
 files: 
+- .gitignore
 - LICENSE
 - README.rdoc
 - Rakefile
 - VERSION
+- examples/fetch.rb
+- examples/fibered-http.rb
 - ext/buffer/em_buffer.c
 - ext/buffer/extconf.rb
-- ext/http11_client/Makefile
 - ext/http11_client/ext_help.h
 - ext/http11_client/extconf.rb
-- ext/http11_client/http11_client.bundle
 - ext/http11_client/http11_client.c
 - ext/http11_client/http11_parser.c
 - ext/http11_client/http11_parser.h

data/ext/http11_client/Makefile

@@ -1,157 +0,0 @@
-
-SHELL = /bin/sh
-
-#### Start of system configuration section. ####
-
-srcdir = .
-topdir = /opt/local/lib/ruby/1.8/i686-darwin10
-hdrdir = $(topdir)
-VPATH = $(srcdir):$(topdir):$(hdrdir)
-exec_prefix = $(prefix)
-prefix = $(DESTDIR)/opt/local
-sharedstatedir = $(prefix)/com
-mandir = $(DESTDIR)/opt/local/share/man
-psdir = $(docdir)
-oldincludedir = $(DESTDIR)/usr/include
-localedir = $(datarootdir)/locale
-bindir = $(exec_prefix)/bin
-libexecdir = $(exec_prefix)/libexec
-sitedir = $(libdir)/ruby/site_ruby
-htmldir = $(docdir)
-vendorarchdir = $(vendorlibdir)/$(sitearch)
-includedir = $(prefix)/include
-infodir = $(datarootdir)/info
-vendorlibdir = $(vendordir)/$(ruby_version)
-sysconfdir = $(prefix)/etc
-libdir = $(exec_prefix)/lib
-sbindir = $(exec_prefix)/sbin
-rubylibdir = $(libdir)/ruby/$(ruby_version)
-docdir = $(datarootdir)/doc/$(PACKAGE)
-dvidir = $(docdir)
-vendordir = $(DESTDIR)/opt/local/lib/ruby/vendor_ruby
-datarootdir = $(prefix)/share
-pdfdir = $(docdir)
-archdir = $(rubylibdir)/$(arch)
-sitearchdir = $(sitelibdir)/$(sitearch)
-datadir = $(datarootdir)
-localstatedir = $(prefix)/var
-sitelibdir = $(sitedir)/$(ruby_version)
-
-CC = /usr/bin/gcc-4.2
-LIBRUBY = $(LIBRUBY_SO)
-LIBRUBY_A = lib$(RUBY_SO_NAME)-static.a
-LIBRUBYARG_SHARED = -l$(RUBY_SO_NAME)
-LIBRUBYARG_STATIC = -l$(RUBY_SO_NAME)-static
-
-RUBY_EXTCONF_H = 
-CFLAGS   =  -fno-common -O2 -arch x86_64  -fno-common -pipe -fno-common $(cflags) -arch x86_64
-INCFLAGS = -I. -I. -I/opt/local/lib/ruby/1.8/i686-darwin10 -I.
-DEFS     = 
-CPPFLAGS =  -I/opt/local/include -D_XOPEN_SOURCE -D_DARWIN_C_SOURCE  -I/opt/local/include
-CXXFLAGS = $(CFLAGS) 
-ldflags  = -L. -L/opt/local/lib
-dldflags = 
-archflag = -arch x86_64
-DLDFLAGS = $(ldflags) $(dldflags) $(archflag)
-LDSHARED = $(CC) -dynamic -bundle -undefined suppress -flat_namespace
-AR = ar
-EXEEXT = 
-
-RUBY_INSTALL_NAME = ruby
-RUBY_SO_NAME = ruby
-arch = i686-darwin10
-sitearch = i686-darwin10
-ruby_version = 1.8
-ruby = /opt/local/bin/ruby
-RUBY = $(ruby)
-RM = rm -f
-MAKEDIRS = mkdir -p
-INSTALL = /usr/bin/install -c
-INSTALL_PROG = $(INSTALL) -m 0755
-INSTALL_DATA = $(INSTALL) -m 644
-COPY = cp
-
-#### End of system configuration section. ####
-
-preload = 
-
-libpath = . $(libdir)
-LIBPATH =  -L. -L$(libdir)
-DEFFILE = 
-
-CLEANFILES = mkmf.log
-DISTCLEANFILES = 
-
-extout = 
-extout_prefix = 
-target_prefix = 
-LOCAL_LIBS = 
-LIBS = $(LIBRUBYARG_SHARED) -lc  -lpthread -ldl -lobjc  
-SRCS = http11_client.c http11_parser.c
-OBJS = http11_client.o http11_parser.o
-TARGET = http11_client
-DLLIB = $(TARGET).bundle
-EXTSTATIC = 
-STATIC_LIB = 
-
-BINDIR        = $(bindir)
-RUBYCOMMONDIR = $(sitedir)$(target_prefix)
-RUBYLIBDIR    = $(sitelibdir)$(target_prefix)
-RUBYARCHDIR   = $(sitearchdir)$(target_prefix)
-
-TARGET_SO     = $(DLLIB)
-CLEANLIBS     = $(TARGET).bundle $(TARGET).il? $(TARGET).tds $(TARGET).map
-CLEANOBJS     = *.o *.a *.s[ol] *.pdb *.exp *.bak
-
-all:		$(DLLIB)
-static:		$(STATIC_LIB)
-
-clean:
-		@-$(RM) $(CLEANLIBS) $(CLEANOBJS) $(CLEANFILES)
-
-distclean:	clean
-		@-$(RM) Makefile $(RUBY_EXTCONF_H) conftest.* mkmf.log
-		@-$(RM) core ruby$(EXEEXT) *~ $(DISTCLEANFILES)
-
-realclean:	distclean
-install: install-so install-rb
-
-install-so: $(RUBYARCHDIR)
-install-so: $(RUBYARCHDIR)/$(DLLIB)
-$(RUBYARCHDIR)/$(DLLIB): $(DLLIB)
-	$(INSTALL_PROG) $(DLLIB) $(RUBYARCHDIR)
-install-rb: pre-install-rb install-rb-default
-install-rb-default: pre-install-rb-default
-pre-install-rb: Makefile
-pre-install-rb-default: Makefile
-$(RUBYARCHDIR):
-	$(MAKEDIRS) $@
-
-site-install: site-install-so site-install-rb
-site-install-so: install-so
-site-install-rb: install-rb
-
-.SUFFIXES: .c .m .cc .cxx .cpp .C .o
-
-.cc.o:
-	$(CXX) $(INCFLAGS) $(CPPFLAGS) $(CXXFLAGS) -c $<
-
-.cxx.o:
-	$(CXX) $(INCFLAGS) $(CPPFLAGS) $(CXXFLAGS) -c $<
-
-.cpp.o:
-	$(CXX) $(INCFLAGS) $(CPPFLAGS) $(CXXFLAGS) -c $<
-
-.C.o:
-	$(CXX) $(INCFLAGS) $(CPPFLAGS) $(CXXFLAGS) -c $<
-
-.c.o:
-	$(CC) $(INCFLAGS) $(CPPFLAGS) $(CFLAGS) -c $<
-
-$(DLLIB): $(OBJS) Makefile
-	@-$(RM) $@
-	$(LDSHARED) -o $@ $(OBJS) $(LIBPATH) $(DLDFLAGS) $(LOCAL_LIBS) $(LIBS)
-
-
-
-$(OBJS): ruby.h defines.h