em-http-request 0.2.12 → 0.2.13

data/Changelog.md

@@ -1,5 +1,10 @@
 # Changelog
 
+## 0.2.13 / 2010-09-25
+
+- added SOCKS5 proxy support
+- bugfix: follow redirects on HEAD requests
+
 ## 0.2.12 / 2010-09-12
 
 - added headers callback (http.headers {|h| p h})

data/README.md

@@ -9,7 +9,7 @@ Asynchronous HTTP client for Ruby, based on EventMachine runtime.
 - Basic-Auth & OAuth support
 - Custom timeout support
 - Stream response processing
-- Proxy support (with SSL Tunneling)
+- Proxy support (with SSL Tunneling): CONNECT, direct & SOCKS5
 - Auto-follow 3xx redirects with custom max depth
 - Bi-directional communication with web-socket services
 - [Native mocking support](http://wiki.github.com/igrigorik/em-http-request/mocking-httprequest) and through [Webmock](http://github.com/bblimke/webmock)
@@ -114,7 +114,6 @@ Allows you to efficiently stream a (large) file from disk via EventMachine's Fil
 
 Proxy example
 -------------
-
 Full transparent proxy support with support for SSL tunneling.
 
       EventMachine.run {
@@ -124,6 +123,17 @@ Full transparent proxy support with support for SSL tunneling.
           :authorization => ['username', 'password'] # authorization is optional
       }
 
+SOCKS5 Proxy example
+-------------
+Tunnel your requests via connect via SOCKS5 proxies (ssh -D port somehost).
+
+    EventMachine.run {
+      http = EventMachine::HttpRequest.new('http://www.website.com/').get :proxy => {
+        :host => 'www.myproxy.com',
+        :port => 8080,
+        :type => :socks
+    }
+
 Auto-follow 3xx redirects
 -------------------------
 

data/VERSION

@@ -1 +1 @@
-0.2.12
+0.2.13

data/em-http-request.gemspec

@@ -5,11 +5,11 @@
 
 Gem::Specification.new do |s|
   s.name = %q{em-http-request}
-  s.version = "0.2.12"
+  s.version = "0.2.13"
 
   s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
   s.authors = ["Ilya Grigorik"]
-  s.date = %q{2010-09-12}
+  s.date = %q{2010-09-25}
   s.description = %q{EventMachine based, async HTTP Request interface}
   s.email = %q{ilya@igvita.com}
   s.extensions = ["ext/buffer/extconf.rb", "ext/http11_client/extconf.rb"]
@@ -28,6 +28,7 @@ Gem::Specification.new do |s|
      "examples/fetch.rb",
      "examples/fibered-http.rb",
      "examples/oauth-tweet.rb",
+     "examples/socks5.rb",
      "examples/websocket-handler.rb",
      "examples/websocket-server.rb",
      "ext/buffer/em_buffer.c",
@@ -76,6 +77,7 @@ Gem::Specification.new do |s|
      "examples/fetch.rb",
      "examples/fibered-http.rb",
      "examples/oauth-tweet.rb",
+     "examples/socks5.rb",
      "examples/websocket-handler.rb",
      "examples/websocket-server.rb"
   ]

data/examples/socks5.rb

@@ -0,0 +1,26 @@
+require 'rubygems'
+require 'eventmachine'
+require '../lib/em-http'
+
+EM.run do
+  # Establish a SOCKS5 tunnel via SSH
+  # ssh -D 8000 some_remote_machine
+
+  # http = EM::HttpRequest.new('http://whatismyip.org/').get({
+  http = EM::HttpRequest.new('http://igvita.com/').get({
+    :proxy => {:host => '127.0.0.1', :port => 8000, :type => :socks},
+    :redirects => 2
+  })
+
+  http.callback {
+    puts "#{http.response_header.status} - #{http.response.length} bytes\n"
+    puts http.response
+    EM.stop
+  }
+
+  http.errback {
+    puts "Error: " + http.error
+    puts http.inspect
+    EM.stop
+  }
+end

data/lib/em-http.rb

@@ -1,20 +1,21 @@
-#--
-# Copyright (C)2008 Ilya Grigorik
-# You can redistribute this under the terms of the Ruby license
-# See file LICENSE for details
-#++
-
-require 'eventmachine'
-
-require File.dirname(__FILE__) + '/http11_client'
-require File.dirname(__FILE__) + '/em_buffer'
-
-require File.dirname(__FILE__) + '/em-http/core_ext/hash'
-require File.dirname(__FILE__) + '/em-http/core_ext/bytesize'
-
-require File.dirname(__FILE__) + '/em-http/client'
-require File.dirname(__FILE__) + '/em-http/multi'
-require File.dirname(__FILE__) + '/em-http/request'
-require File.dirname(__FILE__) + '/em-http/decoders'
-require File.dirname(__FILE__) + '/em-http/http_options'
+#--
+# Copyright (C)2008 Ilya Grigorik
+# You can redistribute this under the terms of the Ruby license
+# See file LICENSE for details
+#++
+
+require 'eventmachine'
+require 'socket'
+
+require File.dirname(__FILE__) + '/http11_client'
+require File.dirname(__FILE__) + '/em_buffer'
+
+require File.dirname(__FILE__) + '/em-http/core_ext/hash'
+require File.dirname(__FILE__) + '/em-http/core_ext/bytesize'
+
+require File.dirname(__FILE__) + '/em-http/client'
+require File.dirname(__FILE__) + '/em-http/multi'
+require File.dirname(__FILE__) + '/em-http/request'
+require File.dirname(__FILE__) + '/em-http/decoders'
+require File.dirname(__FILE__) + '/em-http/http_options'
 require File.dirname(__FILE__) + '/em-http/mock'

data/lib/em-http/client.rb

@@ -125,7 +125,7 @@ module EventMachine
 
       # Non CONNECT proxies require that you provide the full request
       # uri in request header, as opposed to a relative path.
-      query = uri.join(query) if proxy and not proxy[:use_connect]
+      query = uri.join(query) if proxy && proxy[:type] != :socks && !proxy[:use_connect]
 
       HTTP_REQUEST_HEADER % [method.to_s.upcase, query]
     end
@@ -224,19 +224,28 @@ module EventMachine
       @stream = nil
       @disconnect = nil
       @state = :response_header
+      @socks_state = nil
     end
 
     # start HTTP request once we establish connection to host
     def connection_completed
+      # if a socks proxy is specified, then a connection request
+      # has to be made to the socks server and we need to wait
+      # for a response code
+      if socks_proxy? and @state == :response_header
+        @state = :connect_socks_proxy
+        send_socks_handshake
+
       # if we need to negotiate the proxy connection first, then
       # issue a CONNECT query and wait for 200 response
-      if connect_proxy? and @state == :response_header
-        @state = :connect_proxy
+      elsif connect_proxy? and @state == :response_header
+        @state = :connect_http_proxy
         send_request_header
 
         # if connecting via proxy, then state will be :proxy_connected,
         # indicating successful tunnel. from here, initiate normal http
         # exchange
+
       else
         @state = :response_header
         ssl = @options[:tls] || @options[:ssl] || {}
@@ -306,9 +315,44 @@ module EventMachine
       end
     end
 
+    # determines if there is enough data in the buffer
+    def has_bytes?(num)
+      @data.size >= num
+    end
+
     def websocket?; @uri.scheme == 'ws'; end
     def proxy?; !@options[:proxy].nil?; end
-    def connect_proxy?; proxy? && (@options[:proxy][:use_connect] == true); end
+
+    # determines if a proxy should be used that uses
+    # http-headers as proxy-mechanism
+    #
+    # this is the default proxy type if none is specified
+    def http_proxy?; proxy? && [nil, :http].include?(@options[:proxy][:type]); end
+
+    # determines if a http-proxy should be used with
+    # the CONNECT verb
+    def connect_proxy?; http_proxy? && (@options[:proxy][:use_connect] == true); end
+
+    # determines if a SOCKS5 proxy should be used
+    def socks_proxy?; proxy? && (@options[:proxy][:type] == :socks); end
+
+    def socks_methods
+      methods = []
+      methods << 2 if !options[:proxy][:authorization].nil? # 2 => Username/Password Authentication
+      methods << 0 # 0 => No Authentication Required
+
+      methods
+    end
+
+    def send_socks_handshake
+      # Method Negotiation as described on
+      # http://www.faqs.org/rfcs/rfc1928.html Section 3
+
+      @socks_state = :method_negotiation
+
+      methods = socks_methods
+      send_data [5, methods.size].pack('CC') + methods.pack('C*')
+    end
 
     def send_request_header
       query   = @options[:query]
@@ -319,7 +363,7 @@ module EventMachine
 
       request_header = nil
 
-      if proxy
+      if http_proxy?
         # initialize headers for the http proxy
         head = proxy[:head] ? munge_header_keys(proxy[:head]) : {}
         head['proxy-authorization'] = proxy[:authorization] if proxy[:authorization]
@@ -327,7 +371,7 @@ module EventMachine
         # if we need to negotiate the tunnel connection first, then
         # issue a CONNECT query to the proxy first. This is an optional
         # flag, by default we will provide full URIs to the proxy
-        if @state == :connect_proxy
+        if @state == :connect_http_proxy
           request_header = HTTP_REQUEST_HEADER % ['CONNECT', "#{@uri.host}:#{@uri.port}"]
         end
       end
@@ -443,7 +487,9 @@ module EventMachine
 
     def dispatch
       while case @state
-          when :connect_proxy
+          when :connect_socks_proxy
+            parse_socks_response
+          when :connect_http_proxy
             parse_response_header
           when :response_header
             parse_response_header
@@ -499,7 +545,7 @@ module EventMachine
         return false
       end
 
-      if @state == :connect_proxy
+      if @state == :connect_http_proxy
         # when a successfull tunnel is established, the proxy responds with a
         # 200 response code. from here, the tunnel is transparent.
         if @response_header.http_status.to_i == 200
@@ -535,10 +581,13 @@ module EventMachine
         end
       end
 
-      # shortcircuit on HEAD requests
+      # Fire callbacks immediately after recieving header requests
+      # if the request method is HEAD. In case of a redirect, terminate
+      # current connection and reinitialize the process.
       if @method == "HEAD"
         @state = :finished
-        unbind
+        close_connection
+        return false
       end
 
       if websocket?
@@ -570,6 +619,114 @@ module EventMachine
       true
     end
 
+    def send_socks_connect_request
+      # TO-DO: Implement address types for IPv6 and Domain
+      begin
+        ip_address = Socket.gethostbyname(@uri.host).last
+        send_data [5, 1, 0, 1, ip_address, @uri.port].flatten.pack('CCCCA4n')
+
+      rescue
+        @state = :invalid
+        on_error "could not resolve host", true
+        return false
+      end
+
+      true
+    end
+
+    # parses socks 5 server responses as specified
+    # on http://www.faqs.org/rfcs/rfc1928.html
+    def parse_socks_response
+      if @socks_state == :method_negotiation
+        return false unless has_bytes? 2
+
+        _, method = @data.read(2).unpack('CC')
+
+        if socks_methods.include?(method)
+          if method == 0
+            @socks_state = :connecting
+
+            return send_socks_connect_request
+
+          elsif method == 2
+            @socks_state = :authenticating
+
+            credentials = @options[:proxy][:authorization]
+            if credentials.size < 2
+              @state = :invalid
+              on_error "username and password are not supplied"
+              return false
+            end
+
+            username, password = credentials
+
+            send_data [5, username.length, username, password.length, password].pack('CCA*CA*')
+          end
+
+        else
+          @state = :invalid
+          on_error "proxy did not accept method"
+          return false
+        end
+
+      elsif @socks_state == :authenticating
+        return false unless has_bytes? 2
+
+        _, status_code = @data.read(2).unpack('CC')
+
+        if status_code == 0
+          # success
+          @socks_state = :connecting
+
+          return send_socks_connect_request
+
+        else
+          # error
+          @state = :invalid
+          on_error "access denied by proxy"
+          return false
+        end
+
+      elsif @socks_state == :connecting
+        return false unless has_bytes? 10
+
+        _, response_code, _, address_type, _, _ = @data.read(10).unpack('CCCCNn')
+
+        if response_code == 0
+          # success
+          @socks_state = :connected
+          @state = :proxy_connected
+
+          @response_header = HttpResponseHeader.new
+
+          # connection_completed will invoke actions to
+          # start sending all http data transparently
+          # over the socks connection
+          connection_completed
+
+        else
+          # error
+          @state = :invalid
+
+          error_messages = {
+            1 => "general socks server failure",
+            2 => "connection not allowed by ruleset",
+            3 => "network unreachable",
+            4 => "host unreachable",
+            5 => "connection refused",
+            6 => "TTL expired",
+            7 => "command not supported",
+            8 => "address type not supported"
+          }
+          error_message = error_messages[response_code] || "unknown error (code: #{response_code})"
+          on_error "socks5 connect error: #{error_message}"
+          return false
+        end
+      end
+
+      true
+    end
+
     def parse_chunk_header
       return false unless parse_header(@chunk_header)
 

data/lib/em-http/core_ext/bytesize.rb

@@ -1,6 +1,6 @@
-# bytesize was introduced in 1.8.7+
-if RUBY_VERSION <= "1.8.6"
-  class String
-    def bytesize; self.size; end
-  end
+# bytesize was introduced in 1.8.7+
+if RUBY_VERSION <= "1.8.6"
+  class String
+    def bytesize; self.size; end
+  end
 end

data/lib/em-http/core_ext/hash.rb

@@ -1,53 +1,53 @@
-class Hash
-  # Stolen partially from Merb : http://noobkit.com/show/ruby/gems/development/merb/hash/to_params.html
-  # Convert this hash to a query string:
-  #   
-  #   { :name => "Bob",
-  #     :address => {
-  #       :street => '111 Ruby Ave.',
-  #       :city => 'Ruby Central',
-  #       :phones => ['111-111-1111', '222-222-2222']
-  #     }
-  #   }.to_params
-  #   #=> "name=Bob&address[city]=Ruby Central&address[phones]=111-111-1111222-222-2222&address[street]=111 Ruby Ave."
-  # 
-  def to_params
-    params = ''
-    stack = []
-    
-    each do |k, v|
-      if v.is_a?(Hash)
-        stack << [k,v]
-      elsif v.is_a?(Array)
-        stack << [k,Hash.from_array(v)]
-      else
-        params << "#{k}=#{v}&"
-      end
-    end
-    
-    stack.each do |parent, hash|
-      hash.each do |k, v|
-        if v.is_a?(Hash)
-          stack << ["#{parent}[#{k}]", v]
-        else
-          params << "#{parent}[#{k}]=#{v}&"
-        end
-      end
-    end
-    
-    params.chop! # trailing &
-    params
-  end
-  
-  ##
-  # Builds a hash from an array with keys as array indices.
-  def self.from_array(array = [])
-    h = Hash.new
-    array.size.times do |t|
-      h[t] = array[t]
-    end
-    h
-  end
-
-end
-
+class Hash
+  # Stolen partially from Merb : http://noobkit.com/show/ruby/gems/development/merb/hash/to_params.html
+  # Convert this hash to a query string:
+  #   
+  #   { :name => "Bob",
+  #     :address => {
+  #       :street => '111 Ruby Ave.',
+  #       :city => 'Ruby Central',
+  #       :phones => ['111-111-1111', '222-222-2222']
+  #     }
+  #   }.to_params
+  #   #=> "name=Bob&address[city]=Ruby Central&address[phones]=111-111-1111222-222-2222&address[street]=111 Ruby Ave."
+  # 
+  def to_params
+    params = ''
+    stack = []
+    
+    each do |k, v|
+      if v.is_a?(Hash)
+        stack << [k,v]
+      elsif v.is_a?(Array)
+        stack << [k,Hash.from_array(v)]
+      else
+        params << "#{k}=#{v}&"
+      end
+    end
+    
+    stack.each do |parent, hash|
+      hash.each do |k, v|
+        if v.is_a?(Hash)
+          stack << ["#{parent}[#{k}]", v]
+        else
+          params << "#{parent}[#{k}]=#{v}&"
+        end
+      end
+    end
+    
+    params.chop! # trailing &
+    params
+  end
+  
+  ##
+  # Builds a hash from an array with keys as array indices.
+  def self.from_array(array = [])
+    h = Hash.new
+    array.size.times do |t|
+      h[t] = array[t]
+    end
+    h
+  end
+
+end
+

data/spec/hash_spec.rb

@@ -1,24 +1,24 @@
-require 'spec/helper'
-
-describe Hash do
-  
-  describe ".to_params" do
-    it "should transform a basic hash into HTTP POST Params" do
-      {:a => "alpha", :b => "beta"}.to_params.split("&").should include "a=alpha"
-      {:a => "alpha", :b => "beta"}.to_params.split("&").should include "b=beta"
-    end
-    
-    it "should transform a more complex hash into HTTP POST Params" do
-      {:a => "a", :b => ["c", "d", "e"]}.to_params.split("&").should include "a=a"
-      {:a => "a", :b => ["c", "d", "e"]}.to_params.split("&").should include "b[0]=c"
-      {:a => "a", :b => ["c", "d", "e"]}.to_params.split("&").should include "b[1]=d"
-      {:a => "a", :b => ["c", "d", "e"]}.to_params.split("&").should include "b[2]=e"
-    end
-
-    it "should transform a very complex hash into HTTP POST Params" do
-      params = {:a => "a", :b => [{:c => "c", :d => "d"}, {:e => "e", :f => "f"}]}.to_params.split("&")
-      params.should include "a=a"
-      params.should include "b[0][d]=d"
-    end
-  end
-end
+require 'spec/helper'
+
+describe Hash do
+  
+  describe ".to_params" do
+    it "should transform a basic hash into HTTP POST Params" do
+      {:a => "alpha", :b => "beta"}.to_params.split("&").should include "a=alpha"
+      {:a => "alpha", :b => "beta"}.to_params.split("&").should include "b=beta"
+    end
+    
+    it "should transform a more complex hash into HTTP POST Params" do
+      {:a => "a", :b => ["c", "d", "e"]}.to_params.split("&").should include "a=a"
+      {:a => "a", :b => ["c", "d", "e"]}.to_params.split("&").should include "b[0]=c"
+      {:a => "a", :b => ["c", "d", "e"]}.to_params.split("&").should include "b[1]=d"
+      {:a => "a", :b => ["c", "d", "e"]}.to_params.split("&").should include "b[2]=e"
+    end
+
+    it "should transform a very complex hash into HTTP POST Params" do
+      params = {:a => "a", :b => [{:c => "c", :d => "d"}, {:e => "e", :f => "f"}]}.to_params.split("&")
+      params.should include "a=a"
+      params.should include "b[0][d]=d"
+    end
+  end
+end

data/spec/request_spec.rb

@@ -99,6 +99,31 @@ describe EventMachine::HttpRequest do
         }
       }
     end
+
+    it "should follow redirects on HEAD method" do
+      EventMachine.run {
+        http = EventMachine::HttpRequest.new('http://127.0.0.1:8080/redirect/head').head :redirects => 1
+        http.errback { failed }
+        http.callback {
+          http.response_header.status.should == 200
+          http.last_effective_url.to_s.should == 'http://127.0.0.1:8080/'
+          EM.stop
+        }
+      }
+    end
+
+    it "should follow redirects on HEAD method (external)" do
+
+      EventMachine.run {
+        http = EventMachine::HttpRequest.new('http://www.google.com/').head :redirects => 1
+        http.errback { failed }
+        http.callback {
+          http.response_header.status.should == 200
+          EM.stop
+        }
+      }
+    end
+
   end
 
   it "should perform successfull GET with a URI passed as argument" do

data/spec/spec.opts

@@ -1,7 +1,7 @@
---colour
---format
-specdoc
---loadby
-mtime
---reverse
+--colour
+--format
+specdoc
+--loadby
+mtime
+--reverse
 --backtrace

data/spec/stallion.rb

@@ -85,7 +85,7 @@ Stallion.saddle :spec do |stable|
     elsif stable.request.path_info == '/echo_content_length'
       stable.response.write stable.request.content_length
 
-    elsif stable.request.head?
+    elsif stable.request.head? && stable.request.path_info == '/'
       stable.response.status = 200
 
     elsif stable.request.delete?
@@ -121,6 +121,10 @@ Stallion.saddle :spec do |stable|
       stable.response.status = 301
       stable.response["Location"] = "http://127.0.0.1:8080"
 
+    elsif stable.request.path_info == '/redirect/head'
+      stable.response.status = 301
+      stable.response["Location"] = "/"
+
     elsif stable.request.path_info == '/redirect/nohost'
       stable.response.status = 301
       stable.response["Location"] = "http:/"

metadata

@@ -5,8 +5,8 @@ version: !ruby/object:Gem::Version
   segments: 
   - 0
   - 2
-  - 12
-  version: 0.2.12
+  - 13
+  version: 0.2.13
 platform: ruby
 authors: 
 - Ilya Grigorik
@@ -14,7 +14,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2010-09-12 00:00:00 -04:00
+date: 2010-09-25 00:00:00 -04:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency 
@@ -68,6 +68,7 @@ files:
 - examples/fetch.rb
 - examples/fibered-http.rb
 - examples/oauth-tweet.rb
+- examples/socks5.rb
 - examples/websocket-handler.rb
 - examples/websocket-server.rb
 - ext/buffer/em_buffer.c
@@ -142,5 +143,6 @@ test_files:
 - examples/fetch.rb
 - examples/fibered-http.rb
 - examples/oauth-tweet.rb
+- examples/socks5.rb
 - examples/websocket-handler.rb
 - examples/websocket-server.rb