em-http-request 0.2.10 → 0.2.14

data/lib/em-http/client.rb

@@ -23,17 +23,16 @@ module EventMachine
 
     # E-Tag
     def etag
-      self["ETag"]
+      self[HttpClient::ETAG]
     end
 
     def last_modified
-      time = self["Last-Modified"]
-      Time.parse(time) if time
+      self[HttpClient::LAST_MODIFIED]
     end
 
     # HTTP response status as an integer
     def status
-      Integer(http_status) rescue nil
+      Integer(http_status) rescue 0
     end
 
     # Length of content as an integer, or nil if chunked/unspecified
@@ -83,9 +82,9 @@ module EventMachine
 
     # Escapes a URI.
     def escape(s)
-      s.to_s.gsub(/([^ a-zA-Z0-9_.-]+)/n) {
-        '%'+$1.unpack('H2'*$1.bytesize).join('%').upcase
-      }.tr(' ', '+')
+      s.to_s.gsub(/([^a-zA-Z0-9_.-]+)/n) {
+        '%'+$1.unpack('H2'*bytesize($1)).join('%').upcase
+      }
     end
 
     # Unescapes a URI escaped string.
@@ -95,6 +94,16 @@ module EventMachine
       }
     end
 
+    if ''.respond_to?(:bytesize)
+      def bytesize(string)
+        string.bytesize
+      end
+    else
+      def bytesize(string)
+        string.size
+      end
+    end
+
     # Map all header keys to a downcased string version
     def munge_header_keys(head)
       head.inject({}) { |h, (k, v)| h[k.to_s.downcase] = v; h }
@@ -111,21 +120,27 @@ module EventMachine
       end
     end
 
-    def encode_request(method, path, query, uri_query)
-      HTTP_REQUEST_HEADER % [method.to_s.upcase, encode_query(path, query, uri_query)]
+    def encode_request(method, uri, query, proxy)
+      query = encode_query(uri, query)
+
+      # Non CONNECT proxies require that you provide the full request
+      # uri in request header, as opposed to a relative path.
+      query = uri.join(query) if proxy && proxy[:type] != :socks && !proxy[:use_connect]
+
+      HTTP_REQUEST_HEADER % [method.to_s.upcase, query]
     end
 
-    def encode_query(path, query, uri_query)
+    def encode_query(uri, query)
       encoded_query = if query.kind_of?(Hash)
         query.map { |k, v| encode_param(k, v) }.join('&')
       else
         query.to_s
       end
-      if !uri_query.to_s.empty?
-        encoded_query = [encoded_query, uri_query].reject {|part| part.empty?}.join("&")
+
+      if !uri.query.to_s.empty?
+        encoded_query = [encoded_query, uri.query].reject {|part| part.empty?}.join("&")
       end
-      return path if encoded_query.to_s.empty?
-      "#{path}?#{encoded_query}"
+      encoded_query.to_s.empty? ? uri.path : "#{uri.path}?#{encoded_query}"
     end
 
     # URL encodes query parameters:
@@ -138,6 +153,29 @@ module EventMachine
       end
     end
 
+    def form_encode_body(obj)
+      pairs = []
+      recursive = Proc.new do |h, prefix|
+        h.each do |k,v|
+          key = prefix == '' ? escape(k) : "#{prefix}[#{escape(k)}]"
+
+          if v.is_a? Array
+            nh = Hash.new
+            v.size.times { |t| nh[t] = v[t] }
+            recursive.call(nh, key)
+
+          elsif v.is_a? Hash
+            recursive.call(v, key)
+          else
+            pairs << "#{key}=#{escape(v)}"
+          end
+        end
+      end
+
+      recursive.call(obj, '')
+      return pairs.join('&')
+    end
+
     # Encode a field in an HTTP header
     def encode_field(k, v)
       FIELD_ENCODING % [k, v]
@@ -183,10 +221,13 @@ module EventMachine
     TRANSFER_ENCODING="TRANSFER_ENCODING"
     CONTENT_ENCODING="CONTENT_ENCODING"
     CONTENT_LENGTH="CONTENT_LENGTH"
+    LAST_MODIFIED="LAST_MODIFIED"
     KEEP_ALIVE="CONNECTION"
     SET_COOKIE="SET_COOKIE"
     LOCATION="LOCATION"
     HOST="HOST"
+    ETAG="ETAG"
+
     CRLF="\r\n"
 
     attr_accessor :method, :options, :uri
@@ -206,19 +247,28 @@ module EventMachine
       @stream = nil
       @disconnect = nil
       @state = :response_header
+      @socks_state = nil
     end
 
     # start HTTP request once we establish connection to host
     def connection_completed
-      # if connecting to proxy, then first negotiate the connection
-      # to intermediate server and wait for 200 response
-      if @options[:proxy] and @state == :response_header
-        @state = :response_proxy
+      # if a socks proxy is specified, then a connection request
+      # has to be made to the socks server and we need to wait
+      # for a response code
+      if socks_proxy? and @state == :response_header
+        @state = :connect_socks_proxy
+        send_socks_handshake
+
+        # if we need to negotiate the proxy connection first, then
+        # issue a CONNECT query and wait for 200 response
+      elsif connect_proxy? and @state == :response_header
+        @state = :connect_http_proxy
         send_request_header
 
         # if connecting via proxy, then state will be :proxy_connected,
         # indicating successful tunnel. from here, initiate normal http
         # exchange
+
       else
         @state = :response_header
         ssl = @options[:tls] || @options[:ssl] || {}
@@ -247,6 +297,7 @@ module EventMachine
       # fail the connection directly
       dns_error == true ? fail(self) : unbind
     end
+    alias :close :on_error
 
     # assign a stream processing block
     def stream(&blk)
@@ -258,6 +309,11 @@ module EventMachine
       @disconnect = blk
     end
 
+    # assign a headers parse callback
+    def headers(&blk)
+      @headers = blk
+    end
+
     # raw data push from the client (WebSocket) should
     # only be invoked after handshake, otherwise it will
     # inject data into the header exchange
@@ -275,31 +331,75 @@ module EventMachine
     def normalize_body
       @normalized_body ||= begin
         if @options[:body].is_a? Hash
-          @options[:body].to_params
+          form_encode_body(@options[:body])
         else
           @options[:body]
         end
       end
     end
 
+    # determines if there is enough data in the buffer
+    def has_bytes?(num)
+      @data.size >= num
+    end
+
     def websocket?; @uri.scheme == 'ws'; end
+    def proxy?; !@options[:proxy].nil?; end
+
+    # determines if a proxy should be used that uses
+    # http-headers as proxy-mechanism
+    #
+    # this is the default proxy type if none is specified
+    def http_proxy?; proxy? && [nil, :http].include?(@options[:proxy][:type]); end
+
+    # determines if a http-proxy should be used with
+    # the CONNECT verb
+    def connect_proxy?; http_proxy? && (@options[:proxy][:use_connect] == true); end
+
+    # determines if a SOCKS5 proxy should be used
+    def socks_proxy?; proxy? && (@options[:proxy][:type] == :socks); end
+
+    def socks_methods
+      methods = []
+      methods << 2 if !options[:proxy][:authorization].nil? # 2 => Username/Password Authentication
+      methods << 0 # 0 => No Authentication Required
+
+      methods
+    end
+
+    def send_socks_handshake
+      # Method Negotiation as described on
+      # http://www.faqs.org/rfcs/rfc1928.html Section 3
+
+      @socks_state = :method_negotiation
+
+      methods = socks_methods
+      send_data [5, methods.size].pack('CC') + methods.pack('C*')
+    end
 
     def send_request_header
       query   = @options[:query]
       head    = @options[:head] ? munge_header_keys(@options[:head]) : {}
       file    = @options[:file]
+      proxy   = @options[:proxy]
       body    = normalize_body
-      request_header = nil
 
-      if @state == :response_proxy
-        proxy = @options[:proxy]
+      request_header = nil
 
-        # initialize headers to establish the HTTP tunnel
+      if http_proxy?
+        # initialize headers for the http proxy
         head = proxy[:head] ? munge_header_keys(proxy[:head]) : {}
         head['proxy-authorization'] = proxy[:authorization] if proxy[:authorization]
-        request_header = HTTP_REQUEST_HEADER % ['CONNECT', "#{@uri.host}:#{@uri.port}"]
 
-      elsif websocket?
+        # if we need to negotiate the tunnel connection first, then
+        # issue a CONNECT query to the proxy first. This is an optional
+        # flag, by default we will provide full URIs to the proxy
+        if @state == :connect_http_proxy
+          request_header = HTTP_REQUEST_HEADER % ['CONNECT', "#{@uri.host}:#{@uri.port}"]
+        end
+      end
+
+      if websocket?
         head['upgrade'] = 'WebSocket'
         head['connection'] = 'Upgrade'
         head['origin'] = @options[:origin] || @uri.host
@@ -332,7 +432,7 @@ module EventMachine
       @last_effective_url = @uri
 
       # Build the request headers
-      request_header ||= encode_request(@method, @uri.path, query, @uri.query)
+      request_header ||= encode_request(@method, @uri, query, proxy)
       request_header << encode_headers(head)
       request_header << CRLF
       send_data request_header
@@ -376,19 +476,24 @@ module EventMachine
 
     def unbind
       if (@state == :finished) && (@last_effective_url != @uri) && (@redirects < @options[:redirects])
-        # update uri to redirect location if we're allowed to traverse deeper
-        @uri = @last_effective_url
+        begin
+          # update uri to redirect location if we're allowed to traverse deeper
+          @uri = @last_effective_url
 
-        # keep track of the depth of requests we made in this session
-        @redirects += 1
+          # keep track of the depth of requests we made in this session
+          @redirects += 1
 
-        # swap current connection and reassign current handler
-        req = HttpOptions.new(@method, @uri, @options)
-        reconnect(req.host, req.port)
+          # swap current connection and reassign current handler
+          req = HttpOptions.new(@method, @uri, @options)
+          reconnect(req.host, req.port)
 
-        @response_header = HttpResponseHeader.new
-        @state = :response_header
-        @data.clear
+          @response_header = HttpResponseHeader.new
+          @state = :response_header
+          @response = ''
+          @data.clear
+        rescue EventMachine::ConnectionError => e
+          on_error(e.message, true)
+        end
       else
         if @state == :finished || (@state == :body && @bytes_remaining.nil?)
           succeed(self)
@@ -405,7 +510,9 @@ module EventMachine
 
     def dispatch
       while case @state
-          when :response_proxy
+          when :connect_socks_proxy
+            parse_socks_response
+          when :connect_http_proxy
             parse_response_header
           when :response_header
             parse_response_header
@@ -451,13 +558,17 @@ module EventMachine
     def parse_response_header
       return false unless parse_header(@response_header)
 
+      # invoke headers callback after full parse if one
+      # is specified by the user
+      @headers.call(@response_header) if @headers
+
       unless @response_header.http_status and @response_header.http_reason
         @state = :invalid
         on_error "no HTTP response"
         return false
       end
 
-      if @state == :response_proxy
+      if @state == :connect_http_proxy
         # when a successfull tunnel is established, the proxy responds with a
         # 200 response code. from here, the tunnel is transparent.
         if @response_header.http_status.to_i == 200
@@ -475,9 +586,13 @@ module EventMachine
       if @response_header.location
         begin
           location = Addressable::URI.parse(@response_header.location)
+
           if location.relative?
             location = @uri.join(location)
             @response_header[LOCATION] = location.to_s
+          else
+            # if redirect is to an absolute url, check for correct URI structure
+            raise if location.host.nil?
           end
 
           # store last url on any sign of redirect
@@ -489,10 +604,13 @@ module EventMachine
         end
       end
 
-      # shortcircuit on HEAD requests
+      # Fire callbacks immediately after recieving header requests
+      # if the request method is HEAD. In case of a redirect, terminate
+      # current connection and reinitialize the process.
       if @method == "HEAD"
         @state = :finished
-        unbind
+        close_connection
+        return false
       end
 
       if websocket?
@@ -524,6 +642,114 @@ module EventMachine
       true
     end
 
+    def send_socks_connect_request
+      # TO-DO: Implement address types for IPv6 and Domain
+      begin
+        ip_address = Socket.gethostbyname(@uri.host).last
+        send_data [5, 1, 0, 1, ip_address, @uri.port].flatten.pack('CCCCA4n')
+
+      rescue
+        @state = :invalid
+        on_error "could not resolve host", true
+        return false
+      end
+
+      true
+    end
+
+    # parses socks 5 server responses as specified
+    # on http://www.faqs.org/rfcs/rfc1928.html
+    def parse_socks_response
+      if @socks_state == :method_negotiation
+        return false unless has_bytes? 2
+
+        _, method = @data.read(2).unpack('CC')
+
+        if socks_methods.include?(method)
+          if method == 0
+            @socks_state = :connecting
+
+            return send_socks_connect_request
+
+          elsif method == 2
+            @socks_state = :authenticating
+
+            credentials = @options[:proxy][:authorization]
+            if credentials.size < 2
+              @state = :invalid
+              on_error "username and password are not supplied"
+              return false
+            end
+
+            username, password = credentials
+
+            send_data [5, username.length, username, password.length, password].pack('CCA*CA*')
+          end
+
+        else
+          @state = :invalid
+          on_error "proxy did not accept method"
+          return false
+        end
+
+      elsif @socks_state == :authenticating
+        return false unless has_bytes? 2
+
+        _, status_code = @data.read(2).unpack('CC')
+
+        if status_code == 0
+          # success
+          @socks_state = :connecting
+
+          return send_socks_connect_request
+
+        else
+          # error
+          @state = :invalid
+          on_error "access denied by proxy"
+          return false
+        end
+
+      elsif @socks_state == :connecting
+        return false unless has_bytes? 10
+
+        _, response_code, _, address_type, _, _ = @data.read(10).unpack('CCCCNn')
+
+        if response_code == 0
+          # success
+          @socks_state = :connected
+          @state = :proxy_connected
+
+          @response_header = HttpResponseHeader.new
+
+          # connection_completed will invoke actions to
+          # start sending all http data transparently
+          # over the socks connection
+          connection_completed
+
+        else
+          # error
+          @state = :invalid
+
+          error_messages = {
+            1 => "general socks server failure",
+            2 => "connection not allowed by ruleset",
+            3 => "network unreachable",
+            4 => "host unreachable",
+            5 => "connection refused",
+            6 => "TTL expired",
+            7 => "command not supported",
+            8 => "address type not supported"
+          }
+          error_message = error_messages[response_code] || "unknown error (code: #{response_code})"
+          on_error "socks5 connect error: #{error_message}"
+          return false
+        end
+      end
+
+      true
+    end
+
     def parse_chunk_header
       return false unless parse_header(@chunk_header)
 
@@ -628,8 +854,8 @@ module EventMachine
       # slice the message out of the buffer and pass in
       # for processing, and buffer data otherwise
       buffer = @data.read
-      while msg = buffer.slice!(/\000([^\377]*)\377/)
-        msg.gsub!(/^\x00|\xff$/, '')
+      while msg = buffer.slice!(/\000([^\377]*)\377/n)
+        msg.gsub!(/\A\x00|\xff\z/n, '')
         @stream.call(msg)
       end
 

data/lib/em-http/http_options.rb

@@ -2,7 +2,7 @@ class HttpOptions
   attr_reader :uri, :method, :host, :port, :options
 
   def initialize(method, uri, options)
-    raise ArgumentError, "invalid request path" unless /^\// === uri.path
+    uri.normalize!
 
     @options = options
     @method = method.to_s.upcase
@@ -12,7 +12,9 @@ class HttpOptions
       @host = proxy[:host]
       @port = proxy[:port]
     else
-      @host = uri.host
+      # optional host for cases where you may have
+      # pre-resolved the host, or you need an override
+      @host = options.delete(:host) || uri.host
       @port = uri.port
     end
 

data/lib/em-http/mock.rb

@@ -1,91 +1,131 @@
 module EventMachine
+  OriginalHttpRequest = HttpRequest unless const_defined?(:OriginalHttpRequest)
+
   class MockHttpRequest < EventMachine::HttpRequest
-    
+
     include HttpEncoding
-    
-    class FakeHttpClient < EventMachine::HttpClient
 
+    class RegisteredRequest < Struct.new(:uri, :method, :headers)
+      def self.build(uri, method, headers)
+        new(uri, method.to_s.upcase, headers || {})
+      end
+    end
+
+    class FakeHttpClient < EventMachine::HttpClient
+      attr_writer :response
+      attr_reader :data
       def setup(response, uri)
         @uri = uri
         if response == :fail
           fail(self)
         else
-          receive_data(response)
-          succeed(self)
+          if response.respond_to?(:call)
+            response.call(self)
+            @state = :body
+          else
+            receive_data(response)
+          end
+          @state == :body ? succeed(self) : fail(self)
         end
       end
-      
+
       def unbind
       end
-      
     end
-    
-    @@registry = nil
-    @@registry_count = nil
-    
+
+    @@registry = Hash.new
+    @@registry_count = Hash.new{|h,k| h[k] = 0}
+
+    def self.use
+      activate!
+      yield
+    ensure
+      deactivate!
+    end
+
+    def self.activate!
+      EventMachine.send(:remove_const, :HttpRequest)
+      EventMachine.send(:const_set, :HttpRequest, MockHttpRequest)
+    end
+
+    def self.deactivate!
+      EventMachine.send(:remove_const, :HttpRequest)
+      EventMachine.send(:const_set, :HttpRequest, OriginalHttpRequest)
+    end
+
     def self.reset_counts!
-      @@registry_count = Hash.new do |registry,query| 
-        registry[query] = Hash.new{|h,k| h[k] = Hash.new(0)}
-      end
+      @@registry_count.clear
     end
-    
+
     def self.reset_registry!
-      @@registry = Hash.new do |registry,query| 
-        registry[query] = Hash.new{|h,k| h[k] = {}}
-      end
+      @@registry.clear
     end
-    
-    reset_counts!
-    reset_registry!
-    
+
     @@pass_through_requests = true
 
     def self.pass_through_requests=(pass_through_requests)
       @@pass_through_requests = pass_through_requests
     end
-    
+
     def self.pass_through_requests
       @@pass_through_requests
     end
-    
-    def self.register(uri, method, headers, data)
-      method = method.to_s.upcase
-      headers = headers.to_s
-      @@registry[uri][method][headers] = data
+
+    def self.parse_register_args(args, &proc)
+      args << proc{|client| proc.call(client); ''} if proc
+      headers, data = case args.size
+      when 3
+        args[2].is_a?(Hash) ?
+          [args[2][:headers], args[2][:data]] :
+          [{}, args[2]]
+      when 4
+        [args[2], args[3]]
+      else
+        raise
+      end
+
+      url = args[0]
+      method = args[1]
+      [headers, url, method, data]
     end
-    
-    def self.register_file(uri, method, headers, file)
-      register(uri, method, headers, File.read(file))
+
+    def self.register(*args, &proc)
+      headers, url, method, data = parse_register_args(args, &proc)
+      @@registry[RegisteredRequest.build(url, method, headers)] = data
     end
-    
-    def self.count(uri, method, headers)
-      method = method.to_s.upcase
-      headers = headers.to_s
-      @@registry_count[uri][method][headers] rescue 0
+
+    def self.register_file(*args)
+      headers, url, method, data = parse_register_args(args)
+      @@registry[RegisteredRequest.build(url, method, headers)] = File.read(data)
+    end
+
+    def self.count(url, method, headers = {})
+      @@registry_count[RegisteredRequest.build(url, method, headers)]
     end
-    
-    def self.registered?(query, method, headers)
-      @@registry[query] and @@registry[query][method] and @@registry[query][method][headers]
+
+    def self.registered?(url, method, headers = {})
+      @@registry.key?(RegisteredRequest.build(url, method, headers))
     end
-    
-    def self.registered_content(query, method, headers)
-      @@registry[query][method][headers]
+
+    def self.registered_content(url, method, headers = {})
+      @@registry[RegisteredRequest.build(url, method, headers)]
     end
-    
-    def self.increment_access(query, method, headers)
-      @@registry_count[query][method][headers] += 1
+
+    def self.increment_access(url, method, headers = {})
+      @@registry_count[RegisteredRequest.build(url, method, headers)] += 1
     end
-    
+
     alias_method :real_send_request, :send_request
-    
+
     protected
     def send_request(&blk)
-      query = "#{@req.uri.scheme}://#{@req.uri.host}:#{@req.uri.port}#{encode_query(@req.uri.path, @req.options[:query], @req.uri.query)}"
-      headers = @req.options[:head].to_s
+      query = "#{@req.uri.scheme}://#{@req.uri.host}:#{@req.uri.port}#{encode_query(@req.uri, @req.options[:query])}"
+      headers = @req.options[:head]
       if self.class.registered?(query, @req.method, headers)
         self.class.increment_access(query, @req.method, headers)
         client = FakeHttpClient.new(nil)
-        client.setup(self.class.registered_content(query, @req.method, headers), @req.uri)
+        content = self.class.registered_content(query, @req.method, headers)
+        client.setup(content, @req.uri)
         client
       elsif @@pass_through_requests
         real_send_request