em-http-request 0.2.9 → 0.3.0

data/lib/em-http/http_encoding.rb

@@ -0,0 +1,135 @@
+module EventMachine
+  module HttpEncoding
+    HTTP_REQUEST_HEADER="%s %s HTTP/1.1\r\n"
+    FIELD_ENCODING = "%s: %s\r\n"
+
+    # Escapes a URI.
+    def escape(s)
+      EscapeUtils.escape_url(s.to_s)
+    end
+
+    # Unescapes a URI escaped string.
+    def unescape(s)
+      EscapeUtils.unescape_url(s.to_s)
+    end
+
+    if ''.respond_to?(:bytesize)
+      def bytesize(string)
+        string.bytesize
+      end
+    else
+      def bytesize(string)
+        string.size
+      end
+    end
+
+    # Map all header keys to a downcased string version
+    def munge_header_keys(head)
+      head.inject({}) { |h, (k, v)| h[k.to_s.downcase] = v; h }
+    end
+
+    # HTTP is kind of retarded that you have to specify a Host header, but if
+    # you include port 80 then further redirects will tack on the :80 which is
+    # annoying.
+    def encode_host
+      if @uri.port == 80 || @uri.port == 443
+        return @uri.host
+      else
+        @uri.host + ":#{@uri.port}"
+      end
+    end
+
+    def encode_request(method, uri, query, proxy)
+      query = encode_query(uri, query)
+
+      # Non CONNECT proxies require that you provide the full request
+      # uri in request header, as opposed to a relative path.
+      query = uri.join(query) if proxy && proxy[:type] != :socks && !proxy[:use_connect]
+
+      HTTP_REQUEST_HEADER % [method.to_s.upcase, query]
+    end
+
+    def encode_query(uri, query)
+      encoded_query = if query.kind_of?(Hash)
+        query.map { |k, v| encode_param(k, v) }.join('&')
+      else
+        query.to_s
+      end
+
+      if !uri.query.to_s.empty?
+        encoded_query = [encoded_query, uri.query].reject {|part| part.empty?}.join("&")
+      end
+      encoded_query.to_s.empty? ? uri.path : "#{uri.path}?#{encoded_query}"
+    end
+
+    # URL encodes query parameters:
+    # single k=v, or a URL encoded array, if v is an array of values
+    def encode_param(k, v)
+      if v.is_a?(Array)
+        v.map { |e| escape(k) + "[]=" + escape(e) }.join("&")
+      else
+        escape(k) + "=" + escape(v)
+      end
+    end
+
+    def form_encode_body(obj)
+      pairs = []
+      recursive = Proc.new do |h, prefix|
+        h.each do |k,v|
+          key = prefix == '' ? escape(k) : "#{prefix}[#{escape(k)}]"
+
+          if v.is_a? Array
+            nh = Hash.new
+            v.size.times { |t| nh[t] = v[t] }
+            recursive.call(nh, key)
+
+          elsif v.is_a? Hash
+            recursive.call(v, key)
+          else
+            pairs << "#{key}=#{escape(v)}"
+          end
+        end
+      end
+
+      recursive.call(obj, '')
+      return pairs.join('&')
+    end
+
+    # Encode a field in an HTTP header
+    def encode_field(k, v)
+      FIELD_ENCODING % [k, v]
+    end
+
+    # Encode basic auth in an HTTP header
+    # In: Array ([user, pass]) - for basic auth
+    #     String - custom auth string (OAuth, etc)
+    def encode_auth(k,v)
+      if v.is_a? Array
+        FIELD_ENCODING % [k, ["Basic", Base64.encode64(v.join(":")).chomp].join(" ")]
+      else
+        encode_field(k,v)
+      end
+    end
+
+    def encode_headers(head)
+      head.inject('') do |result, (key, value)|
+        # Munge keys from foo-bar-baz to Foo-Bar-Baz
+        key = key.split('-').map { |k| k.to_s.capitalize }.join('-')
+        result << case key
+          when 'Authorization', 'Proxy-Authorization'
+            encode_auth(key, value)
+          else
+            encode_field(key, value)
+        end
+      end
+    end
+
+    def encode_cookie(cookie)
+      if cookie.is_a? Hash
+        cookie.inject('') { |result, (k, v)| result <<  encode_param(k, v) + ";" }
+      else
+        cookie
+      end
+    end
+  end
+end

data/lib/em-http/http_header.rb

@@ -0,0 +1,71 @@
+module EventMachine
+  # A simple hash is returned for each request made by HttpClient with the
+  # headers that were given by the server for that request.
+  class HttpResponseHeader < Hash
+    # The reason returned in the http response ("OK","File not found",etc.)
+    attr_accessor :http_reason
+
+    # The HTTP version returned.
+    attr_accessor :http_version
+
+    # The status code (as a string!)
+    attr_accessor :http_status
+
+    # E-Tag
+    def etag
+      self[HttpClient::ETAG]
+    end
+
+    def last_modified
+      self[HttpClient::LAST_MODIFIED]
+    end
+
+    # HTTP response status as an integer
+    def status
+      Integer(http_status) rescue 0
+    end
+
+    # Length of content as an integer, or nil if chunked/unspecified
+    def content_length
+      @content_length ||= ((s = self[HttpClient::CONTENT_LENGTH]) &&
+                           (s =~ /^(\d+)$/)) ? $1.to_i : nil
+    end
+
+    # Cookie header from the server
+    def cookie
+      self[HttpClient::SET_COOKIE]
+    end
+
+    # Is the transfer encoding chunked?
+    def chunked_encoding?
+      /chunked/i === self[HttpClient::TRANSFER_ENCODING]
+    end
+
+    def keep_alive?
+      /keep-alive/i === self[HttpClient::KEEP_ALIVE]
+    end
+
+    def compressed?
+      /gzip|compressed|deflate/i === self[HttpClient::CONTENT_ENCODING]
+    end
+
+    def location
+      self[HttpClient::LOCATION]
+    end
+  end
+
+  class HttpChunkHeader < Hash
+    # When parsing chunked encodings this is set
+    attr_accessor :http_chunk_size
+
+    def initialize
+      super
+      @http_chunk_size = '0'
+    end
+
+    # Size of the chunk as an integer
+    def chunk_size
+      @http_chunk_size.to_i(base=16)
+    end
+  end
+end

data/lib/em-http/http_options.rb

@@ -2,7 +2,7 @@ class HttpOptions
   attr_reader :uri, :method, :host, :port, :options
 
   def initialize(method, uri, options)
-    raise ArgumentError, "invalid request path" unless /^\// === uri.path
+    uri.path = '/' if uri.path.empty?
 
     @options = options
     @method = method.to_s.upcase
@@ -12,12 +12,15 @@ class HttpOptions
       @host = proxy[:host]
       @port = proxy[:port]
     else
-      @host = uri.host
+      # optional host for cases where you may have
+      # pre-resolved the host, or you need an override
+      @host = options.delete(:host) || uri.host
       @port = uri.port
     end
 
-    @options[:timeout]    ||= 10  # default connect & inactivity timeouts
-    @options[:redirects]  ||= 0   # default number of redirects to follow
+    @options[:timeout]    ||= 10    # default connect & inactivity timeouts
+    @options[:redirects]  ||= 0     # default number of redirects to follow
+    @options[:keepalive]  ||= false # default to single request per connection
 
     # Make sure the ports are set as Addressable::URI doesn't
     # set the port if it isn't there

data/lib/em-http/mock.rb

@@ -1,91 +1,131 @@
 module EventMachine
+  OriginalHttpRequest = HttpRequest unless const_defined?(:OriginalHttpRequest)
+
   class MockHttpRequest < EventMachine::HttpRequest
-    
+
     include HttpEncoding
-    
-    class FakeHttpClient < EventMachine::HttpClient
 
+    class RegisteredRequest < Struct.new(:uri, :method, :headers)
+      def self.build(uri, method, headers)
+        new(uri, method.to_s.upcase, headers || {})
+      end
+    end
+
+    class FakeHttpClient < EventMachine::HttpClient
+      attr_writer :response
+      attr_reader :data
       def setup(response, uri)
         @uri = uri
         if response == :fail
           fail(self)
         else
-          receive_data(response)
-          succeed(self)
+          if response.respond_to?(:call)
+            response.call(self)
+            @state = :body
+          else
+            receive_data(response)
+          end
+          @state == :body ? succeed(self) : fail(self)
         end
       end
-      
+
       def unbind
       end
-      
     end
-    
-    @@registry = nil
-    @@registry_count = nil
-    
+
+    @@registry = Hash.new
+    @@registry_count = Hash.new{|h,k| h[k] = 0}
+
+    def self.use
+      activate!
+      yield
+    ensure
+      deactivate!
+    end
+
+    def self.activate!
+      EventMachine.send(:remove_const, :HttpRequest)
+      EventMachine.send(:const_set, :HttpRequest, MockHttpRequest)
+    end
+
+    def self.deactivate!
+      EventMachine.send(:remove_const, :HttpRequest)
+      EventMachine.send(:const_set, :HttpRequest, OriginalHttpRequest)
+    end
+
     def self.reset_counts!
-      @@registry_count = Hash.new do |registry,query| 
-        registry[query] = Hash.new{|h,k| h[k] = Hash.new(0)}
-      end
+      @@registry_count.clear
     end
-    
+
     def self.reset_registry!
-      @@registry = Hash.new do |registry,query| 
-        registry[query] = Hash.new{|h,k| h[k] = {}}
-      end
+      @@registry.clear
     end
-    
-    reset_counts!
-    reset_registry!
-    
+
     @@pass_through_requests = true
 
     def self.pass_through_requests=(pass_through_requests)
       @@pass_through_requests = pass_through_requests
     end
-    
+
     def self.pass_through_requests
       @@pass_through_requests
     end
-    
-    def self.register(uri, method, headers, data)
-      method = method.to_s.upcase
-      headers = headers.to_s
-      @@registry[uri][method][headers] = data
+
+    def self.parse_register_args(args, &proc)
+      args << proc{|client| proc.call(client); ''} if proc
+      headers, data = case args.size
+      when 3
+        args[2].is_a?(Hash) ?
+          [args[2][:headers], args[2][:data]] :
+          [{}, args[2]]
+      when 4
+        [args[2], args[3]]
+      else
+        raise
+      end
+
+      url = args[0]
+      method = args[1]
+      [headers, url, method, data]
     end
-    
-    def self.register_file(uri, method, headers, file)
-      register(uri, method, headers, File.read(file))
+
+    def self.register(*args, &proc)
+      headers, url, method, data = parse_register_args(args, &proc)
+      @@registry[RegisteredRequest.build(url, method, headers)] = data
     end
-    
-    def self.count(uri, method, headers)
-      method = method.to_s.upcase
-      headers = headers.to_s
-      @@registry_count[uri][method][headers] rescue 0
+
+    def self.register_file(*args)
+      headers, url, method, data = parse_register_args(args)
+      @@registry[RegisteredRequest.build(url, method, headers)] = File.read(data)
+    end
+
+    def self.count(url, method, headers = {})
+      @@registry_count[RegisteredRequest.build(url, method, headers)]
     end
-    
-    def self.registered?(query, method, headers)
-      @@registry[query] and @@registry[query][method] and @@registry[query][method][headers]
+
+    def self.registered?(url, method, headers = {})
+      @@registry.key?(RegisteredRequest.build(url, method, headers))
     end
-    
-    def self.registered_content(query, method, headers)
-      @@registry[query][method][headers]
+
+    def self.registered_content(url, method, headers = {})
+      @@registry[RegisteredRequest.build(url, method, headers)]
     end
-    
-    def self.increment_access(query, method, headers)
-      @@registry_count[query][method][headers] += 1
+
+    def self.increment_access(url, method, headers = {})
+      @@registry_count[RegisteredRequest.build(url, method, headers)] += 1
     end
-    
+
     alias_method :real_send_request, :send_request
-    
+
     protected
     def send_request(&blk)
-      query = "#{@req.uri.scheme}://#{@req.uri.host}:#{@req.uri.port}#{encode_query(@req.uri.path, @req.options[:query], @req.uri.query)}"
-      headers = @req.options[:head].to_s
+      query = "#{@req.uri.scheme}://#{@req.uri.host}:#{@req.uri.port}#{encode_query(@req.uri, @req.options[:query])}"
+      headers = @req.options[:head]
       if self.class.registered?(query, @req.method, headers)
         self.class.increment_access(query, @req.method, headers)
         client = FakeHttpClient.new(nil)
-        client.setup(self.class.registered_content(query, @req.method, headers), @req.uri)
+        content = self.class.registered_content(query, @req.method, headers)
+        client.setup(content, @req.uri)
         client
       elsif @@pass_through_requests
         real_send_request

data/lib/em-http/multi.rb

@@ -1,51 +1,55 @@
-module EventMachine
-
-  # EventMachine based Multi request client, based on a streaming HTTPRequest class, 
-  # which allows you to open multiple parallel connections and return only when all
-  # of them finish. (i.e. ideal for parallelizing workloads)
-  # 
-  # == Example
-  # 
-  #  EventMachine.run {
-  #   
-  #   multi = EventMachine::MultiRequest.new
-  #    
-  #   # add multiple requests to the multi-handler
-  #   multi.add(EventMachine::HttpRequest.new('http://www.google.com/').get)
-  #   multi.add(EventMachine::HttpRequest.new('http://www.yahoo.com/').get)
-  #    
-  #    multi.callback  {
-  #       p multi.responses[:succeeded]
-  #       p multi.responses[:failed]
-  #     
-  #       EventMachine.stop
-  #    }
-  #  }
-  # 
-  
-  class MultiRequest
-    include EventMachine::Deferrable
-
-    attr_reader :requests, :responses
-    
-    def initialize
-      @requests = []
-      @responses = {:succeeded => [], :failed => []}
-    end
-    
-    def add(conn)
-      @requests.push(conn)
-
-      conn.callback { @responses[:succeeded].push(conn); check_progress }
-      conn.errback  { @responses[:failed].push(conn); check_progress }
-    end
-    
-    protected
-   
-    # invoke callback if all requests have completed
-    def check_progress
-      succeed if (@responses[:succeeded].size + @responses[:failed].size) == @requests.size
-    end
-    
-  end
-end
+module EventMachine
+
+  # EventMachine based Multi request client, based on a streaming HTTPRequest class,
+  # which allows you to open multiple parallel connections and return only when all
+  # of them finish. (i.e. ideal for parallelizing workloads)
+  #
+  # == Example
+  #
+  #  EventMachine.run {
+  #
+  #   multi = EventMachine::MultiRequest.new
+  #
+  #   # add multiple requests to the multi-handler
+  #   multi.add(EventMachine::HttpRequest.new('http://www.google.com/').get)
+  #   multi.add(EventMachine::HttpRequest.new('http://www.yahoo.com/').get)
+  #
+  #    multi.callback  {
+  #       p multi.responses[:succeeded]
+  #       p multi.responses[:failed]
+  #
+  #       EventMachine.stop
+  #    }
+  #  }
+  #
+
+  class MultiRequest
+    include EventMachine::Deferrable
+
+    attr_reader :requests, :responses
+
+    def initialize(conns=[], &block)
+      @requests  = []
+      @responses = {:succeeded => [], :failed => []}
+
+      conns.each {|conn| add(conn)}
+      callback(&block) if block_given?
+    end
+
+    def add(conn)
+      @requests.push(conn)
+
+      conn.callback { @responses[:succeeded].push(conn); check_progress }
+      conn.errback  { @responses[:failed].push(conn); check_progress }
+    end
+
+    protected
+
+    # invoke callback if all requests have completed
+    def check_progress
+      succeed(self) if (@responses[:succeeded].size +
+                        @responses[:failed].size) == @requests.size
+    end
+
+  end
+end

data/lib/em-http/request.rb

@@ -1,6 +1,3 @@
-require 'base64'
-require 'addressable/uri'
-
 module EventMachine
 
   # EventMachine based HTTP request class with support for streaming consumption
@@ -27,7 +24,7 @@ module EventMachine
     attr_reader :options, :method
 
     def initialize(host)
-      @uri = host.kind_of?(Addressable::URI) ? host : Addressable::URI::parse(host)
+      @uri = host.kind_of?(Addressable::URI) ? host : Addressable::URI::parse(host.to_s)
     end
 
     # Send an HTTP request and consume the response. Supported options:
@@ -72,6 +69,7 @@ module EventMachine
       rescue EventMachine::ConnectionError => e
         conn = EventMachine::HttpClient.new("")
         conn.on_error(e.message, true)
+        conn.uri = @req.uri
         conn
       end
     end

data/lib/em-http/version.rb

@@ -0,0 +1,5 @@
+module EventMachine
+  class HttpRequest
+    VERSION = "0.3.0"
+  end
+end

data/lib/em-http.rb

@@ -1,19 +1,19 @@
-#--
-# Copyright (C)2008 Ilya Grigorik
-# You can redistribute this under the terms of the Ruby license
-# See file LICENSE for details
-#++
-
-require 'eventmachine'
-
-require File.dirname(__FILE__) + '/http11_client'
-require File.dirname(__FILE__) + '/em_buffer'
-
-require File.dirname(__FILE__) + '/em-http/core_ext/hash'
-require File.dirname(__FILE__) + '/em-http/core_ext/bytesize'
-
-require File.dirname(__FILE__) + '/em-http/client'
-require File.dirname(__FILE__) + '/em-http/multi'
-require File.dirname(__FILE__) + '/em-http/request'
-require File.dirname(__FILE__) + '/em-http/decoders'
-require File.dirname(__FILE__) + '/em-http/http_options'
+require 'eventmachine'
+require 'escape_utils'
+require 'addressable/uri'
+
+require 'base64'
+require 'socket'
+
+require 'http11_client'
+require 'em_buffer'
+
+require 'em-http/core_ext/bytesize'
+require 'em-http/http_header'
+require 'em-http/http_encoding'
+require 'em-http/http_options'
+require 'em-http/client'
+require 'em-http/multi'
+require 'em-http/request'
+require 'em-http/decoders'
+require 'em-http/mock'

data/spec/encoding_spec.rb

@@ -0,0 +1,40 @@
+require 'helper'
+
+describe EventMachine::HttpEncoding do
+  include EventMachine::HttpEncoding
+
+  it "should transform a basic hash into HTTP POST Params" do
+    form_encode_body({:a => "alpha", :b => "beta"}).should == "a=alpha&b=beta"
+  end
+
+  it "should transform a more complex hash into HTTP POST Params" do
+    form_encode_body({:a => "a", :b => ["c", "d", "e"]}).should == "a=a&b[0]=c&b[1]=d&b[2]=e"
+  end
+
+  it "should transform a very complex hash into HTTP POST Params" do
+    params = form_encode_body({:a => "a", :b => [{:c => "c", :d => "d"}, {:e => "e", :f => "f"}]})
+    params.should == "a=a&b[0][c]=c&b[0][d]=d&b[1][e]=e&b[1][f]=f"
+  end
+
+  it "should escape values" do
+    params = form_encode_body({:stuff => 'string&string'})
+    params.should == "stuff=string%26string"
+  end
+
+  it "should escape keys" do
+    params = form_encode_body({'bad&str'=> {'key&key' => [:a, :b]}})
+    params.should == 'bad%26str[key%26key][0]=a&bad%26str[key%26key][1]=b'
+  end
+
+  it "should escape keys and values" do
+    params = form_encode_body({'bad&str'=> {'key&key' => ['bad+&stuff', '[test]']}})
+    params.should == "bad%26str[key%26key][0]=bad%2B%26stuff&bad%26str[key%26key][1]=%5Btest%5D"
+  end
+
+  it "should be fast on long string escapes" do
+    s = Time.now
+    5000.times { |n| form_encode_body({:a => "{a:'b', d:'f', g:['a','b']}"*50}) }
+    (Time.now - s).should satisfy { |t| t < 1.5 }
+  end
+
+end