em-http-request 0.2.15 → 0.3.0

data/ext/http11_client/http11_parser.h

@@ -1,48 +1,48 @@
-/**
- * Copyright (c) 2005 Zed A. Shaw
- * You can redistribute it and/or modify it under the same terms as Ruby.
- */
-
-#ifndef http11_parser_h
-#define http11_parser_h
-
-#include <sys/types.h>
-
-#if defined(_WIN32)
-#include <stddef.h>
-#endif
-
-typedef void (*element_cb)(void *data, const char *at, size_t length);
-typedef void (*field_cb)(void *data, const char *field, size_t flen, const char *value, size_t vlen);
-
-typedef struct httpclient_parser { 
-  int cs;
-  size_t body_start;
-  int content_len;
-  size_t nread;
-  size_t mark;
-  size_t field_start;
-  size_t field_len;
-
-  void *data;
-
-  field_cb http_field;
-  element_cb reason_phrase;
-  element_cb status_code;
-  element_cb chunk_size;
-  element_cb http_version;
-  element_cb header_done;
-  element_cb last_chunk;
-  
-  
-} httpclient_parser;
-
-int httpclient_parser_init(httpclient_parser *parser);
-int httpclient_parser_finish(httpclient_parser *parser);
-size_t httpclient_parser_execute(httpclient_parser *parser, const char *data, size_t len, size_t off);
-int httpclient_parser_has_error(httpclient_parser *parser);
-int httpclient_parser_is_finished(httpclient_parser *parser);
-
-#define httpclient_parser_nread(parser) (parser)->nread 
-
-#endif
+/**
+ * Copyright (c) 2005 Zed A. Shaw
+ * You can redistribute it and/or modify it under the same terms as Ruby.
+ */
+
+#ifndef http11_parser_h
+#define http11_parser_h
+
+#include <sys/types.h>
+
+#if defined(_WIN32)
+#include <stddef.h>
+#endif
+
+typedef void (*element_cb)(void *data, const char *at, size_t length);
+typedef void (*field_cb)(void *data, const char *field, size_t flen, const char *value, size_t vlen);
+
+typedef struct httpclient_parser { 
+  int cs;
+  size_t body_start;
+  int content_len;
+  size_t nread;
+  size_t mark;
+  size_t field_start;
+  size_t field_len;
+
+  void *data;
+
+  field_cb http_field;
+  element_cb reason_phrase;
+  element_cb status_code;
+  element_cb chunk_size;
+  element_cb http_version;
+  element_cb header_done;
+  element_cb last_chunk;
+  
+  
+} httpclient_parser;
+
+int httpclient_parser_init(httpclient_parser *parser);
+int httpclient_parser_finish(httpclient_parser *parser);
+size_t httpclient_parser_execute(httpclient_parser *parser, const char *data, size_t len, size_t off);
+int httpclient_parser_has_error(httpclient_parser *parser);
+int httpclient_parser_is_finished(httpclient_parser *parser);
+
+#define httpclient_parser_nread(parser) (parser)->nread 
+
+#endif

data/lib/em-http.rb

@@ -1,20 +1,19 @@
-#--
-# Copyright (C)2008 Ilya Grigorik
-# You can redistribute this under the terms of the Ruby license
-# See file LICENSE for details
-#++
-
 require 'eventmachine'
-require 'socket'
+require 'escape_utils'
+require 'addressable/uri'
 
-require File.dirname(__FILE__) + '/http11_client'
-require File.dirname(__FILE__) + '/em_buffer'
+require 'base64'
+require 'socket'
 
-require File.dirname(__FILE__) + '/em-http/core_ext/bytesize'
+require 'http11_client'
+require 'em_buffer'
 
-require File.dirname(__FILE__) + '/em-http/client'
-require File.dirname(__FILE__) + '/em-http/multi'
-require File.dirname(__FILE__) + '/em-http/request'
-require File.dirname(__FILE__) + '/em-http/decoders'
-require File.dirname(__FILE__) + '/em-http/http_options'
-require File.dirname(__FILE__) + '/em-http/mock'
+require 'em-http/core_ext/bytesize'
+require 'em-http/http_header'
+require 'em-http/http_encoding'
+require 'em-http/http_options'
+require 'em-http/client'
+require 'em-http/multi'
+require 'em-http/request'
+require 'em-http/decoders'
+require 'em-http/mock'

data/lib/em-http/client.rb

@@ -9,223 +9,14 @@
 
 module EventMachine
 
-  # A simple hash is returned for each request made by HttpClient with the
-  # headers that were given by the server for that request.
-  class HttpResponseHeader < Hash
-    # The reason returned in the http response ("OK","File not found",etc.)
-    attr_accessor :http_reason
-
-    # The HTTP version returned.
-    attr_accessor :http_version
-
-    # The status code (as a string!)
-    attr_accessor :http_status
-
-    # E-Tag
-    def etag
-      self[HttpClient::ETAG]
-    end
-
-    def last_modified
-      self[HttpClient::LAST_MODIFIED]
-    end
-
-    # HTTP response status as an integer
-    def status
-      Integer(http_status) rescue 0
-    end
-
-    # Length of content as an integer, or nil if chunked/unspecified
-    def content_length
-      @content_length ||= ((s = self[HttpClient::CONTENT_LENGTH]) &&
-                           (s =~ /^(\d+)$/)) ? $1.to_i : nil
-    end
-
-    # Cookie header from the server
-    def cookie
-      self[HttpClient::SET_COOKIE]
-    end
-
-    # Is the transfer encoding chunked?
-    def chunked_encoding?
-      /chunked/i === self[HttpClient::TRANSFER_ENCODING]
-    end
-
-    def keep_alive?
-      /keep-alive/i === self[HttpClient::KEEP_ALIVE]
-    end
-
-    def compressed?
-      /gzip|compressed|deflate/i === self[HttpClient::CONTENT_ENCODING]
-    end
-
-    def location
-      self[HttpClient::LOCATION]
-    end
-  end
-
-  class HttpChunkHeader < Hash
-    # When parsing chunked encodings this is set
-    attr_accessor :http_chunk_size
-
-    def initialize
-      super
-      @http_chunk_size = '0'
-    end
-
-    # Size of the chunk as an integer
-    def chunk_size
-      @http_chunk_size.to_i(base=16)
-    end
-  end
-
-  # Methods for building HTTP requests
-  module HttpEncoding
-    HTTP_REQUEST_HEADER="%s %s HTTP/1.1\r\n"
-    FIELD_ENCODING = "%s: %s\r\n"
-
-    # Escapes a URI.
-    def escape(s)
-      s.to_s.gsub(/([^a-zA-Z0-9_.-]+)/n) {
-        '%'+$1.unpack('H2'*bytesize($1)).join('%').upcase
-      }
-    end
-
-    # Unescapes a URI escaped string.
-    def unescape(s)
-      s.tr('+', ' ').gsub(/((?:%[0-9a-fA-F]{2})+)/n){
-        [$1.delete('%')].pack('H*')
-      }
-    end
-
-    if ''.respond_to?(:bytesize)
-      def bytesize(string)
-        string.bytesize
-      end
-    else
-      def bytesize(string)
-        string.size
-      end
-    end
-
-    # Map all header keys to a downcased string version
-    def munge_header_keys(head)
-      head.inject({}) { |h, (k, v)| h[k.to_s.downcase] = v; h }
-    end
-
-    # HTTP is kind of retarded that you have to specify a Host header, but if
-    # you include port 80 then further redirects will tack on the :80 which is
-    # annoying.
-    def encode_host
-      if @uri.port == 80 || @uri.port == 443
-        return @uri.host
-      else
-        @uri.host + ":#{@uri.port}"
-      end
-    end
-
-    def encode_request(method, uri, query, proxy)
-      query = encode_query(uri, query)
-
-      # Non CONNECT proxies require that you provide the full request
-      # uri in request header, as opposed to a relative path.
-      query = uri.join(query) if proxy && proxy[:type] != :socks && !proxy[:use_connect]
-
-      HTTP_REQUEST_HEADER % [method.to_s.upcase, query]
-    end
-
-    def encode_query(uri, query)
-      encoded_query = if query.kind_of?(Hash)
-        query.map { |k, v| encode_param(k, v) }.join('&')
-      else
-        query.to_s
-      end
-
-      if !uri.query.to_s.empty?
-        encoded_query = [encoded_query, uri.query].reject {|part| part.empty?}.join("&")
-      end
-      encoded_query.to_s.empty? ? uri.path : "#{uri.path}?#{encoded_query}"
-    end
-
-    # URL encodes query parameters:
-    # single k=v, or a URL encoded array, if v is an array of values
-    def encode_param(k, v)
-      if v.is_a?(Array)
-        v.map { |e| escape(k) + "[]=" + escape(e) }.join("&")
-      else
-        escape(k) + "=" + escape(v)
-      end
-    end
-
-    def form_encode_body(obj)
-      pairs = []
-      recursive = Proc.new do |h, prefix|
-        h.each do |k,v|
-          key = prefix == '' ? escape(k) : "#{prefix}[#{escape(k)}]"
-
-          if v.is_a? Array
-            nh = Hash.new
-            v.size.times { |t| nh[t] = v[t] }
-            recursive.call(nh, key)
-
-          elsif v.is_a? Hash
-            recursive.call(v, key)
-          else
-            pairs << "#{key}=#{escape(v)}"
-          end
-        end
-      end
-
-      recursive.call(obj, '')
-      return pairs.join('&')
-    end
-
-    # Encode a field in an HTTP header
-    def encode_field(k, v)
-      FIELD_ENCODING % [k, v]
-    end
-
-    # Encode basic auth in an HTTP header
-    # In: Array ([user, pass]) - for basic auth
-    #     String - custom auth string (OAuth, etc)
-    def encode_auth(k,v)
-      if v.is_a? Array
-        FIELD_ENCODING % [k, ["Basic", Base64.encode64(v.join(":")).chomp].join(" ")]
-      else
-        encode_field(k,v)
-      end
-    end
-
-    def encode_headers(head)
-      head.inject('') do |result, (key, value)|
-        # Munge keys from foo-bar-baz to Foo-Bar-Baz
-        key = key.split('-').map { |k| k.to_s.capitalize }.join('-')
-        result << case key
-          when 'Authorization', 'Proxy-authorization'
-            encode_auth(key, value)
-          else
-            encode_field(key, value)
-        end
-      end
-    end
-
-    def encode_cookie(cookie)
-      if cookie.is_a? Hash
-        cookie.inject('') { |result, (k, v)| result <<  encode_param(k, v) + ";" }
-      else
-        cookie
-      end
-    end
-  end
-
   class HttpClient < Connection
     include EventMachine::Deferrable
-    include HttpEncoding
+    include EventMachine::HttpEncoding
 
     TRANSFER_ENCODING="TRANSFER_ENCODING"
     CONTENT_ENCODING="CONTENT_ENCODING"
     CONTENT_LENGTH="CONTENT_LENGTH"
-    CONTENT_TYPE="CONTENT_TYPE".freeze
+    CONTENT_TYPE="CONTENT_TYPE"
     LAST_MODIFIED="LAST_MODIFIED"
     KEEP_ALIVE="CONNECTION"
     SET_COOKIE="SET_COOKIE"
@@ -236,7 +27,7 @@ module EventMachine
     CRLF="\r\n"
 
     attr_accessor :method, :options, :uri
-    attr_reader   :response, :response_header, :error, :redirects, :last_effective_url
+    attr_reader   :response, :response_header, :error, :redirects, :last_effective_url, :content_charset
 
     def post_init
       @parser = HttpClientParser.new
@@ -425,7 +216,12 @@ module EventMachine
 
         # Set content-type header if missing and body is a Ruby hash
         if not head['content-type'] and options[:body].is_a? Hash
-          head['content-type'] = "application/x-www-form-urlencoded"
+          head['content-type'] = 'application/x-www-form-urlencoded'
+        end
+
+        # Set connection close unless keepalive
+        unless options[:keepalive]
+          head['connection'] = 'close'
         end
       end
 
@@ -474,7 +270,7 @@ module EventMachine
     end
 
     def on_decoded_body_data(data)
-      data.force_encoding @content_charset  if @content_charset
+      data.force_encoding @content_charset if @content_charset
       if @stream
         @stream.call(data)
       else
@@ -652,7 +448,7 @@ module EventMachine
       end
 
       if ''.respond_to?(:force_encoding) && /;\s*charset=\s*(.+?)\s*(;|$)/.match(response_header[CONTENT_TYPE])
-        @content_charset = Encoding.find $1
+        @content_charset = Encoding.find($1.gsub(/^\"|\"$/, '')) rescue Encoding.default_external
       end
 
       true
@@ -852,13 +648,10 @@ module EventMachine
         on_request_complete
 
       else
-        if @data.empty?
-          @state = :finished
-          on_request_complete
-        else
-          @state = :invalid
-          on_error "garbage at end of body"
-        end
+
+        @data.clear
+        @state = :finished
+        on_request_complete
       end
 
       false

data/lib/em-http/http_encoding.rb

@@ -0,0 +1,135 @@
+module EventMachine
+  module HttpEncoding
+    HTTP_REQUEST_HEADER="%s %s HTTP/1.1\r\n"
+    FIELD_ENCODING = "%s: %s\r\n"
+
+    # Escapes a URI.
+    def escape(s)
+      EscapeUtils.escape_url(s.to_s)
+    end
+
+    # Unescapes a URI escaped string.
+    def unescape(s)
+      EscapeUtils.unescape_url(s.to_s)
+    end
+
+    if ''.respond_to?(:bytesize)
+      def bytesize(string)
+        string.bytesize
+      end
+    else
+      def bytesize(string)
+        string.size
+      end
+    end
+
+    # Map all header keys to a downcased string version
+    def munge_header_keys(head)
+      head.inject({}) { |h, (k, v)| h[k.to_s.downcase] = v; h }
+    end
+
+    # HTTP is kind of retarded that you have to specify a Host header, but if
+    # you include port 80 then further redirects will tack on the :80 which is
+    # annoying.
+    def encode_host
+      if @uri.port == 80 || @uri.port == 443
+        return @uri.host
+      else
+        @uri.host + ":#{@uri.port}"
+      end
+    end
+
+    def encode_request(method, uri, query, proxy)
+      query = encode_query(uri, query)
+
+      # Non CONNECT proxies require that you provide the full request
+      # uri in request header, as opposed to a relative path.
+      query = uri.join(query) if proxy && proxy[:type] != :socks && !proxy[:use_connect]
+
+      HTTP_REQUEST_HEADER % [method.to_s.upcase, query]
+    end
+
+    def encode_query(uri, query)
+      encoded_query = if query.kind_of?(Hash)
+        query.map { |k, v| encode_param(k, v) }.join('&')
+      else
+        query.to_s
+      end
+
+      if !uri.query.to_s.empty?
+        encoded_query = [encoded_query, uri.query].reject {|part| part.empty?}.join("&")
+      end
+      encoded_query.to_s.empty? ? uri.path : "#{uri.path}?#{encoded_query}"
+    end
+
+    # URL encodes query parameters:
+    # single k=v, or a URL encoded array, if v is an array of values
+    def encode_param(k, v)
+      if v.is_a?(Array)
+        v.map { |e| escape(k) + "[]=" + escape(e) }.join("&")
+      else
+        escape(k) + "=" + escape(v)
+      end
+    end
+
+    def form_encode_body(obj)
+      pairs = []
+      recursive = Proc.new do |h, prefix|
+        h.each do |k,v|
+          key = prefix == '' ? escape(k) : "#{prefix}[#{escape(k)}]"
+
+          if v.is_a? Array
+            nh = Hash.new
+            v.size.times { |t| nh[t] = v[t] }
+            recursive.call(nh, key)
+
+          elsif v.is_a? Hash
+            recursive.call(v, key)
+          else
+            pairs << "#{key}=#{escape(v)}"
+          end
+        end
+      end
+
+      recursive.call(obj, '')
+      return pairs.join('&')
+    end
+
+    # Encode a field in an HTTP header
+    def encode_field(k, v)
+      FIELD_ENCODING % [k, v]
+    end
+
+    # Encode basic auth in an HTTP header
+    # In: Array ([user, pass]) - for basic auth
+    #     String - custom auth string (OAuth, etc)
+    def encode_auth(k,v)
+      if v.is_a? Array
+        FIELD_ENCODING % [k, ["Basic", Base64.encode64(v.join(":")).chomp].join(" ")]
+      else
+        encode_field(k,v)
+      end
+    end
+
+    def encode_headers(head)
+      head.inject('') do |result, (key, value)|
+        # Munge keys from foo-bar-baz to Foo-Bar-Baz
+        key = key.split('-').map { |k| k.to_s.capitalize }.join('-')
+        result << case key
+          when 'Authorization', 'Proxy-Authorization'
+            encode_auth(key, value)
+          else
+            encode_field(key, value)
+        end
+      end
+    end
+
+    def encode_cookie(cookie)
+      if cookie.is_a? Hash
+        cookie.inject('') { |result, (k, v)| result <<  encode_param(k, v) + ";" }
+      else
+        cookie
+      end
+    end
+  end
+end