em-http-request 0.2.10 → 0.2.11

data/ext/http11_client/http11_parser.h

@@ -1,48 +1,48 @@
-/**
- * Copyright (c) 2005 Zed A. Shaw
- * You can redistribute it and/or modify it under the same terms as Ruby.
- */
-
-#ifndef http11_parser_h
-#define http11_parser_h
-
-#include <sys/types.h>
-
-#if defined(_WIN32)
-#include <stddef.h>
-#endif
-
-typedef void (*element_cb)(void *data, const char *at, size_t length);
-typedef void (*field_cb)(void *data, const char *field, size_t flen, const char *value, size_t vlen);
-
-typedef struct httpclient_parser { 
-  int cs;
-  size_t body_start;
-  int content_len;
-  size_t nread;
-  size_t mark;
-  size_t field_start;
-  size_t field_len;
-
-  void *data;
-
-  field_cb http_field;
-  element_cb reason_phrase;
-  element_cb status_code;
-  element_cb chunk_size;
-  element_cb http_version;
-  element_cb header_done;
-  element_cb last_chunk;
-  
-  
-} httpclient_parser;
-
-int httpclient_parser_init(httpclient_parser *parser);
-int httpclient_parser_finish(httpclient_parser *parser);
-size_t httpclient_parser_execute(httpclient_parser *parser, const char *data, size_t len, size_t off);
-int httpclient_parser_has_error(httpclient_parser *parser);
-int httpclient_parser_is_finished(httpclient_parser *parser);
-
-#define httpclient_parser_nread(parser) (parser)->nread 
-
-#endif
+/**
+ * Copyright (c) 2005 Zed A. Shaw
+ * You can redistribute it and/or modify it under the same terms as Ruby.
+ */
+
+#ifndef http11_parser_h
+#define http11_parser_h
+
+#include <sys/types.h>
+
+#if defined(_WIN32)
+#include <stddef.h>
+#endif
+
+typedef void (*element_cb)(void *data, const char *at, size_t length);
+typedef void (*field_cb)(void *data, const char *field, size_t flen, const char *value, size_t vlen);
+
+typedef struct httpclient_parser { 
+  int cs;
+  size_t body_start;
+  int content_len;
+  size_t nread;
+  size_t mark;
+  size_t field_start;
+  size_t field_len;
+
+  void *data;
+
+  field_cb http_field;
+  element_cb reason_phrase;
+  element_cb status_code;
+  element_cb chunk_size;
+  element_cb http_version;
+  element_cb header_done;
+  element_cb last_chunk;
+  
+  
+} httpclient_parser;
+
+int httpclient_parser_init(httpclient_parser *parser);
+int httpclient_parser_finish(httpclient_parser *parser);
+size_t httpclient_parser_execute(httpclient_parser *parser, const char *data, size_t len, size_t off);
+int httpclient_parser_has_error(httpclient_parser *parser);
+int httpclient_parser_is_finished(httpclient_parser *parser);
+
+#define httpclient_parser_nread(parser) (parser)->nread 
+
+#endif

data/ext/http11_client/http11_parser.rl

@@ -1,170 +1,170 @@
-/**
- * Copyright (c) 2005 Zed A. Shaw
- * You can redistribute it and/or modify it under the same terms as Ruby.
- */
-
-#include "http11_parser.h"
-#include <stdio.h>
-#include <assert.h>
-#include <stdlib.h>
-#include <ctype.h>
-#include <string.h>
-
-#define LEN(AT, FPC) (FPC - buffer - parser->AT)
-#define MARK(M,FPC) (parser->M = (FPC) - buffer)
-#define PTR_TO(F) (buffer + parser->F)
-#define L(M) fprintf(stderr, "" # M "\n");
-
-
-/** machine **/
-%%{
-  machine httpclient_parser;
-
-  action mark {MARK(mark, fpc); }
-
-  action start_field { MARK(field_start, fpc); }
-
-  action write_field { 
-    parser->field_len = LEN(field_start, fpc);
-  }
-
-  action start_value { MARK(mark, fpc); }
-
-  action write_value { 
-    parser->http_field(parser->data, PTR_TO(field_start), parser->field_len, PTR_TO(mark), LEN(mark, fpc));
-  }
-
-  action reason_phrase { 
-    parser->reason_phrase(parser->data, PTR_TO(mark), LEN(mark, fpc));
-  }
-
-  action status_code { 
-    parser->status_code(parser->data, PTR_TO(mark), LEN(mark, fpc));
-  }
-
-  action http_version {	
-    parser->http_version(parser->data, PTR_TO(mark), LEN(mark, fpc));
-  }
-
-  action chunk_size {
-    parser->chunk_size(parser->data, PTR_TO(mark), LEN(mark, fpc));
-  }
-
-  action last_chunk {
-    parser->last_chunk(parser->data, NULL, 0);
-  }
-
-  action done { 
-    parser->body_start = fpc - buffer + 1; 
-    if(parser->header_done != NULL)
-      parser->header_done(parser->data, fpc + 1, pe - fpc - 1);
-    fbreak;
-  }
-
-# line endings
-  CRLF = ("\r\n" | "\n");
-
-# character types
-  CTL = (cntrl | 127);
-  tspecials = ("(" | ")" | "<" | ">" | "@" | "," | ";" | ":" | "\\" | "\"" | "/" | "[" | "]" | "?" | "=" | "{" | "}" | " " | "\t");
-
-# elements
-  token = (ascii -- (CTL | tspecials));
-
-  Reason_Phrase = (any -- CRLF)* >mark %reason_phrase;
-  Status_Code = digit{3} >mark %status_code;
-  http_number = (digit+ "." digit+) ;
-  HTTP_Version = ("HTTP/" http_number) >mark %http_version ;
-  Status_Line = HTTP_Version " " Status_Code " "? Reason_Phrase :> CRLF;
-
-  field_name = token+ >start_field %write_field;
-  field_value = any* >start_value %write_value;
-  message_header = field_name ":" " "* field_value :> CRLF;
-
-  Response = 	Status_Line (message_header)* (CRLF @done);
-
-  chunk_ext_val = token+;
-  chunk_ext_name = token+;
-  chunk_extension = (";" chunk_ext_name >start_field %write_field %start_value ("=" chunk_ext_val >start_value)? %write_value )*;
-  last_chunk = "0"? chunk_extension :> (CRLF @last_chunk @done);
-  chunk_size = xdigit+;
-  chunk = chunk_size >mark %chunk_size chunk_extension space* :> (CRLF @done);
-  Chunked_Header = (chunk | last_chunk);
-
-  main := Response | Chunked_Header;
-}%%
-
-/** Data **/
-%% write data;
-
-int httpclient_parser_init(httpclient_parser *parser)  {
-  int cs = 0;
-  %% write init;
-  parser->cs = cs;
-  parser->body_start = 0;
-  parser->content_len = 0;
-  parser->mark = 0;
-  parser->nread = 0;
-  parser->field_len = 0;
-  parser->field_start = 0;    
-
-  return(1);
-}
-
-
-/** exec **/
-size_t httpclient_parser_execute(httpclient_parser *parser, const char *buffer, size_t len, size_t off)  {
-  const char *p, *pe;
-  int cs = parser->cs;
-
-  assert(off <= len && "offset past end of buffer");
-
-  p = buffer+off;
-  pe = buffer+len;
-
-  assert(*pe == '\0' && "pointer does not end on NUL");
-  assert(pe - p == len - off && "pointers aren't same distance");
-
-
-  %% write exec;
-
-  parser->cs = cs;
-  parser->nread += p - (buffer + off);
-
-  assert(p <= pe && "buffer overflow after parsing execute");
-  assert(parser->nread <= len && "nread longer than length");
-  assert(parser->body_start <= len && "body starts after buffer end");
-  assert(parser->mark < len && "mark is after buffer end");
-  assert(parser->field_len <= len && "field has length longer than whole buffer");
-  assert(parser->field_start < len && "field starts after buffer end");
-
-  if(parser->body_start) {
-    /* final \r\n combo encountered so stop right here */
-    parser->nread = parser->body_start;
-  }
-
-  return(parser->nread);
-}
-
-int httpclient_parser_finish(httpclient_parser *parser)
-{
-  int cs = parser->cs;
-
-  parser->cs = cs;
-
-  if (httpclient_parser_has_error(parser) ) {
-    return -1;
-  } else if (httpclient_parser_is_finished(parser) ) {
-    return 1;
-  } else {
-    return 0;
-  }
-}
-
-int httpclient_parser_has_error(httpclient_parser *parser) {
-  return parser->cs == httpclient_parser_error;
-}
-
-int httpclient_parser_is_finished(httpclient_parser *parser) {
-  return parser->cs == httpclient_parser_first_final;
-}
+/**
+ * Copyright (c) 2005 Zed A. Shaw
+ * You can redistribute it and/or modify it under the same terms as Ruby.
+ */
+
+#include "http11_parser.h"
+#include <stdio.h>
+#include <assert.h>
+#include <stdlib.h>
+#include <ctype.h>
+#include <string.h>
+
+#define LEN(AT, FPC) (FPC - buffer - parser->AT)
+#define MARK(M,FPC) (parser->M = (FPC) - buffer)
+#define PTR_TO(F) (buffer + parser->F)
+#define L(M) fprintf(stderr, "" # M "\n");
+
+
+/** machine **/
+%%{
+  machine httpclient_parser;
+
+  action mark {MARK(mark, fpc); }
+
+  action start_field { MARK(field_start, fpc); }
+
+  action write_field { 
+    parser->field_len = LEN(field_start, fpc);
+  }
+
+  action start_value { MARK(mark, fpc); }
+
+  action write_value { 
+    parser->http_field(parser->data, PTR_TO(field_start), parser->field_len, PTR_TO(mark), LEN(mark, fpc));
+  }
+
+  action reason_phrase { 
+    parser->reason_phrase(parser->data, PTR_TO(mark), LEN(mark, fpc));
+  }
+
+  action status_code { 
+    parser->status_code(parser->data, PTR_TO(mark), LEN(mark, fpc));
+  }
+
+  action http_version {	
+    parser->http_version(parser->data, PTR_TO(mark), LEN(mark, fpc));
+  }
+
+  action chunk_size {
+    parser->chunk_size(parser->data, PTR_TO(mark), LEN(mark, fpc));
+  }
+
+  action last_chunk {
+    parser->last_chunk(parser->data, NULL, 0);
+  }
+
+  action done { 
+    parser->body_start = fpc - buffer + 1; 
+    if(parser->header_done != NULL)
+      parser->header_done(parser->data, fpc + 1, pe - fpc - 1);
+    fbreak;
+  }
+
+# line endings
+  CRLF = ("\r\n" | "\n");
+
+# character types
+  CTL = (cntrl | 127);
+  tspecials = ("(" | ")" | "<" | ">" | "@" | "," | ";" | ":" | "\\" | "\"" | "/" | "[" | "]" | "?" | "=" | "{" | "}" | " " | "\t");
+
+# elements
+  token = (ascii -- (CTL | tspecials));
+
+  Reason_Phrase = (any -- CRLF)* >mark %reason_phrase;
+  Status_Code = digit{3} >mark %status_code;
+  http_number = (digit+ "." digit+) ;
+  HTTP_Version = ("HTTP/" http_number) >mark %http_version ;
+  Status_Line = HTTP_Version " " Status_Code " "? Reason_Phrase :> CRLF;
+
+  field_name = token+ >start_field %write_field;
+  field_value = any* >start_value %write_value;
+  message_header = field_name ":" " "* field_value :> CRLF;
+
+  Response = 	Status_Line (message_header)* (CRLF @done);
+
+  chunk_ext_val = token+;
+  chunk_ext_name = token+;
+  chunk_extension = (";" chunk_ext_name >start_field %write_field %start_value ("=" chunk_ext_val >start_value)? %write_value )*;
+  last_chunk = "0"? chunk_extension :> (CRLF @last_chunk @done);
+  chunk_size = xdigit+;
+  chunk = chunk_size >mark %chunk_size chunk_extension space* :> (CRLF @done);
+  Chunked_Header = (chunk | last_chunk);
+
+  main := Response | Chunked_Header;
+}%%
+
+/** Data **/
+%% write data;
+
+int httpclient_parser_init(httpclient_parser *parser)  {
+  int cs = 0;
+  %% write init;
+  parser->cs = cs;
+  parser->body_start = 0;
+  parser->content_len = 0;
+  parser->mark = 0;
+  parser->nread = 0;
+  parser->field_len = 0;
+  parser->field_start = 0;    
+
+  return(1);
+}
+
+
+/** exec **/
+size_t httpclient_parser_execute(httpclient_parser *parser, const char *buffer, size_t len, size_t off)  {
+  const char *p, *pe;
+  int cs = parser->cs;
+
+  assert(off <= len && "offset past end of buffer");
+
+  p = buffer+off;
+  pe = buffer+len;
+
+  assert(*pe == '\0' && "pointer does not end on NUL");
+  assert(pe - p == len - off && "pointers aren't same distance");
+
+
+  %% write exec;
+
+  parser->cs = cs;
+  parser->nread += p - (buffer + off);
+
+  assert(p <= pe && "buffer overflow after parsing execute");
+  assert(parser->nread <= len && "nread longer than length");
+  assert(parser->body_start <= len && "body starts after buffer end");
+  assert(parser->mark < len && "mark is after buffer end");
+  assert(parser->field_len <= len && "field has length longer than whole buffer");
+  assert(parser->field_start < len && "field starts after buffer end");
+
+  if(parser->body_start) {
+    /* final \r\n combo encountered so stop right here */
+    parser->nread = parser->body_start;
+  }
+
+  return(parser->nread);
+}
+
+int httpclient_parser_finish(httpclient_parser *parser)
+{
+  int cs = parser->cs;
+
+  parser->cs = cs;
+
+  if (httpclient_parser_has_error(parser) ) {
+    return -1;
+  } else if (httpclient_parser_is_finished(parser) ) {
+    return 1;
+  } else {
+    return 0;
+  }
+}
+
+int httpclient_parser_has_error(httpclient_parser *parser) {
+  return parser->cs == httpclient_parser_error;
+}
+
+int httpclient_parser_is_finished(httpclient_parser *parser) {
+  return parser->cs == httpclient_parser_first_final;
+}

data/lib/em-http-request.rb

@@ -1 +1 @@
-require 'em-http'
+require 'em-http'

data/lib/em-http.rb

@@ -16,4 +16,5 @@ require File.dirname(__FILE__) + '/em-http/client'
 require File.dirname(__FILE__) + '/em-http/multi'
 require File.dirname(__FILE__) + '/em-http/request'
 require File.dirname(__FILE__) + '/em-http/decoders'
-require File.dirname(__FILE__) + '/em-http/http_options'
+require File.dirname(__FILE__) + '/em-http/http_options'
+require File.dirname(__FILE__) + '/em-http/mock'

data/lib/em-http/client.rb

@@ -23,12 +23,11 @@ module EventMachine
 
     # E-Tag
     def etag
-      self["ETag"]
+      self[HttpClient::ETAG]
     end
 
     def last_modified
-      time = self["Last-Modified"]
-      Time.parse(time) if time
+      self[HttpClient::LAST_MODIFIED]
     end
 
     # HTTP response status as an integer
@@ -84,7 +83,7 @@ module EventMachine
     # Escapes a URI.
     def escape(s)
       s.to_s.gsub(/([^ a-zA-Z0-9_.-]+)/n) {
-        '%'+$1.unpack('H2'*$1.bytesize).join('%').upcase
+        '%'+$1.unpack('H2'*bytesize($1)).join('%').upcase
       }.tr(' ', '+')
     end
 
@@ -95,6 +94,16 @@ module EventMachine
       }
     end
 
+    if ''.respond_to?(:bytesize)
+      def bytesize(string)
+        string.bytesize
+      end
+    else
+      def bytesize(string)
+        string.size
+      end
+    end
+
     # Map all header keys to a downcased string version
     def munge_header_keys(head)
       head.inject({}) { |h, (k, v)| h[k.to_s.downcase] = v; h }
@@ -111,21 +120,27 @@ module EventMachine
       end
     end
 
-    def encode_request(method, path, query, uri_query)
-      HTTP_REQUEST_HEADER % [method.to_s.upcase, encode_query(path, query, uri_query)]
+    def encode_request(method, uri, query, proxy)
+      query = encode_query(uri, query)
+
+      # Non CONNECT proxies require that you provide the full request
+      # uri in request header, as opposed to a relative path.
+      query = uri.join(query) if proxy and not proxy[:use_connect]
+
+      HTTP_REQUEST_HEADER % [method.to_s.upcase, query]
     end
 
-    def encode_query(path, query, uri_query)
+    def encode_query(uri, query)
       encoded_query = if query.kind_of?(Hash)
         query.map { |k, v| encode_param(k, v) }.join('&')
       else
         query.to_s
       end
-      if !uri_query.to_s.empty?
-        encoded_query = [encoded_query, uri_query].reject {|part| part.empty?}.join("&")
+
+      if !uri.query.to_s.empty?
+        encoded_query = [encoded_query, uri.query].reject {|part| part.empty?}.join("&")
       end
-      return path if encoded_query.to_s.empty?
-      "#{path}?#{encoded_query}"
+      encoded_query.to_s.empty? ? uri.path : "#{uri.path}?#{encoded_query}"
     end
 
     # URL encodes query parameters:
@@ -183,10 +198,13 @@ module EventMachine
     TRANSFER_ENCODING="TRANSFER_ENCODING"
     CONTENT_ENCODING="CONTENT_ENCODING"
     CONTENT_LENGTH="CONTENT_LENGTH"
+    LAST_MODIFIED="LAST_MODIFIED"
     KEEP_ALIVE="CONNECTION"
     SET_COOKIE="SET_COOKIE"
     LOCATION="LOCATION"
     HOST="HOST"
+    ETAG="ETAG"
+
     CRLF="\r\n"
 
     attr_accessor :method, :options, :uri
@@ -210,10 +228,10 @@ module EventMachine
 
     # start HTTP request once we establish connection to host
     def connection_completed
-      # if connecting to proxy, then first negotiate the connection
-      # to intermediate server and wait for 200 response
-      if @options[:proxy] and @state == :response_header
-        @state = :response_proxy
+      # if we need to negotiate the proxy connection first, then
+      # issue a CONNECT query and wait for 200 response
+      if connect_proxy? and @state == :response_header
+        @state = :connect_proxy
         send_request_header
 
         # if connecting via proxy, then state will be :proxy_connected,
@@ -283,23 +301,32 @@ module EventMachine
     end
 
     def websocket?; @uri.scheme == 'ws'; end
+    def proxy?; !@options[:proxy].nil?; end
+    def connect_proxy?; proxy? && (@options[:proxy][:use_connect] == true); end
 
     def send_request_header
       query   = @options[:query]
       head    = @options[:head] ? munge_header_keys(@options[:head]) : {}
       file    = @options[:file]
+      proxy   = @options[:proxy]
       body    = normalize_body
-      request_header = nil
 
-      if @state == :response_proxy
-        proxy = @options[:proxy]
+      request_header = nil
 
-        # initialize headers to establish the HTTP tunnel
+      if proxy
+        # initialize headers for the http proxy
         head = proxy[:head] ? munge_header_keys(proxy[:head]) : {}
         head['proxy-authorization'] = proxy[:authorization] if proxy[:authorization]
-        request_header = HTTP_REQUEST_HEADER % ['CONNECT', "#{@uri.host}:#{@uri.port}"]
 
-      elsif websocket?
+        # if we need to negotiate the tunnel connection first, then
+        # issue a CONNECT query to the proxy first. This is an optional
+        # flag, by default we will provide full URIs to the proxy
+        if @state == :connect_proxy
+          request_header = HTTP_REQUEST_HEADER % ['CONNECT', "#{@uri.host}:#{@uri.port}"]
+        end
+      end
+
+      if websocket?
         head['upgrade'] = 'WebSocket'
         head['connection'] = 'Upgrade'
         head['origin'] = @options[:origin] || @uri.host
@@ -332,7 +359,7 @@ module EventMachine
       @last_effective_url = @uri
 
       # Build the request headers
-      request_header ||= encode_request(@method, @uri.path, query, @uri.query)
+      request_header ||= encode_request(@method, @uri, query, proxy)
       request_header << encode_headers(head)
       request_header << CRLF
       send_data request_header
@@ -388,6 +415,7 @@ module EventMachine
 
         @response_header = HttpResponseHeader.new
         @state = :response_header
+        @response = ''
         @data.clear
       else
         if @state == :finished || (@state == :body && @bytes_remaining.nil?)
@@ -405,7 +433,7 @@ module EventMachine
 
     def dispatch
       while case @state
-          when :response_proxy
+          when :connect_proxy
             parse_response_header
           when :response_header
             parse_response_header
@@ -457,7 +485,7 @@ module EventMachine
         return false
       end
 
-      if @state == :response_proxy
+      if @state == :connect_proxy
         # when a successfull tunnel is established, the proxy responds with a
         # 200 response code. from here, the tunnel is transparent.
         if @response_header.http_status.to_i == 200
@@ -475,9 +503,13 @@ module EventMachine
       if @response_header.location
         begin
           location = Addressable::URI.parse(@response_header.location)
+
           if location.relative?
             location = @uri.join(location)
             @response_header[LOCATION] = location.to_s
+          else
+            # if redirect is to an absolute url, check for correct URI structure
+            raise if location.host.nil?
           end
 
           # store last url on any sign of redirect
@@ -628,8 +660,8 @@ module EventMachine
       # slice the message out of the buffer and pass in
       # for processing, and buffer data otherwise
       buffer = @data.read
-      while msg = buffer.slice!(/\000([^\377]*)\377/)
-        msg.gsub!(/^\x00|\xff$/, '')
+      while msg = buffer.slice!(/\000([^\377]*)\377/n)
+        msg.gsub!(/\A\x00|\xff\z/n, '')
         @stream.call(msg)
       end