em-http-request 0.2.14 → 0.2.15

data/ext/http11_client/http11_parser.rl

@@ -1,170 +1,170 @@
-/**
- * Copyright (c) 2005 Zed A. Shaw
- * You can redistribute it and/or modify it under the same terms as Ruby.
- */
-
-#include "http11_parser.h"
-#include <stdio.h>
-#include <assert.h>
-#include <stdlib.h>
-#include <ctype.h>
-#include <string.h>
-
-#define LEN(AT, FPC) (FPC - buffer - parser->AT)
-#define MARK(M,FPC) (parser->M = (FPC) - buffer)
-#define PTR_TO(F) (buffer + parser->F)
-#define L(M) fprintf(stderr, "" # M "\n");
-
-
-/** machine **/
-%%{
-  machine httpclient_parser;
-
-  action mark {MARK(mark, fpc); }
-
-  action start_field { MARK(field_start, fpc); }
-
-  action write_field { 
-    parser->field_len = LEN(field_start, fpc);
-  }
-
-  action start_value { MARK(mark, fpc); }
-
-  action write_value { 
-    parser->http_field(parser->data, PTR_TO(field_start), parser->field_len, PTR_TO(mark), LEN(mark, fpc));
-  }
-
-  action reason_phrase { 
-    parser->reason_phrase(parser->data, PTR_TO(mark), LEN(mark, fpc));
-  }
-
-  action status_code { 
-    parser->status_code(parser->data, PTR_TO(mark), LEN(mark, fpc));
-  }
-
-  action http_version {	
-    parser->http_version(parser->data, PTR_TO(mark), LEN(mark, fpc));
-  }
-
-  action chunk_size {
-    parser->chunk_size(parser->data, PTR_TO(mark), LEN(mark, fpc));
-  }
-
-  action last_chunk {
-    parser->last_chunk(parser->data, NULL, 0);
-  }
-
-  action done { 
-    parser->body_start = fpc - buffer + 1; 
-    if(parser->header_done != NULL)
-      parser->header_done(parser->data, fpc + 1, pe - fpc - 1);
-    fbreak;
-  }
-
-# line endings
-  CRLF = ("\r\n" | "\n");
-
-# character types
-  CTL = (cntrl | 127);
-  tspecials = ("(" | ")" | "<" | ">" | "@" | "," | ";" | ":" | "\\" | "\"" | "/" | "[" | "]" | "?" | "=" | "{" | "}" | " " | "\t");
-
-# elements
-  token = (ascii -- (CTL | tspecials));
-
-  Reason_Phrase = (any -- CRLF)* >mark %reason_phrase;
-  Status_Code = digit{3} >mark %status_code;
-  http_number = (digit+ "." digit+) ;
-  HTTP_Version = ("HTTP/" http_number) >mark %http_version ;
-  Status_Line = HTTP_Version " " Status_Code " "? Reason_Phrase :> CRLF;
-
-  field_name = token+ >start_field %write_field;
-  field_value = any* >start_value %write_value;
-  message_header = field_name ":" " "* field_value :> CRLF;
-
-  Response = 	Status_Line (message_header)* (CRLF @done);
-
-  chunk_ext_val = token+;
-  chunk_ext_name = token+;
-  chunk_extension = (";" chunk_ext_name >start_field %write_field %start_value ("=" chunk_ext_val >start_value)? %write_value )*;
-  last_chunk = "0"? chunk_extension :> (CRLF @last_chunk @done);
-  chunk_size = xdigit+;
-  chunk = chunk_size >mark %chunk_size chunk_extension space* :> (CRLF @done);
-  Chunked_Header = (chunk | last_chunk);
-
-  main := Response | Chunked_Header;
-}%%
-
-/** Data **/
-%% write data;
-
-int httpclient_parser_init(httpclient_parser *parser)  {
-  int cs = 0;
-  %% write init;
-  parser->cs = cs;
-  parser->body_start = 0;
-  parser->content_len = 0;
-  parser->mark = 0;
-  parser->nread = 0;
-  parser->field_len = 0;
-  parser->field_start = 0;    
-
-  return(1);
-}
-
-
-/** exec **/
-size_t httpclient_parser_execute(httpclient_parser *parser, const char *buffer, size_t len, size_t off)  {
-  const char *p, *pe;
-  int cs = parser->cs;
-
-  assert(off <= len && "offset past end of buffer");
-
-  p = buffer+off;
-  pe = buffer+len;
-
-  assert(*pe == '\0' && "pointer does not end on NUL");
-  assert(pe - p == len - off && "pointers aren't same distance");
-
-
-  %% write exec;
-
-  parser->cs = cs;
-  parser->nread += p - (buffer + off);
-
-  assert(p <= pe && "buffer overflow after parsing execute");
-  assert(parser->nread <= len && "nread longer than length");
-  assert(parser->body_start <= len && "body starts after buffer end");
-  assert(parser->mark < len && "mark is after buffer end");
-  assert(parser->field_len <= len && "field has length longer than whole buffer");
-  assert(parser->field_start < len && "field starts after buffer end");
-
-  if(parser->body_start) {
-    /* final \r\n combo encountered so stop right here */
-    parser->nread = parser->body_start;
-  }
-
-  return(parser->nread);
-}
-
-int httpclient_parser_finish(httpclient_parser *parser)
-{
-  int cs = parser->cs;
-
-  parser->cs = cs;
-
-  if (httpclient_parser_has_error(parser) ) {
-    return -1;
-  } else if (httpclient_parser_is_finished(parser) ) {
-    return 1;
-  } else {
-    return 0;
-  }
-}
-
-int httpclient_parser_has_error(httpclient_parser *parser) {
-  return parser->cs == httpclient_parser_error;
-}
-
-int httpclient_parser_is_finished(httpclient_parser *parser) {
-  return parser->cs == httpclient_parser_first_final;
-}
+/**
+ * Copyright (c) 2005 Zed A. Shaw
+ * You can redistribute it and/or modify it under the same terms as Ruby.
+ */
+
+#include "http11_parser.h"
+#include <stdio.h>
+#include <assert.h>
+#include <stdlib.h>
+#include <ctype.h>
+#include <string.h>
+
+#define LEN(AT, FPC) (FPC - buffer - parser->AT)
+#define MARK(M,FPC) (parser->M = (FPC) - buffer)
+#define PTR_TO(F) (buffer + parser->F)
+#define L(M) fprintf(stderr, "" # M "\n");
+
+
+/** machine **/
+%%{
+  machine httpclient_parser;
+
+  action mark {MARK(mark, fpc); }
+
+  action start_field { MARK(field_start, fpc); }
+
+  action write_field { 
+    parser->field_len = LEN(field_start, fpc);
+  }
+
+  action start_value { MARK(mark, fpc); }
+
+  action write_value { 
+    parser->http_field(parser->data, PTR_TO(field_start), parser->field_len, PTR_TO(mark), LEN(mark, fpc));
+  }
+
+  action reason_phrase { 
+    parser->reason_phrase(parser->data, PTR_TO(mark), LEN(mark, fpc));
+  }
+
+  action status_code { 
+    parser->status_code(parser->data, PTR_TO(mark), LEN(mark, fpc));
+  }
+
+  action http_version {	
+    parser->http_version(parser->data, PTR_TO(mark), LEN(mark, fpc));
+  }
+
+  action chunk_size {
+    parser->chunk_size(parser->data, PTR_TO(mark), LEN(mark, fpc));
+  }
+
+  action last_chunk {
+    parser->last_chunk(parser->data, NULL, 0);
+  }
+
+  action done { 
+    parser->body_start = fpc - buffer + 1; 
+    if(parser->header_done != NULL)
+      parser->header_done(parser->data, fpc + 1, pe - fpc - 1);
+    fbreak;
+  }
+
+# line endings
+  CRLF = ("\r\n" | "\n");
+
+# character types
+  CTL = (cntrl | 127);
+  tspecials = ("(" | ")" | "<" | ">" | "@" | "," | ";" | ":" | "\\" | "\"" | "/" | "[" | "]" | "?" | "=" | "{" | "}" | " " | "\t");
+
+# elements
+  token = (ascii -- (CTL | tspecials));
+
+  Reason_Phrase = (any -- CRLF)* >mark %reason_phrase;
+  Status_Code = digit{3} >mark %status_code;
+  http_number = (digit+ "." digit+) ;
+  HTTP_Version = ("HTTP/" http_number) >mark %http_version ;
+  Status_Line = HTTP_Version " " Status_Code " "? Reason_Phrase :> CRLF;
+
+  field_name = token+ >start_field %write_field;
+  field_value = any* >start_value %write_value;
+  message_header = field_name ":" " "* field_value :> CRLF;
+
+  Response = 	Status_Line (message_header)* (CRLF @done);
+
+  chunk_ext_val = token+;
+  chunk_ext_name = token+;
+  chunk_extension = (";" chunk_ext_name >start_field %write_field %start_value ("=" chunk_ext_val >start_value)? %write_value )*;
+  last_chunk = "0"? chunk_extension :> (CRLF @last_chunk @done);
+  chunk_size = xdigit+;
+  chunk = chunk_size >mark %chunk_size chunk_extension space* :> (CRLF @done);
+  Chunked_Header = (chunk | last_chunk);
+
+  main := Response | Chunked_Header;
+}%%
+
+/** Data **/
+%% write data;
+
+int httpclient_parser_init(httpclient_parser *parser)  {
+  int cs = 0;
+  %% write init;
+  parser->cs = cs;
+  parser->body_start = 0;
+  parser->content_len = 0;
+  parser->mark = 0;
+  parser->nread = 0;
+  parser->field_len = 0;
+  parser->field_start = 0;    
+
+  return(1);
+}
+
+
+/** exec **/
+size_t httpclient_parser_execute(httpclient_parser *parser, const char *buffer, size_t len, size_t off)  {
+  const char *p, *pe;
+  int cs = parser->cs;
+
+  assert(off <= len && "offset past end of buffer");
+
+  p = buffer+off;
+  pe = buffer+len;
+
+  assert(*pe == '\0' && "pointer does not end on NUL");
+  assert(pe - p == len - off && "pointers aren't same distance");
+
+
+  %% write exec;
+
+  parser->cs = cs;
+  parser->nread += p - (buffer + off);
+
+  assert(p <= pe && "buffer overflow after parsing execute");
+  assert(parser->nread <= len && "nread longer than length");
+  assert(parser->body_start <= len && "body starts after buffer end");
+  assert(parser->mark < len && "mark is after buffer end");
+  assert(parser->field_len <= len && "field has length longer than whole buffer");
+  assert(parser->field_start < len && "field starts after buffer end");
+
+  if(parser->body_start) {
+    /* final \r\n combo encountered so stop right here */
+    parser->nread = parser->body_start;
+  }
+
+  return(parser->nread);
+}
+
+int httpclient_parser_finish(httpclient_parser *parser)
+{
+  int cs = parser->cs;
+
+  parser->cs = cs;
+
+  if (httpclient_parser_has_error(parser) ) {
+    return -1;
+  } else if (httpclient_parser_is_finished(parser) ) {
+    return 1;
+  } else {
+    return 0;
+  }
+}
+
+int httpclient_parser_has_error(httpclient_parser *parser) {
+  return parser->cs == httpclient_parser_error;
+}
+
+int httpclient_parser_is_finished(httpclient_parser *parser) {
+  return parser->cs == httpclient_parser_first_final;
+}

data/lib/em-http/client.rb

@@ -68,10 +68,14 @@ module EventMachine
     # When parsing chunked encodings this is set
     attr_accessor :http_chunk_size
 
+    def initialize
+      super
+      @http_chunk_size = '0'
+    end
+
     # Size of the chunk as an integer
     def chunk_size
-      return @chunk_size unless @chunk_size.nil?
-      @chunk_size = @http_chunk_size ? @http_chunk_size.to_i(base=16) : 0
+      @http_chunk_size.to_i(base=16)
     end
   end
 
@@ -221,6 +225,7 @@ module EventMachine
     TRANSFER_ENCODING="TRANSFER_ENCODING"
     CONTENT_ENCODING="CONTENT_ENCODING"
     CONTENT_LENGTH="CONTENT_LENGTH"
+    CONTENT_TYPE="CONTENT_TYPE".freeze
     LAST_MODIFIED="LAST_MODIFIED"
     KEEP_ALIVE="CONNECTION"
     SET_COOKIE="SET_COOKIE"
@@ -242,8 +247,10 @@ module EventMachine
       @redirects = 0
       @response = ''
       @error = ''
+      @headers = nil
       @last_effective_url = nil
       @content_decoder = nil
+      @content_charset = nil
       @stream = nil
       @disconnect = nil
       @state = :response_header
@@ -467,6 +474,7 @@ module EventMachine
     end
 
     def on_decoded_body_data(data)
+      data.force_encoding @content_charset  if @content_charset
       if @stream
         @stream.call(data)
       else
@@ -474,8 +482,12 @@ module EventMachine
       end
     end
 
+    def finished?
+      @state == :finished || (@state == :body && @bytes_remaining.nil?)
+    end
+
     def unbind
-      if (@state == :finished) && (@last_effective_url != @uri) && (@redirects < @options[:redirects])
+      if finished? && (@last_effective_url != @uri) && (@redirects < @options[:redirects])
         begin
           # update uri to redirect location if we're allowed to traverse deeper
           @uri = @last_effective_url
@@ -495,7 +507,7 @@ module EventMachine
           on_error(e.message, true)
         end
       else
-        if @state == :finished || (@state == :body && @bytes_remaining.nil?)
+        if finished?
           succeed(self)
         else
           @disconnect.call(self) if @state == :websocket and @disconnect
@@ -639,6 +651,10 @@ module EventMachine
         end
       end
 
+      if ''.respond_to?(:force_encoding) && /;\s*charset=\s*(.+?)\s*(;|$)/.match(response_header[CONTENT_TYPE])
+        @content_charset = Encoding.find $1
+      end
+
       true
     end
 

data/lib/em-http/multi.rb

@@ -1,55 +1,55 @@
-module EventMachine
-
-  # EventMachine based Multi request client, based on a streaming HTTPRequest class,
-  # which allows you to open multiple parallel connections and return only when all
-  # of them finish. (i.e. ideal for parallelizing workloads)
-  #
-  # == Example
-  #
-  #  EventMachine.run {
-  #
-  #   multi = EventMachine::MultiRequest.new
-  #
-  #   # add multiple requests to the multi-handler
-  #   multi.add(EventMachine::HttpRequest.new('http://www.google.com/').get)
-  #   multi.add(EventMachine::HttpRequest.new('http://www.yahoo.com/').get)
-  #
-  #    multi.callback  {
-  #       p multi.responses[:succeeded]
-  #       p multi.responses[:failed]
-  #
-  #       EventMachine.stop
-  #    }
-  #  }
-  #
-
-  class MultiRequest
-    include EventMachine::Deferrable
-
-    attr_reader :requests, :responses
-
-    def initialize(conns=[], &block)
-      @requests  = []
-      @responses = {:succeeded => [], :failed => []}
-
-      conns.each {|conn| add(conn)}
-      callback(&block) if block_given?
-    end
-
-    def add(conn)
-      @requests.push(conn)
-
-      conn.callback { @responses[:succeeded].push(conn); check_progress }
-      conn.errback  { @responses[:failed].push(conn); check_progress }
-    end
-
-    protected
-
-    # invoke callback if all requests have completed
-    def check_progress
-      succeed(self) if (@responses[:succeeded].size +
-                        @responses[:failed].size) == @requests.size
-    end
-
-  end
-end
+module EventMachine
+
+  # EventMachine based Multi request client, based on a streaming HTTPRequest class,
+  # which allows you to open multiple parallel connections and return only when all
+  # of them finish. (i.e. ideal for parallelizing workloads)
+  #
+  # == Example
+  #
+  #  EventMachine.run {
+  #
+  #   multi = EventMachine::MultiRequest.new
+  #
+  #   # add multiple requests to the multi-handler
+  #   multi.add(EventMachine::HttpRequest.new('http://www.google.com/').get)
+  #   multi.add(EventMachine::HttpRequest.new('http://www.yahoo.com/').get)
+  #
+  #    multi.callback  {
+  #       p multi.responses[:succeeded]
+  #       p multi.responses[:failed]
+  #
+  #       EventMachine.stop
+  #    }
+  #  }
+  #
+
+  class MultiRequest
+    include EventMachine::Deferrable
+
+    attr_reader :requests, :responses
+
+    def initialize(conns=[], &block)
+      @requests  = []
+      @responses = {:succeeded => [], :failed => []}
+
+      conns.each {|conn| add(conn)}
+      callback(&block) if block_given?
+    end
+
+    def add(conn)
+      @requests.push(conn)
+
+      conn.callback { @responses[:succeeded].push(conn); check_progress }
+      conn.errback  { @responses[:failed].push(conn); check_progress }
+    end
+
+    protected
+
+    # invoke callback if all requests have completed
+    def check_progress
+      succeed(self) if (@responses[:succeeded].size +
+                        @responses[:failed].size) == @requests.size
+    end
+
+  end
+end