tmm1-em-http-request 0.1.0

data/ext/http11_client/http11_parser.h

@@ -0,0 +1,48 @@
+/**
+ * Copyright (c) 2005 Zed A. Shaw
+ * You can redistribute it and/or modify it under the same terms as Ruby.
+ */
+
+#ifndef http11_parser_h
+#define http11_parser_h
+
+#include <sys/types.h>
+
+#if defined(_WIN32)
+#include <stddef.h>
+#endif
+
+typedef void (*element_cb)(void *data, const char *at, size_t length);
+typedef void (*field_cb)(void *data, const char *field, size_t flen, const char *value, size_t vlen);
+
+typedef struct httpclient_parser { 
+  int cs;
+  size_t body_start;
+  int content_len;
+  size_t nread;
+  size_t mark;
+  size_t field_start;
+  size_t field_len;
+
+  void *data;
+
+  field_cb http_field;
+  element_cb reason_phrase;
+  element_cb status_code;
+  element_cb chunk_size;
+  element_cb http_version;
+  element_cb header_done;
+  element_cb last_chunk;
+  
+  
+} httpclient_parser;
+
+int httpclient_parser_init(httpclient_parser *parser);
+int httpclient_parser_finish(httpclient_parser *parser);
+size_t httpclient_parser_execute(httpclient_parser *parser, const char *data, size_t len, size_t off);
+int httpclient_parser_has_error(httpclient_parser *parser);
+int httpclient_parser_is_finished(httpclient_parser *parser);
+
+#define httpclient_parser_nread(parser) (parser)->nread 
+
+#endif

data/ext/http11_client/http11_parser.rl

@@ -0,0 +1,173 @@
+/**
+ * Copyright (c) 2005 Zed A. Shaw
+ * You can redistribute it and/or modify it under the same terms as Ruby.
+ */
+
+#include "http11_parser.h"
+#include <stdio.h>
+#include <assert.h>
+#include <stdlib.h>
+#include <ctype.h>
+#include <string.h>
+
+#define LEN(AT, FPC) (FPC - buffer - parser->AT)
+#define MARK(M,FPC) (parser->M = (FPC) - buffer)
+#define PTR_TO(F) (buffer + parser->F)
+#define L(M) fprintf(stderr, "" # M "\n");
+
+
+/** machine **/
+%%{
+  machine httpclient_parser;
+
+  action mark {MARK(mark, fpc); }
+
+  action start_field { MARK(field_start, fpc); }
+
+  action write_field { 
+    parser->field_len = LEN(field_start, fpc);
+  }
+
+  action start_value { MARK(mark, fpc); }
+
+  action write_value { 
+    parser->http_field(parser->data, PTR_TO(field_start), parser->field_len, PTR_TO(mark), LEN(mark, fpc));
+  }
+
+  action reason_phrase { 
+    parser->reason_phrase(parser->data, PTR_TO(mark), LEN(mark, fpc));
+  }
+
+  action status_code { 
+    parser->status_code(parser->data, PTR_TO(mark), LEN(mark, fpc));
+  }
+
+  action http_version {	
+    parser->http_version(parser->data, PTR_TO(mark), LEN(mark, fpc));
+  }
+
+  action chunk_size {
+    parser->chunk_size(parser->data, PTR_TO(mark), LEN(mark, fpc));
+  }
+
+  action last_chunk {
+    parser->last_chunk(parser->data, NULL, 0);
+  }
+
+  action done { 
+    parser->body_start = fpc - buffer + 1; 
+    if(parser->header_done != NULL)
+      parser->header_done(parser->data, fpc + 1, pe - fpc - 1);
+    fbreak;
+  }
+
+# line endings
+  CRLF = "\r\n";
+
+# character types
+  CTL = (cntrl | 127);
+  tspecials = ("(" | ")" | "<" | ">" | "@" | "," | ";" | ":" | "\\" | "\"" | "/" | "[" | "]" | "?" | "=" | "{" | "}" | " " | "\t");
+
+# elements
+  token = (ascii -- (CTL | tspecials));
+
+  Reason_Phrase = (any -- CRLF)* >mark %reason_phrase;
+  Status_Code = digit{3} >mark %status_code;
+  http_number = (digit+ "." digit+) ;
+  HTTP_Version = ("HTTP/" http_number) >mark %http_version ;
+  Status_Line = HTTP_Version " " Status_Code " "? Reason_Phrase :> CRLF;
+
+  field_name = token+ >start_field %write_field;
+  field_value = any* >start_value %write_value;
+  message_header = field_name ":" " "* field_value :> CRLF;
+
+  Response = 	Status_Line (message_header)* (CRLF @done);
+
+  chunk_ext_val = token+;
+  chunk_ext_name = token+;
+  chunk_extension = (";" chunk_ext_name >start_field %write_field %start_value ("=" chunk_ext_val >start_value)? %write_value )*;
+  last_chunk = "0"? chunk_extension :> (CRLF @last_chunk @done);
+  chunk_size = xdigit+;
+  chunk = chunk_size >mark %chunk_size chunk_extension space* :> (CRLF @done);
+  Chunked_Header = (chunk | last_chunk);
+
+  main := Response | Chunked_Header;
+}%%
+
+/** Data **/
+%% write data;
+
+int httpclient_parser_init(httpclient_parser *parser)  {
+  int cs = 0;
+  %% write init;
+  parser->cs = cs;
+  parser->body_start = 0;
+  parser->content_len = 0;
+  parser->mark = 0;
+  parser->nread = 0;
+  parser->field_len = 0;
+  parser->field_start = 0;    
+
+  return(1);
+}
+
+
+/** exec **/
+size_t httpclient_parser_execute(httpclient_parser *parser, const char *buffer, size_t len, size_t off)  {
+  const char *p, *pe;
+  int cs = parser->cs;
+
+  assert(off <= len && "offset past end of buffer");
+
+  p = buffer+off;
+  pe = buffer+len;
+
+  assert(*pe == '\0' && "pointer does not end on NUL");
+  assert(pe - p == len - off && "pointers aren't same distance");
+
+
+  %% write exec;
+
+  parser->cs = cs;
+  parser->nread += p - (buffer + off);
+
+  assert(p <= pe && "buffer overflow after parsing execute");
+  assert(parser->nread <= len && "nread longer than length");
+  assert(parser->body_start <= len && "body starts after buffer end");
+  assert(parser->mark < len && "mark is after buffer end");
+  assert(parser->field_len <= len && "field has length longer than whole buffer");
+  assert(parser->field_start < len && "field starts after buffer end");
+
+  if(parser->body_start) {
+    /* final \r\n combo encountered so stop right here */
+    %%write eof;
+    parser->nread++;
+  }
+
+  return(parser->nread);
+}
+
+int httpclient_parser_finish(httpclient_parser *parser)
+{
+  int cs = parser->cs;
+
+  %%write eof;
+
+  parser->cs = cs;
+
+  if (httpclient_parser_has_error(parser) ) {
+    return -1;
+  } else if (httpclient_parser_is_finished(parser) ) {
+    return 1;
+  } else {
+    return 0;
+  }
+}
+
+int httpclient_parser_has_error(httpclient_parser *parser) {
+  return parser->cs == httpclient_parser_error;
+}
+
+int httpclient_parser_is_finished(httpclient_parser *parser) {
+  return parser->cs == httpclient_parser_first_final;
+}

data/lib/em-http/client.rb

@@ -0,0 +1,404 @@
+# #--
+# Copyright (C)2008 Ilya Grigorik
+#
+# Includes portion originally Copyright (C)2007 Tony Arcieri
+# Includes portion originally Copyright (C)2005 Zed Shaw
+# You can redistribute this under the terms of the Ruby
+# license See file LICENSE for details
+# #--
+
+module EventMachine
+
+  # A simple hash is returned for each request made by HttpClient with the
+  # headers that were given by the server for that request.
+  class HttpResponseHeader < Hash
+    # The reason returned in the http response ("OK","File not found",etc.)
+    attr_accessor :http_reason
+
+    # The HTTP version returned.
+    attr_accessor :http_version
+
+    # The status code (as a string!)
+    attr_accessor :http_status
+
+    # HTTP response status as an integer
+    def status
+      Integer(http_status) rescue nil
+    end
+
+    # Length of content as an integer, or nil if chunked/unspecified
+    def content_length
+      Integer(self[HttpClient::CONTENT_LENGTH]) rescue nil
+    end
+
+    # Is the transfer encoding chunked?
+    def chunked_encoding?
+      /chunked/i === self[HttpClient::TRANSFER_ENCODING]
+    end
+
+    def keep_alive?
+      /keep-alive/i === self[HttpClient::KEEP_ALIVE]
+    end
+
+    def compressed?
+      /gzip|compressed|deflate/i === self[HttpClient::CONTENT_ENCODING]
+    end
+  end
+
+  class HttpChunkHeader < Hash
+    # When parsing chunked encodings this is set
+    attr_accessor :http_chunk_size
+
+    # Size of the chunk as an integer
+    def chunk_size
+      return @chunk_size unless @chunk_size.nil?
+      @chunk_size = @http_chunk_size ? @http_chunk_size.to_i(base=16) : 0
+    end
+  end
+
+  # Methods for building HTTP requests
+  module HttpEncoding
+    HTTP_REQUEST_HEADER="%s %s HTTP/1.1\r\n"
+    FIELD_ENCODING = "%s: %s\r\n"
+    BASIC_AUTH_ENCODING = "%s: Basic %s\r\n"
+
+    # Escapes a URI.
+    def escape(s)
+      s.to_s.gsub(/([^ a-zA-Z0-9_.-]+)/n) {
+        '%'+$1.unpack('H2'*$1.size).join('%').upcase
+      }.tr(' ', '+')
+    end
+
+    # Unescapes a URI escaped string.
+    def unescape(s)
+      s.tr('+', ' ').gsub(/((?:%[0-9a-fA-F]{2})+)/n){
+        [$1.delete('%')].pack('H*')
+      }
+    end
+
+    # Map all header keys to a downcased string version
+    def munge_header_keys(head)
+      head.inject({}) { |h, (k, v)| h[k.to_s.downcase] = v; h }
+    end
+
+    # HTTP is kind of retarded that you have to specify a Host header, but if
+    # you include port 80 then further redirects will tack on the :80 which is
+    # annoying.
+    def encode_host
+      @uri.host + (@uri.port.to_i != 80 ? ":#{@uri.port}" : "")
+    end
+
+    def encode_request(method, path, query)
+      HTTP_REQUEST_HEADER % [method.to_s.upcase, encode_query(path, query)]
+    end
+
+    def encode_query(path, query)
+      return path unless query
+      path + "?" + query.map { |k, v| encode_param(k, v) }.join('&')
+    end
+
+    # URL encodes query parameters:
+    # single k=v, or a URL encoded array, if v is an array of values
+    def encode_param(k, v)
+      if v.is_a?(Array)
+        v.map { |e| escape(k) + "[]=" + escape(e) }.join("&")
+      else
+        escape(k) + "=" + escape(v)
+      end
+    end
+
+    # Encode a field in an HTTP header
+    def encode_field(k, v)
+      FIELD_ENCODING % [k, v]
+    end
+
+    # Encode basic auth in an HTTP header
+    def encode_basic_auth(k,v)
+      BASIC_AUTH_ENCODING % [k, Base64.encode64(v.join(":")).chomp]
+    end
+
+    def encode_headers(head)
+      head.inject('') do |result, (key, value)|
+        # Munge keys from foo-bar-baz to Foo-Bar-Baz
+        key = key.split('-').map { |k| k.capitalize }.join('-')
+        unless key == "Authorization"
+          result << encode_field(key, value)
+        else
+          result << encode_basic_auth(key, value)
+        end
+      end
+    end
+
+    def encode_cookies(cookies)
+      cookies.inject('') { |result, (k, v)| result << encode_field('Cookie', encode_param(k, v)) }
+    end
+  end
+
+  class HttpClient < Connection
+    include EventMachine::Deferrable
+    include HttpEncoding
+
+    TRANSFER_ENCODING="TRANSFER_ENCODING"
+    CONTENT_ENCODING="CONTENT_ENCODING"
+    CONTENT_LENGTH="CONTENT_LENGTH"
+    KEEP_ALIVE="CONNECTION"
+    SET_COOKIE="SET_COOKIE"
+    LOCATION="LOCATION"
+    HOST="HOST"
+    CRLF="\r\n"
+
+    attr_accessor :method, :options, :uri
+    attr_reader   :response, :response_header, :errors
+
+    def post_init
+      self.comm_inactivity_timeout = 5
+
+      @parser = HttpClientParser.new
+      @data = EventMachine::Buffer.new
+      @response_header = HttpResponseHeader.new
+      @chunk_header = HttpChunkHeader.new
+
+      @state = :response_header
+      @parser_nbytes = 0
+      @inflate = []
+      @response = ''
+      @errors = ''
+    end
+
+    # start HTTP request once we establish connection to host
+    def connection_completed
+      send_request_header
+      send_request_body
+    end
+
+    # request is done, invoke the callback
+    def on_request_complete
+
+      if @response_header.compressed? and @inflate.include?(response_header[CONTENT_ENCODING])
+        case response_header[CONTENT_ENCODING]
+        when 'deflate' then 
+          @response = Zlib::Inflate.inflate(@response)
+        when 'gzip', 'compressed' then
+          @response = Zlib::GzipReader.new(StringIO.new(@response)).read
+        end
+      end
+			
+      unbind
+    end
+
+    # request failed, invoke errback
+    def on_error(msg)
+      @errors = msg
+      unbind
+    end
+
+    def send_request_header
+      query   = @options[:query]
+      head    = @options[:head] ? munge_header_keys(@options[:head]) : {}
+      body    = @options[:body]
+
+      # Set the Host header if it hasn't been specified already
+      head['host'] ||= encode_host
+
+      # Set the Content-Length if body is given
+      head['content-length'] =  body.length if body
+
+      # Set the User-Agent if it hasn't been specified
+      head['user-agent'] ||= "EventMachine HttpClient"
+
+      # Set auto-inflate flags
+      if head['accept-encoding']
+        @inflate = head['accept-encoding'].split(',').map {|t| t.strip}
+      end
+
+      # Build the request
+      request_header = encode_request(@method, @uri.path, query)
+      request_header << encode_headers(head)
+      request_header << CRLF
+
+      send_data request_header
+    end
+
+    def send_request_body
+      send_data @options[:body] if @options[:body]
+    end
+
+    def receive_data(data)
+      @data << data
+      dispatch
+    end
+
+    # Called when part of the body has been read
+    def on_body_data(data)
+      @response << data
+    end
+
+    def unbind
+      (@state == :finished) ? succeed : fail
+      close_connection
+    end
+
+    #
+    # Response processing
+    #
+
+    def dispatch
+      while case @state
+        when :response_header
+          parse_response_header
+        when :chunk_header
+          parse_chunk_header
+        when :chunk_body
+          process_chunk_body
+        when :chunk_footer
+          process_chunk_footer
+        when :response_footer
+          process_response_footer
+        when :body
+          process_body
+        when :finished, :invalid
+          break
+        else raise RuntimeError, "invalid state: #{@state}"
+        end
+      end
+    end
+
+    def parse_header(header)
+      return false if @data.empty?
+
+      begin
+        @parser_nbytes = @parser.execute(header, @data.to_str, @parser_nbytes)
+      rescue EventMachine::HttpClientParserError
+        @state = :invalid
+        on_error "invalid HTTP format, parsing fails"
+      end
+
+      return false unless @parser.finished?
+
+      # Clear parsed data from the buffer
+      @data.read(@parser_nbytes)
+      @parser.reset
+      @parser_nbytes = 0
+
+      true
+    end
+
+    def parse_response_header
+      return false unless parse_header(@response_header)
+
+      unless @response_header.http_status and @response_header.http_reason
+        @state = :invalid
+        on_error "no HTTP response"
+        return false
+      end
+
+      if @response_header.chunked_encoding?
+        @state = :chunk_header
+      else
+        @state = :body
+        @bytes_remaining = @response_header.content_length
+      end
+
+      true
+    end
+
+    def parse_chunk_header
+      return false unless parse_header(@chunk_header)
+
+      @bytes_remaining = @chunk_header.chunk_size
+      @chunk_header = HttpChunkHeader.new
+
+      @state = @bytes_remaining > 0 ? :chunk_body : :response_footer
+      true
+    end
+
+    def process_chunk_body
+      if @data.size < @bytes_remaining
+        @bytes_remaining -= @data.size
+        on_body_data @data.read
+        return false
+      end
+
+      on_body_data @data.read(@bytes_remaining)
+      @bytes_remaining = 0
+
+      @state = :chunk_footer
+      true
+    end
+
+    def process_chunk_footer
+      return false if @data.size < 2
+
+      if @data.read(2) == CRLF
+        @state = :chunk_header
+      else
+        @state = :invalid
+        on_error "non-CRLF chunk footer"
+      end
+
+      true
+    end
+
+    def process_response_footer
+      return false if @data.size < 2
+
+      if @data.read(2) == CRLF
+        if @data.empty?
+          @state = :finished
+          on_request_complete
+        else
+          @state = :invalid
+          on_error "garbage at end of chunked response"
+        end
+      else
+        @state = :invalid
+        on_error "non-CRLF response footer"
+      end
+
+      false
+    end
+
+    def process_body
+      if @bytes_remaining.nil?
+        on_body_data @data.read
+        return false
+      end
+
+      if @bytes_remaining.zero?
+        @state = :finished
+        on_request_complete
+        return false
+      end
+
+      if @data.size < @bytes_remaining
+        @bytes_remaining -= @data.size
+        on_body_data @data.read
+        return false
+      end
+
+      on_body_data @data.read(@bytes_remaining)
+      @bytes_remaining = 0
+
+      # If Keep-Alive is enabled, the server may be pushing more data to us
+      # after the first request is complete. Hence, finish first request, and
+      # reset state.
+      if @response_header.keep_alive?
+        @data.clear # hard reset, TODO: add support for keep-alive connections!
+        @state = :finished
+        on_request_complete
+
+      else
+        if @data.empty?
+          @state = :finished
+          on_request_complete
+        else
+          @state = :invalid
+          on_error "garbage at end of body"
+        end
+      end
+
+      false
+    end
+  end
+
+end

data/lib/em-http/multi.rb

@@ -0,0 +1,51 @@
+module EventMachine
+
+  # EventMachine based Multi request client, based on a streaming HTTPRequest class, 
+  # which allows you to open multiple parallel connections and return only when all
+  # of them finish. (i.e. ideal for parallelizing workloads)
+  # 
+  # == Example
+  # 
+  #  EventMachine.run {
+  #   
+  #   multi = EventMachine::MultiRequest.new
+  #    
+  #   # add multiple requests to the multi-handler
+  #   multi.add(EventMachine::HttpRequest.new('http://www.google.com/').get)
+  #   multi.add(EventMachine::HttpRequest.new('http://www.yahoo.com/').get)
+  #    
+  #    multi.callback  {
+  #       p multi.responses[:succeeded]
+  #       p multi.responses[:failed]
+  #     
+  #       EventMachine.stop
+  #    }
+  #  }
+  # 
+  
+  class MultiRequest
+    include EventMachine::Deferrable
+
+    attr_reader :requests, :responses
+    
+    def initialize
+      @requests = []
+      @responses = {:succeeded => [], :failed => []}
+    end
+    
+    def add(conn)
+      conn.callback { @responses[:succeeded].push(conn); check_progress }
+      conn.errback  { @responses[:failed].push(conn); check_progress }
+      
+      @requests.push(conn)
+    end
+    
+    protected
+   
+    # invoke callback if all requests have completed
+    def check_progress
+      succeed if (@responses[:succeeded].size + @responses[:failed].size) == @requests.size
+    end
+    
+  end
+end