fast_http 0.1.1

data/ext/http11_client/http11_parser.h

@@ -0,0 +1,48 @@
+/**
+ * Copyright (c) 2005 Zed A. Shaw
+ * You can redistribute it and/or modify it under the same terms as Ruby.
+ */
+
+#ifndef http11_parser_h
+#define http11_parser_h
+
+#include <sys/types.h>
+
+#if defined(_WIN32)
+#include <stddef.h>
+#endif
+
+typedef void (*element_cb)(void *data, const char *at, size_t length);
+typedef void (*field_cb)(void *data, const char *field, size_t flen, const char *value, size_t vlen);
+
+typedef struct httpclient_parser { 
+  int cs;
+  size_t body_start;
+  int content_len;
+  size_t nread;
+  size_t mark;
+  size_t field_start;
+  size_t field_len;
+
+  void *data;
+
+  field_cb http_field;
+  element_cb reason_phrase;
+  element_cb status_code;
+  element_cb chunk_size;
+  element_cb http_version;
+  element_cb header_done;
+  element_cb last_chunk;
+  
+  
+} httpclient_parser;
+
+int httpclient_parser_init(httpclient_parser *parser);
+int httpclient_parser_finish(httpclient_parser *parser);
+size_t httpclient_parser_execute(httpclient_parser *parser, const char *data, size_t len, size_t off);
+int httpclient_parser_has_error(httpclient_parser *parser);
+int httpclient_parser_is_finished(httpclient_parser *parser);
+
+#define httpclient_parser_nread(parser) (parser)->nread 
+
+#endif

data/ext/http11_client/http11_parser.rl

@@ -0,0 +1,173 @@
+/**
+ * Copyright (c) 2005 Zed A. Shaw
+ * You can redistribute it and/or modify it under the same terms as Ruby.
+ */
+
+#include "http11_parser.h"
+#include <stdio.h>
+#include <assert.h>
+#include <stdlib.h>
+#include <ctype.h>
+#include <string.h>
+
+#define LEN(AT, FPC) (FPC - buffer - parser->AT)
+#define MARK(M,FPC) (parser->M = (FPC) - buffer)
+#define PTR_TO(F) (buffer + parser->F)
+#define L(M) fprintf(stderr, "" # M "\n");
+
+
+/** machine **/
+%%{
+  machine httpclient_parser;
+
+  action mark {MARK(mark, fpc); }
+
+  action start_field { MARK(field_start, fpc); }
+
+  action write_field { 
+    parser->field_len = LEN(field_start, fpc);
+  }
+
+  action start_value { MARK(mark, fpc); }
+
+  action write_value { 
+    parser->http_field(parser->data, PTR_TO(field_start), parser->field_len, PTR_TO(mark), LEN(mark, fpc));
+  }
+
+  action reason_phrase { 
+    parser->reason_phrase(parser->data, PTR_TO(mark), LEN(mark, fpc));
+  }
+
+  action status_code { 
+    parser->status_code(parser->data, PTR_TO(mark), LEN(mark, fpc));
+  }
+
+  action http_version {	
+    parser->http_version(parser->data, PTR_TO(mark), LEN(mark, fpc));
+  }
+
+  action chunk_size {
+    parser->chunk_size(parser->data, PTR_TO(mark), LEN(mark, fpc));
+  }
+
+  action last_chunk {
+    parser->last_chunk(parser->data, NULL, 0);
+  }
+
+  action done { 
+    parser->body_start = fpc - buffer + 1; 
+    if(parser->header_done != NULL)
+      parser->header_done(parser->data, fpc + 1, pe - fpc - 1);
+    fbreak;
+  }
+
+# line endings
+  CRLF = "\r\n";
+
+# character types
+  CTL = (cntrl | 127);
+  tspecials = ("(" | ")" | "<" | ">" | "@" | "," | ";" | ":" | "\\" | "\"" | "/" | "[" | "]" | "?" | "=" | "{" | "}" | " " | "\t");
+
+# elements
+  token = (ascii -- (CTL | tspecials));
+
+  Reason_Phrase = (any -- CRLF)+ >mark %reason_phrase;
+  Status_Code = digit+ >mark %status_code;
+  http_number = (digit+ "." digit+) ;
+  HTTP_Version = ("HTTP/" http_number) >mark %http_version ;
+  Status_Line = HTTP_Version " " Status_Code " " Reason_Phrase :> CRLF;
+
+  field_name = token+ >start_field %write_field;
+  field_value = any* >start_value %write_value;
+  message_header = field_name ":" " "* field_value :> CRLF;
+
+  Response = 	Status_Line (message_header)* (CRLF @done);
+
+  chunk_ext_val = token+;
+  chunk_ext_name = token+;
+  chunk_extension = (";" chunk_ext_name >start_field %write_field %start_value ("=" chunk_ext_val >start_value)? %write_value )*;
+  last_chunk = "0"? chunk_extension :> (CRLF @last_chunk @done);
+  chunk_size = xdigit+;
+  chunk = chunk_size >mark %chunk_size chunk_extension :> (CRLF @done);
+  Chunked_Header = (chunk | last_chunk);
+
+  main := Response | Chunked_Header;
+}%%
+
+/** Data **/
+%% write data;
+
+int httpclient_parser_init(httpclient_parser *parser)  {
+  int cs = 0;
+  %% write init;
+  parser->cs = cs;
+  parser->body_start = 0;
+  parser->content_len = 0;
+  parser->mark = 0;
+  parser->nread = 0;
+  parser->field_len = 0;
+  parser->field_start = 0;    
+
+  return(1);
+}
+
+
+/** exec **/
+size_t httpclient_parser_execute(httpclient_parser *parser, const char *buffer, size_t len, size_t off)  {
+  const char *p, *pe;
+  int cs = parser->cs;
+
+  assert(off <= len && "offset past end of buffer");
+
+  p = buffer+off;
+  pe = buffer+len;
+
+  assert(*pe == '\0' && "pointer does not end on NUL");
+  assert(pe - p == len - off && "pointers aren't same distance");
+
+
+  %% write exec;
+
+  parser->cs = cs;
+  parser->nread += p - (buffer + off);
+
+  assert(p <= pe && "buffer overflow after parsing execute");
+  assert(parser->nread <= len && "nread longer than length");
+  assert(parser->body_start <= len && "body starts after buffer end");
+  assert(parser->mark < len && "mark is after buffer end");
+  assert(parser->field_len <= len && "field has length longer than whole buffer");
+  assert(parser->field_start < len && "field starts after buffer end");
+
+  if(parser->body_start) {
+    /* final \r\n combo encountered so stop right here */
+    %%write eof;
+    parser->nread++;
+  }
+
+  return(parser->nread);
+}
+
+int httpclient_parser_finish(httpclient_parser *parser)
+{
+  int cs = parser->cs;
+
+  %%write eof;
+
+  parser->cs = cs;
+
+  if (httpclient_parser_has_error(parser) ) {
+    return -1;
+  } else if (httpclient_parser_is_finished(parser) ) {
+    return 1;
+  } else {
+    return 0;
+  }
+}
+
+int httpclient_parser_has_error(httpclient_parser *parser) {
+  return parser->cs == httpclient_parser_error;
+}
+
+int httpclient_parser_is_finished(httpclient_parser *parser) {
+  return parser->cs == httpclient_parser_first_final;
+}

data/ext/http11_client/mkmf.log

@@ -0,0 +1,12 @@
+have_library: checking for main() in -lc... -------------------- yes
+
+"gcc -o conftest -I. -I/usr/local/lib/ruby/1.8/i686-darwin9.8.0 -I.  -D_XOPEN_SOURCE -D_DARWIN_C_SOURCE   -g -O2 -pipe -fno-common   conftest.c  -L. -L/usr/local/lib -L.      -lruby-static -lc  -ldl -lobjc  "
+checked program was:
+/* begin */
+1: /*top*/
+2: int main() { return 0; }
+3: int t() { void ((*volatile p)()); p = (void ((*)()))main; return 0; }
+/* end */
+
+--------------------
+

data/lib/fast_http.rb

@@ -0,0 +1,3 @@
+require File.join(File.dirname(__FILE__), *%w[.. ext http11_client http11_client])
+require File.join(File.dirname(__FILE__), *%w[fast_http client])
+require File.join(File.dirname(__FILE__), *%w[fast_http pushbackio])

data/lib/fast_http/client.rb

@@ -0,0 +1,441 @@
+require 'socket'
+require 'timeout'
+
+module FastHttp
+
+  # Thrown for errors not related to the protocol format (HttpClientParserError are
+  # thrown for that).
+  class HttpClientError < StandardError; end
+
+  # A simple hash is returned for each request made by HttpClient with
+  # the headers that were given by the server for that request.
+  class HttpResponse < Hash
+    # The reason returned in the http response ("OK","File not found",etc.)
+    attr_accessor :http_reason
+
+    # The HTTP version returned.
+    attr_accessor :http_version
+
+    # The status code (as a string!)
+    attr_accessor :http_status
+
+    # The http body of the response, in the raw
+    attr_accessor :http_body
+
+    # When parsing chunked encodings this is set
+    attr_accessor :http_chunk_size
+
+    # The actual chunks taken from the chunked encoding
+    attr_accessor :raw_chunks
+
+    # Converts the http_chunk_size string properly
+    def chunk_size
+      if @chunk_size == nil
+        @chunk_size = @http_chunk_size ? @http_chunk_size.to_i(base=16) : 0
+      end
+
+      @chunk_size
+    end
+
+    # true if this is the last chunk, nil otherwise (false)
+    def last_chunk?
+      @last_chunk || chunk_size == 0
+    end
+
+    # Easier way to find out if this is a chunked encoding
+    def chunked_encoding?
+      /chunked/i === self[HttpClient::TRANSFER_ENCODING]
+    end
+  end
+
+  # A mixin that has most of the HTTP encoding methods you need to work
+  # with the protocol.  It's used by HttpClient, but you can use it
+  # as well.
+  module HttpEncoding
+    COOKIE="Cookie"
+    FIELD_ENCODING="%s: %s\r\n" 
+
+    # Converts a Hash of cookies to the appropriate simple cookie
+    # headers.
+    def encode_cookies(cookies)
+      result = ""
+      cookies.each do |k,v|
+        if v.kind_of? Array
+          v.each {|x| result << encode_field(COOKIE, encode_param(k,x)) }
+        else
+          result << encode_field(COOKIE, encode_param(k,v))
+        end
+      end
+      return result
+    end
+
+    # Encode HTTP header fields of "k: v\r\n"
+    def encode_field(k,v)
+      FIELD_ENCODING % [k,v]
+    end
+
+    # Encodes the headers given in the hash returning a string
+    # you can use.
+    def encode_headers(head)
+      @headers_cache = {}
+      @headers_cache[head] || 
+      result = "" 
+      head.each do |k,v|
+        if v.kind_of? Array
+          v.each {|x| result << encode_field(k,x) }
+        else
+          result << encode_field(k,v)
+        end
+      end
+      return result
+    end
+
+    # URL encodes a single k=v parameter.
+    def encode_param(k,v)
+      escape(k) + "=" + escape(v)
+    end
+
+    # Takes a query string and encodes it as a URL encoded 
+    # set of key=value pairs with & separating them.
+    def encode_query(uri, query)
+      params = []
+
+      if query
+        query.each do |k,v|
+          if v.kind_of? Array
+            v.each {|x| params << encode_param(k,x) } 
+          else
+            params << encode_param(k,v)
+          end
+        end
+
+        uri += "?" + params.join('&')
+      end
+
+      return uri
+    end
+
+    # HTTP is kind of retarded that you have to specify
+    # a Host header, but if you include port 80 then further
+    # redirects will tack on the :80 which is annoying.
+    def encode_host(host, port)
+      host + (port.to_i != 80 ? ":#{port}" : "")
+    end
+
+    # Escapes a URI.
+    def escape(s)
+      s.to_s.gsub(/([^ a-zA-Z0-9_.-]+)/n) {
+        '%'+$1.unpack('H2'*$1.size).join('%').upcase
+      }.tr(' ', '+') 
+    end
+
+
+    # Unescapes a URI escaped string.
+    def unescape(s)
+      s.tr('+', ' ').gsub(/((?:%[0-9a-fA-F]{2})+)/n){
+        [$1.delete('%')].pack('H*')
+      } 
+    end
+
+    # Parses a query string by breaking it up at the '&' 
+    # and ';' characters.  You can also use this to parse
+    # cookies by changing the characters used in the second
+    # parameter (which defaults to '&;'.
+    def query_parse(qs, d = '&;')
+      params = {}
+      (qs||'').split(/[#{d}] */n).inject(params) { |h,p|
+        k, v=unescape(p).split('=',2)
+        if cur = params[k]
+          if cur.class == Array
+            params[k] << v
+          else
+            params[k] = [cur, v]
+          end
+        else
+          params[k] = v
+        end
+      }
+
+      return params
+    end
+  end
+
+
+  # The actual HttpClient that does the work with the thinnest
+  # layer between you and the protocol.  All exceptions and leaks
+  # are allowed to pass through since those are important when
+  # testing.  It doesn't pretend to be a full client, but instead
+  # is just enough client to track cookies, form proper HTTP requests,
+  # and return HttpResponse hashes with the results.
+  #
+  # It's designed so that you create one client, and then you work it
+  # with a minimum of parameters as you need.  The initialize method
+  # lets you pass in defaults for most of the parameters you'll need,
+  # and you can simple call the method you want and it'll be translated
+  # to an HTTP method (client.get => GET, client.foobar = FOOBAR).
+  #
+  # Here's a few examples:
+  #
+  #   client = HttpClient.new(:head => {"X-DefaultHeader" => "ONE"})
+  #   resp = client.post("/test")
+  #   resp = client.post("/test", :head => {"X-TestSend" => "Status"}, :body => "TEST BODY")
+  #   resp = client.put("/testput", :query => {"q" => "test"}, :body => "SOME JUNK")
+  #   client.reset
+  #
+  # The HttpClient.reset call clears cookies that are maintained.
+  #
+  # It uses method_missing to do the translation of .put to "PUT /testput HTTP/1.1"
+  # so you can get into trouble if you're calling unknown methods on it.  By
+  # default the methods are PUT, GET, POST, DELETE, HEAD.  You can change
+  # the allowed methods by passing :allowed_methods => [:put, :get, ..] to
+  # the initialize for the object.
+  #
+  # == Notifications
+  #
+  # You can register a "notifier" with the client that will get called when
+  # different events happen.  Right now the Notifier class just has a few
+  # functions for the common parts of an HTTP request that each take a 
+  # symbol and some extra parameters.  See FastHttp::Notifier for more 
+  # information.
+  #
+  # == Parameters
+  #
+  #   :head => {K => V}  or {K => [V1,V2]}
+  #   :query => {K => V} or {K => [V1,V2]}
+  #   :body => "some body" (you must encode for now)
+  #   :cookies => {K => V} or {K => [V1, V2]}
+  #   :allowed_methods => [:put, :get, :post, :delete, :head]
+  #   :notifier => Notifier.new
+  #   :redirect => false (give it a number and it'll follow redirects for that count)
+  #
+  class HttpClient
+    include HttpEncoding
+
+    TRANSFER_ENCODING="TRANSFER_ENCODING"
+    CONTENT_LENGTH="CONTENT_LENGTH"
+    SET_COOKIE="SET_COOKIE"
+    LOCATION="LOCATION"
+    HOST="HOST"
+    HTTP_REQUEST_HEADER="%s %s HTTP/1.1\r\n"
+    REQ_CONTENT_LENGTH="Content-Length"
+    REQ_HOST="Host"
+    CHUNK_SIZE=1024 * 16
+    CRLF="\r\n"
+
+    # Access to the host, port, default options, and cookies currently in play
+    attr_accessor :host, :port, :options, :cookies, :allowed_methods, :sock
+
+    # Doesn't make the connect until you actually call a .put,.get, etc.
+    def initialize(host, port, options = {})
+      @options = options
+      @host = host
+      @port = port
+      @cookies = options[:cookies]
+      @allowed_methods = options[:allowed_methods] || [:put, :get, :post, :delete, :head]
+      @redirect = options[:redirect] || false
+      @parser = HttpClientParser.new
+      @ignore_data = options[:ignore_data]
+    end
+
+
+    # Builds a full request from the method, uri, req, and @cookies
+    # using the default @options and writes it to out (should be an IO).
+    def build_request(out, method, uri, req)
+      ops = @options.merge(req)
+      query = ops[:query]
+
+      # merge head differently since that's typically what they mean
+      head = req[:head] || {}
+      head = ops[:head].merge(head) if ops[:head]
+
+      # setup basic headers we always need
+      head[REQ_HOST] = encode_host(@host,@port)
+      head[REQ_CONTENT_LENGTH] = ops[:body] ? ops[:body].length : 0
+
+      # blast it out
+      out.write(HTTP_REQUEST_HEADER % [method, encode_query(uri,query)])
+      out.write(encode_headers(head))
+      if @cookies
+        out.write(encode_cookies(@cookies.merge(req[:cookies] || {})))
+      elsif req[:cookies]
+        out.write(encode_cookies(req[:cookies]))
+      end
+      out.write(CRLF)
+    end
+
+    # Does the read operations needed to parse a header with the @parser.
+    # A "header" in this case is either an HTTP header or a Chunked encoding
+    # header (since the @parser handles both).
+    def read_parsed_header
+      @parser.reset
+      resp = HttpResponse.new
+      data = @sock.read(CHUNK_SIZE, partial=true)
+      nread = @parser.execute(resp, data, 0)
+
+      while !@parser.finished?
+        data << @sock.read(CHUNK_SIZE, partial=true)
+        nread = @parser.execute(resp, data, nread)
+      end
+
+      return resp
+    end
+
+
+    # Used to process chunked headers and then read up their bodies.
+    def read_chunked_header
+      resp = read_parsed_header
+      @sock.push(resp.http_body)
+
+      if !resp.last_chunk?
+        resp.http_body = @sock.read(resp.chunk_size)
+
+        trail = @sock.read(2)
+        if trail != CRLF
+          raise HttpClientParserError.new("Chunk ended in #{trail.inspect} not #{CRLF.inspect}")
+        end
+      end
+
+      return resp
+    end
+
+
+    # Collects up a chunked body both collecting the body together *and*
+    # collecting the chunks into HttpResponse.raw_chunks[] for alternative
+    # analysis.
+    def read_chunked_body(header)
+      @sock.push(header.http_body)
+      header.http_body = ""
+      header.raw_chunks = []
+
+      while true
+        chunk = read_chunked_header
+        header.raw_chunks << chunk unless @ignore_data
+        if !chunk.last_chunk?
+          header.http_body << chunk.http_body unless @ignore_data
+        else
+          break # last chunk, done
+        end
+      end
+
+      header
+    end
+
+    # Reads the SET_COOKIE string out of resp and translates it into 
+    # the @cookies store for this HttpClient.
+    def store_cookies(resp)
+      if @cookies and resp[SET_COOKIE]
+        cookies = query_parse(resp[SET_COOKIE], ';')
+        @cookies.merge! cookies
+        @cookies.delete "path"
+      end
+    end
+
+    # Reads an HTTP response from the given socket.  It uses 
+    # readpartial which only appeared in Ruby 1.8.4.  The result
+    # is a fully formed HttpResponse object for you to play with.
+    # 
+    # As with other methods in this class it doesn't stop any exceptions
+    # from reaching your code.  It's for experts who want these exceptions
+    # so either write a wrapper, use net/http, or deal with it on your end.
+    def read_response
+      resp = HttpResponse.new
+
+      resp = read_parsed_header
+
+      if resp.chunked_encoding?
+        read_chunked_body(resp)
+      elsif resp[CONTENT_LENGTH]
+        needs = resp[CONTENT_LENGTH].to_i - resp.http_body.length
+        # Some requests can actually give a content length, and then not have content
+        # so we ignore HttpClientError exceptions and pray that's good enough
+        if @ignore_data
+          @sock.read(needs) if needs > 0 rescue HttpClientError
+        else
+          resp.http_body += @sock.read(needs) if needs > 0 rescue HttpClientError
+        end
+      else
+        while true
+          begin
+            if @ignore_data
+              @sock.read(CHUNK_SIZE, partial=true)
+            else
+              resp.http_body += @sock.read(CHUNK_SIZE, partial=true)
+            end
+          rescue HttpClientError
+            break # this is fine, they closed the socket then
+          end
+        end
+      end
+
+      store_cookies(resp) if @cookies
+      return resp
+    end
+
+    # Does the socket connect and then build_request, read_response
+    # calls finally returning the result.
+    def send_request(method, uri, req)
+      begin
+        @sock = PushBackIO.new(TCPSocket.new(@host, @port))
+
+        out = StringIO.new
+        build_request(out, method, uri, req)
+        body = req[:body] || "" unless method == :head or method == :get
+
+        @sock.write(out.string + body)
+        @sock.flush
+
+        return read_response
+      rescue Object
+        raise $!
+      ensure
+        if @sock
+          @sock.close
+        end
+      end
+    end
+
+
+    # Translates unknown function calls into PUT, GET, POST, DELETE, HEAD 
+    # methods.  The allowed HTTP methods allowed are restricted by the
+    # @allowed_methods attribute which you can set after construction or
+    # during construction with :allowed_methods => [:put, :get, ...]
+    def method_missing(symbol, *args)
+      if @allowed_methods.include? symbol
+        method = symbol.to_s.upcase
+        resp = send_request(method, args[0], args[1] || {})
+        resp = redirect(symbol, resp) if @redirect
+
+        return resp
+      else
+        raise HttpClientError.new("Invalid method: #{symbol}")
+      end
+    end
+
+    # Keeps doing requests until it doesn't receive a 3XX request.
+    def redirect(method, resp, *args)
+      @redirect.times do
+        break if resp.http_status.index("3") != 0
+
+        host = encode_host(@host,@port)
+        location = resp[LOCATION]
+
+        if location.index(host) == 0
+          # begins with the host so strip that off
+          location = location[host.length .. -1]
+        end
+
+        resp = self.send(method, location, *args)
+      end
+
+      return resp
+    end
+
+    # Clears out the cookies in use so far in order to get
+    # a clean slate.
+    def reset
+      @cookies.clear
+    end
+  end
+  
+end
+