http_tools 0.1.0 → 0.2.0

data/README.rdoc

@@ -6,20 +6,26 @@ including a fast-as-possible pure Ruby HTTP parser.
 * rdoc[http://sourcetagsandcodes.com/http_tools/doc/]
 * source[https://github.com/matsadler/http_tools]
 
+== Platform Support
+
+Written purely in Ruby, with no dependencies outside of the standard library, it
+should run across all Ruby implementations compatible with 1.8 or later, and
+install in environments without a compiler available.
+
 == HTTPTools::Parser
 
 HTTPTools::Parser is a HTTP request & response parser with an evented API.
-Written purely in Ruby, with no dependencies, it should run across all Ruby
-implementations, and install in environments without a compiler available.
 Despite being just Ruby, every effort has been made to ensure it is as fast as
 possible.
 
 === Example
 
   parser = HTTPTools::Parser.new
-  parser.on(:status) {|status, message| puts "#{status} #{message}"}
-  parser.on(:headers) {|headers| puts headers.inspect}
-  parser.on(:body) {|body| puts body}
+  parser.on(:header) do |header|
+    puts parser.status_code + " " + parser.method
+    puts parser.header.inspect
+  end
+  parser.on(:stream) {|chunk| print chunk}
   
   parser << "HTTP/1.1 200 OK\r\n"
   parser << "Content-Length: 20\r\n\r\n"

data/bench/parser/request_bench.rb

@@ -21,29 +21,25 @@ Benchmark.bm(41) do |x|
   
   x.report("HTTPTools::Parser (reset, with callbacks)") do
     parser = HTTPTools::Parser.new
-    parser.on(:method) {|arg|}
-    parser.on(:path) {|arg, arg2|}
-    parser.on(:headers) {|arg|}
+    parser.on(:headers) {}
     10_000.times do
        parser << request
        parser.reset
     end
   end
   
-  x.report("HTTPTools::Parser (reset, with delegate)") do
-    class TestDelegate
-      def on_method(arg)
-      end
-      def on_path(arg, arg2)
-      end
-      def on_headers(arg)
+  begin
+    require 'rubygems'
+    require 'http/parser'
+    x.report("Http::Parser") do
+      10_000.times do
+        parser = Http::Parser.new
+        parser.on_headers_complete = Proc.new {}
+        parser.on_message_complete = Proc.new {}
+        parser << request
       end
     end
-    parser = HTTPTools::Parser.new(TestDelegate.new)
-    10_000.times do
-       parser << request
-       parser.reset
-    end
+  rescue LoadError
   end
   
   begin

data/bench/parser/response_bench.rb

@@ -18,4 +18,16 @@ Benchmark.bm(25) do |x|
        parser.reset
     end
   end
+  
+  begin
+    require 'rubygems'
+    require 'http/parser'
+    x.report("Http::Parser") do
+      10_000.times do
+        parser = Http::Parser.new
+        parser << response
+      end
+    end
+  rescue LoadError
+  end
 end

data/bench/transfer_encoding_chunked_bench.rb

@@ -0,0 +1,26 @@
+base = File.expand_path(File.dirname(__FILE__) + '/../lib')
+require base + '/http_tools'
+require 'benchmark'
+
+Benchmark.bm(36) do |x|
+  x.report("lots of very short chunks") do
+    encoded = "1\r\na\r\n" * 100 + "0\r\n"
+    1_000.times do
+       HTTPTools::Encoding.transfer_encoding_chunked_decode(encoded)
+    end
+  end
+  
+  x.report("slightly less slightly longer chunks") do
+    encoded = "16\r\n<h1>Hello world</h1>\r\n\r\n12\r\n<p>Lorem ipsum</p>\r\n" * 50 + "0\r\n"
+    1_000.times do
+       HTTPTools::Encoding.transfer_encoding_chunked_decode(encoded)
+    end
+  end
+  
+  x.report("a couple of big chunks") do
+    encoded = "2710\r\n#{"a" * 10000}\r\n" * 2 + "0\r\n"
+    1_000.times do
+       HTTPTools::Encoding.transfer_encoding_chunked_decode(encoded)
+    end
+  end
+end

data/example/http_client.rb

@@ -29,9 +29,12 @@ module HTTP
     CONTENT_LENGTH = "Content-Length".freeze
     WWW_FORM = "application/x-www-form-urlencoded".freeze
     
+    attr_writer :keepalive
+    
     def initialize(host, port=80)
       @host = host
       @port = port
+      @pipeline = []
     end
     
     def socket
@@ -58,9 +61,9 @@ module HTTP
       if headers[CONTENT_LENGTH]
         # ok
       elsif body.respond_to?(:length)
-        headers[CONTENT_LENGTH] ||= body.length
+        headers[CONTENT_LENGTH] = body.length
       elsif body.respond_to?(:stat)
-        headers[CONTENT_LENGTH] ||= body.stat.size
+        headers[CONTENT_LENGTH] = body.stat.size
       else
         raise "Content-Length must be supplied"
       end
@@ -68,47 +71,88 @@ module HTTP
       request(:post, path, body, headers, &block)
     end
     
+    def pipeline
+      @pipelining = true
+      yield self
+      pipeline_requests(@pipeline)
+    ensure
+      @pipelining = false
+    end
+    
+    def keepalive?
+      @keepalive
+    end
+    
+    def keepalive
+      self.keepalive, original = true, keepalive?
+      yield self
+    ensure
+      self.keepalive = original
+    end
+    
     private
-    def request(method, path, request_body=nil, request_headers={}, response_has_body=true, &block)
+    def request(method, path, body=nil, headers={}, response_has_body=true, &b)
+      request = {
+        :method => method,
+        :path => path,
+        :body => body,
+        :headers => headers,
+        :response_has_body => response_has_body,
+        :block => b}
+      if @pipelining
+        @pipeline << request
+        nil
+      else
+        pipeline_requests([request]).first
+      end
+    end
+    
+    def pipeline_requests(requests)
       parser = HTTPTools::Parser.new
-      parser.force_no_body = !response_has_body
-      response = nil
+      parser.allow_html_without_header = true
+      responses = []
       
-      parser.add_listener(:status) {|s, m| response = Response.new(s, m)}
-      parser.add_listener(:headers) do |headers|
-        response.headers = headers
-        if block
-          response.parser = parser
-          block.call(response)
-          response.parser = nil
+      parser.on(:finish) do |remainder|
+        if responses.length < requests.length
+          parser.reset
+          parser << remainder.lstrip if remainder
+          throw :reset
         end
       end
-      parser.add_listener(:body) {|body| response.body = body} unless block
-      
-      socket << HTTPTools::Builder.request(method, @host, path, request_headers)
-      if request_body
-        socket << request_body.read(1024 * 16) until request_body.eof?
+      parser.on(:header) do
+        request = requests[responses.length]
+        parser.force_no_body = !request[:response_has_body]
+        response = Response.new(parser.status_code, parser.message)
+        response.headers = parser.header
+        parser.on(:stream) {|chunk| response.receive_chunk(chunk)}
+        responses.push(response)
       end
       
-      until parser.finished?
-        begin
-          readable, = select([socket], nil, nil)
-          parser << socket.read_nonblock(1024 * 16) if readable.any?
-        rescue EOFError
-          parser.finish
-          break
+      requests.each do |r|
+        socket << HTTPTools::Builder.request(r[:method], @host, r[:path], r[:headers])
+        if body = r[:body]
+          socket << body.read(1024 * 16) until body.eof?
         end
       end
-      response
+      
+      begin
+        catch(:reset) {parser << socket.sysread(1024 * 16)}
+      rescue EOFError
+        @socket = nil
+        parser.finish
+        break
+      end until parser.finished?
+      
+      @socket = nil unless keepalive?
+      responses
     end
   end
   
   class Response
     attr_reader :status, :message
     attr_accessor :headers, :body
-    attr_accessor :parser # :nodoc:
     
-    def initialize(status, message, headers={}, body=nil)
+    def initialize(status, message, headers={}, body="")
       @status = status
       @message = message
       @headers = headers
@@ -116,17 +160,22 @@ module HTTP
     end
     
     def stream(&block)
-      if parser
-        parser.add_listener(:stream, block)
-      else
-        block.call(body)
-      end
+      @stream_callback = block
       nil
     end
     
+    def receive_chunk(chunk) # :nodoc:
+      body << chunk
+      @stream_callback.call(chunk) if @stream_callback
+    end
+    
     def inspect
       bytesize = body.respond_to?(:bytesize) ? body.bytesize : body.to_s.length
       "#<Response #{status} #{message}: #{bytesize} bytes>"
     end
+    
+    def to_s
+      body.to_s
+    end
   end
 end

data/example/http_server.rb

@@ -0,0 +1,82 @@
+require 'socket'
+require 'stringio'
+require 'rubygems'
+require 'http_tools'
+
+module HTTP
+  class Server
+    RACK_INPUT = "rack.input".freeze
+    NO_BODY = {"GET" => true, "HEAD" => true}
+    CONNECTION = "Connection".freeze
+    KEEP_ALIVE = "Keep-Alive".freeze
+    CLOSE = "close".freeze
+    ONE_ONE = "1.1".freeze
+    
+    def initialize(app, options={})
+      host = options[:host] || options[:Host] || "0.0.0.0"
+      port = (options[:port] || options[:Port] || 9292).to_s
+      @app = app
+      @instance_env = {"SERVER_NAME" => host, "SERVER_PORT" => port,
+        "rack.multithread" => true}
+      @server = TCPServer.new(host, port)
+      @server.listen(1024)
+    end
+    
+    def self.run(app, options={})
+      new(app, options).listen
+    end
+    
+    def listen
+      while socket = @server.accept
+        Thread.new do
+          begin
+            on_connection(socket)
+          rescue StandardError, LoadError, SyntaxError => e
+            STDERR.puts("#{e.class}: #{e.message} #{e.backtrace.join("\n")}")
+          end
+        end
+      end
+    end
+    
+    private
+    def on_connection(socket)
+      parser = HTTPTools::Parser.new
+      env, input = nil
+      
+      parser.on(:header) do
+        parser.force_no_body = NO_BODY[parser.request_method]
+        input = StringIO.new
+        env = parser.env.merge!(RACK_INPUT => input).merge!(@instance_env)
+      end
+      parser.on(:stream) {|chunk| input << chunk}
+      parser.on(:finish) do |remainder|
+        input.rewind
+        status, header, body = @app.call(env)
+        keep_alive = keep_alive?(parser.version, parser.header[CONNECTION])
+        header[CONNECTION] = keep_alive ? KEEP_ALIVE : CLOSE
+        socket << HTTPTools::Builder.response(status, header)
+        body.each {|chunk| socket << chunk}
+        body.close if body.respond_to?(:close)
+        if keep_alive
+          parser.reset
+          parser << remainder.lstrip if remainder
+          throw :reset
+        end
+      end
+      
+      begin
+        readable, = select([socket], nil, nil, 30)
+        break unless readable
+        catch(:reset) {parser << socket.read_nonblock(1024 * 16)}
+      rescue EOFError
+        break
+      end until parser.finished?
+      socket.close
+    end
+    
+    def keep_alive?(http_version, connection)
+      http_version == ONE_ONE && connection != CLOSE || connection == KEEP_ALIVE
+    end
+    
+  end
+end

data/lib/http_tools.rb

@@ -96,6 +96,9 @@ module HTTPTools
   NO_BODY.merge!(204 => true, 304 => true, nil => false)
   100.upto(199) {|status_code| NO_BODY[status_code] = true}
   
+  ARRAY_VALUE_HEADERS = Hash.new {|hash, key| hash[key] = false}
+  ARRAY_VALUE_HEADERS.merge!("Set-Cookie" => true)
+  
   CRLF = "\r\n".freeze
   SPACE = " ".freeze
   
@@ -106,5 +109,6 @@ module HTTPTools
   autoload :ParseError, require_base + 'errors'
   autoload :EndOfMessageError, require_base + 'errors'
   autoload :MessageIncompleteError, require_base + 'errors'
+  autoload :EmptyMessageError, require_base + 'errors'
   
 end

data/lib/http_tools/encoding.rb

@@ -134,35 +134,36 @@ module HTTPTools
     #   while remainder
     #     remainder << get_data
     #     chunk, remainder = transfer_encoding_chunked_decode(remainder)
-    #     decoded << chunk
+    #     decoded << chunk if chunk
     #   end
     # 
-    def transfer_encoding_chunked_decode(scanner)
-      unless scanner.is_a?(StringScanner)
-        scanner = StringScanner.new(scanner.dup)
-      end
-      hex_chunk_length = scanner.scan(/[0-9a-fA-F]+\r?\n/)
-      return [nil, scanner.string] unless hex_chunk_length
-      
-      chunk_length = hex_chunk_length.to_i(16)
-      return [nil, nil] if chunk_length == 0
+    def transfer_encoding_chunked_decode(str, scanner=StringScanner.new(str))
+      decoded = ""
       
-      chunk = scanner.rest.slice(0, chunk_length)
-      begin
-        scanner.pos += chunk_length
-        separator = scanner.scan(/\n|\r\n/)
-      rescue RangeError
+      remainder = while true
+        start_pos = scanner.pos
+        hex_chunk_length = scanner.scan(/[0-9a-f]+ *\r?\n/i)
+        break scanner.rest unless hex_chunk_length
+        
+        chunk_length = hex_chunk_length.to_i(16)
+        break nil if chunk_length == 0
+        
+        begin
+          chunk = scanner.rest.slice(0, chunk_length)
+          scanner.pos += chunk_length
+          if chunk && scanner.skip(/\r?\n/i)
+            decoded << chunk
+          else
+            scanner.pos = start_pos
+            break scanner.rest
+          end
+        rescue RangeError
+          scanner.pos = start_pos
+          break scanner.rest
+        end
       end
       
-      if separator && chunk.length == chunk_length
-        scanner.string.replace(scanner.rest)
-        scanner.reset
-        rest, remainder = transfer_encoding_chunked_decode(scanner)
-        chunk << rest if rest
-        [chunk, remainder]
-      else
-        [nil, scanner.string]
-      end
+      [(decoded if decoded.length > 0), remainder]
     end
     
   end

data/lib/http_tools/errors.rb

@@ -2,4 +2,5 @@ module HTTPTools
   class ParseError < StandardError; end
   class EndOfMessageError < ParseError; end
   class MessageIncompleteError < EndOfMessageError; end
+  class EmptyMessageError < MessageIncompleteError; end
 end