http_tools 0.4.0 → 0.4.1

data/README.rdoc

@@ -12,6 +12,10 @@ Written purely in Ruby, with no dependencies outside of the standard library, it
 should run across all Ruby implementations compatible with 1.8 or later, and
 install in environments without a compiler available.
 
+All tests are currently run (and pass) against ruby 1.8.6, 1.8.7, 1.9.2, jruby 1.6.2, rubinius 1.2.4, MacRuby 0.10
+
+Performance tuning is mainly aimed at ruby 1.9, with ruby 1.8 and jruby taken in to consideration. jruby is generally fastest.
+
 == HTTPTools::Parser
 
 HTTPTools::Parser is a HTTP request & response parser with an evented API.
@@ -21,8 +25,8 @@ possible.
 === Example
 
   parser = HTTPTools::Parser.new
-  parser.on(:header) do |header|
-    puts parser.status_code + " " + parser.request_method
+  parser.on(:header) do
+    puts parser.status_code + " " + parser.message
     puts parser.header.inspect
   end
   parser.on(:finish) {print parser.body}

data/example/websocket_client.rb

@@ -0,0 +1,96 @@
+require 'socket'
+require 'digest/md5'
+require 'uri'
+require 'rubygems'
+require 'http_tools'
+
+# Very basic implmentation of a WebSocket client according to
+# http://tools.ietf.org/html/draft-hixie-thewebsocketprotocol-76
+# 
+# Example:
+#   sock = WebSocket.new("ws://echo.websocket.org/")
+#   sock.puts("test")
+#   puts sock.gets
+# 
+class WebSocket
+  attr_accessor :host, :port, :path, :origin
+  
+  def initialize(url, origin="localhost")
+    uri = URI.parse(url)
+    @host = uri.host
+    @port = uri.port || 80
+    @path = uri.path
+    @origin = origin
+    @socket = TCPSocket.new(host, port)
+    handshake(@socket)
+  end
+  
+  def gets
+    get_frame(@socket)
+  end
+  
+  def puts(str)
+    put_frame(@socket, str)
+    nil
+  end
+  
+  private
+  def handshake(socket)
+    number1, key1 = number_and_key
+    number2, key2 = number_and_key
+    key3 = (0..7).to_a.inject("") {|memo, i| memo << rand(256)}
+    header = {
+      "Connection" => "Upgrade",
+      "Upgrade" => "WebSocket",
+      "Origin" => origin,
+      "Sec-WebSocket-Key1" => key1,
+      "Sec-WebSocket-Key2" => key2}
+    hostport = host
+    hostport += port.to_s unless port == 80
+    socket << HTTPTools::Builder.request(:get, hostport, path, header) << key3
+    
+    parser = HTTPTools::Parser.new
+    code = nil
+    
+    parser.on(:header) do
+      raise "status is not 101" unless parser.status_code == 101
+      unless parser.header["Sec-WebSocket-Origin"] == origin
+        raise "origin missmatch"
+      end
+    end
+    parser << socket.sysread(1024 * 16) until parser.finished?
+    reply = parser.rest
+    reply << socket.read(16 - reply.length) if reply.length < 16
+    unless reply == Digest::MD5.digest([number1, number2].pack("N*") + key3)
+      raise "handshake failed" 
+    end
+  end
+  
+  def number_and_key
+    spaces = rand(12) + 1
+    number = rand((4_294_967_295 / spaces) + 1)
+    key = (number * spaces).to_s
+    chars = ("!".."/").to_a + (":".."~").to_a
+    (rand(12) + 1).times {key[rand(key.length), 0] = chars[rand(chars.length)]}
+    spaces.times {key[rand(key.length - 2) + 1, 0] = " "}
+    [number, key]
+  end
+  
+  def get_frame(socket)
+    type = socket.getbyte
+    raise "don't understand frame type #{type}" unless type & 128 == 0
+    str = socket.gets("\377")
+    str.chomp!("\377")
+    str.force_encoding("UTF-8") if str.respond_to?(:force_encoding)
+    str
+  end
+  
+  def put_frame(socket, str)
+    str.force_encoding("UTF-8") if str.respond_to?(:force_encoding)
+    socket.putc("\000")
+    written = socket.write(str)
+    socket.putc("\377")
+    written
+  end
+  
+end

data/example/websocket_server.rb

@@ -0,0 +1,92 @@
+require 'socket'
+require 'digest/md5'
+require 'rubygems'
+require 'http_tools'
+
+module WebSocket
+  
+  # Very basic implmentation of a WebSocket server according to
+  # http://tools.ietf.org/html/draft-hixie-thewebsocketprotocol-76
+  # 
+  # Example:
+  #   server = WebSocket::Server.new("localhost", 9292)
+  #   while sock = server.accept
+  #     Thread.new {loop {sock.puts sock.gets}}
+  #   end
+  # 
+  # Test at http://websocket.org/echo.html, with location ws://localhost:9292/
+  # Works with Safari 5.1 and Chrome 12
+  # 
+  class Server
+    def initialize(host, port)
+      @server = TCPServer.new(host, port)
+    end
+    
+    def accept
+      ServerSocket.new(@server.accept)
+    end
+  end
+  
+  class ServerSocket
+    def initialize(socket)
+      @socket = socket
+      handshake(socket)
+    end
+    
+    def gets
+      get_frame(@socket)
+    end
+    
+    def puts(str)
+      put_frame(@socket, str)
+      nil
+    end
+    
+    private
+    def handshake(socket)
+      parser = HTTPTools::Parser.new
+      key1, key2 = nil
+      response_header = {"Connection" => "Upgrade", "Upgrade" => "WebSocket"}
+      
+      parser.on(:header) do
+        key1 = parser.header["Sec-WebSocket-Key1"]
+        key2 = parser.header["Sec-WebSocket-Key2"]
+        location = "ws://" + parser.header["Host"] + parser.path_info
+        response_header["Sec-WebSocket-Location"] = location
+        response_header["Sec-WebSocket-Origin"] = parser.header["Origin"]
+      end
+      parser << socket.sysread(1024 * 16) until parser.finished?
+      key3 = parser.rest
+      key3 << socket.read(8 - key3.length) if key3.length < 8
+      
+      socket << HTTPTools::Builder.response(101, response_header)
+      socket << response_key(key1, key2, key3)
+    end
+    
+    def response_key(key1, key2, key3)
+      Digest::MD5.digest([key1, key2].map(&method(:process_key)).join + key3)
+    end
+    
+    def process_key(key)
+      [key.scan(/\d+/).join.to_i / key.scan(/ /).length].pack("N*")
+    end
+    
+    def get_frame(socket)
+      type = socket.getbyte
+      raise unless type == 0
+      str = socket.gets("\377")
+      str.chomp!("\377")
+      str.force_encoding("UTF-8") if str.respond_to?(:force_encoding)
+      str
+    end
+    
+    def put_frame(socket, str)
+      str.force_encoding("UTF-8") if str.respond_to?(:force_encoding)
+      socket.putc("\000")
+      written = socket.write(str)
+      socket.putc("\377")
+      written
+    end
+    
+  end
+end

data/lib/http_tools/parser.rb

@@ -12,8 +12,8 @@ module HTTPTools
   # 
   # Example:
   #   parser = HTTPTools::Parser.new
-  #   parser.on(:header) do |header|
-  #     puts parser.status_code + " " + parser.request_method
+  #   parser.on(:header) do
+  #     puts parser.status_code + " " + parser.message
   #     puts parser.header.inspect
   #   end
   #   parser.on(:finish) {print parser.body}
@@ -75,6 +75,9 @@ module HTTPTools
     # Allow responses with no status line or headers if it looks like HTML.
     attr_accessor :allow_html_without_header
     
+    # Max size for a 'Transfer-Encoding: chunked' body chunk. nil for no limit.
+    attr_accessor :max_chunk_size
+    
     # :call-seq: Parser.new -> parser
     # 
     # Create a new HTTPTools::Parser.
@@ -87,7 +90,9 @@ module HTTPTools
       @force_no_body = nil
       @allow_html_without_header = nil
       @force_trailer = nil
+      @max_chunk_size = nil
       @status_code = nil
+      @header_done = nil
       @content_left = nil
       @chunked = nil
       @body = nil
@@ -196,6 +201,15 @@ module HTTPTools
       @state == :end_of_message
     end
     
+    # :call-seq: parser.header? -> bool
+    # 
+    # Returns true when the parser has received the complete header, false
+    # otherwise.
+    # 
+    def header?
+      @header_done
+    end
+    
     # :call-seq: parser.rest -> string
     # 
     # Returns unconsumed data in the parser's buffer.
@@ -204,6 +218,14 @@ module HTTPTools
       @buffer.rest
     end
     
+    # :call-seq: parser.rest_size -> int
+    # 
+    # Returns the size in bytes of the unconsumed data in the parser's buffer.
+    # 
+    def rest_size
+      @buffer.rest_size
+    end
+    
     # :call-seq: parser.reset -> parser
     # 
     # Reset the parser so it can be used to process a new request.
@@ -219,12 +241,17 @@ module HTTPTools
       @request_uri = nil
       @version = nil
       @status_code = nil
+      @header_done = nil
       @header = {}
       @trailer = {}
       @last_key = nil
       @content_left = nil
       @chunked = nil
       @trailer_expected = nil
+      @body = nil
+      if @stream_callback == method(:stream_callback)
+        @stream_callback = method(:setup_stream_callback)
+      end
       self
     end
     
@@ -406,11 +433,17 @@ module HTTPTools
     end
     
     def start_body
+      @header_done = true
       if @request_method && !(@content_left || @chunked) ||
         NO_BODY.key?(@status_code) || @force_no_body
         end_of_message
       elsif @content_left
         @buffer = [@buffer.rest]
+        if @content_left >= @scanner.rest_size
+          @scanner.terminate
+        else
+          @scanner.pos += @content_left
+        end
         body_with_length
       elsif @chunked
         @trailer_expected = @header.any? {|k,v| TRAILER.casecmp(k) == 0}
@@ -426,7 +459,7 @@ module HTTPTools
       if !chunk.empty?
         chunk_length = chunk.length
         if chunk_length > @content_left
-          @scanner << chunk.slice!(@content_left..-1)
+          @scanner.terminate << chunk.slice!(@content_left..-1)
         end
         @stream_callback.call(chunk) if @stream_callback
         @content_left -= chunk_length
@@ -458,6 +491,8 @@ module HTTPTools
           else
             break end_of_message
           end
+        elsif @max_chunk_size && chunk_length > @max_chunk_size
+          raise ParseError.new("Maximum chunk size exceeded")
         end
         
         begin

data/test/encoding/url_encoding_test.rb

@@ -1,3 +1,4 @@
+# encoding: utf-8
 base = File.expand_path(File.dirname(__FILE__) + '/../../lib')
 require base + '/http_tools'
 require 'test/unit'
@@ -34,4 +35,4 @@ class URLEncodingTest < Test::Unit::TestCase
     assert_equal("À", result)
   end
   
-end
+end

data/test/encoding/www_form_test.rb

@@ -4,18 +4,29 @@ require 'test/unit'
 
 class WWWFormTest < Test::Unit::TestCase
   
+  def ruby_one_nine_or_greater?
+    ruby_version = RUBY_VERSION.split(".").map {|d| d.to_i}
+    ruby_version[0] > 1 || (ruby_version[0] == 1 && ruby_version[1] >= 9)
+  end
+  
   def test_encode
     result = HTTPTools::Encoding.www_form_encode("foo" => "bar", "baz" => "qux")
     
-    # may fail under Ruby < 1.9 because of the unpredictable ordering of Hash
-    assert_equal("foo=bar&baz=qux", result)
+    if ruby_one_nine_or_greater?
+      assert_equal("foo=bar&baz=qux", result)
+    else
+      assert_equal(["baz=qux", "foo=bar"], result.split("&").sort)
+    end
   end
   
   def test_encode_with_array
     result = HTTPTools::Encoding.www_form_encode("lang" => ["en", "fr"], "q" => ["foo", "bar"])
     
-    # may fail under Ruby < 1.9 because of the unpredictable ordering of Hash
-    assert_equal("lang=en&lang=fr&q=foo&q=bar", result)
+    if ruby_one_nine_or_greater?
+      assert_equal("lang=en&lang=fr&q=foo&q=bar", result)
+    else
+      assert_equal(["lang=en", "lang=fr", "q=bar", "q=foo"], result.split("&").sort)
+    end
   end
   
   def test_decode
@@ -39,4 +50,4 @@ class WWWFormTest < Test::Unit::TestCase
     assert_equal(orginal, decoded)
   end
   
-end
+end

data/test/parser/request_test.rb

@@ -379,6 +379,56 @@ class ParserRequestTest < Test::Unit::TestCase
     assert_equal(2, calls)
   end
   
+  def test_default_body_with_reset
+    parser = HTTPTools::Parser.new
+    
+    parser << "POST /example HTTP/1.1\r\n"
+    parser << "Host: www.example.com\r\n"
+    parser << "Content-Length: 3\r\n\r\n"
+    parser << "foo"
+    
+    assert_equal("foo", parser.body)
+    
+    parser.reset
+    
+    parser << "POST /example HTTP/1.1\r\n"
+    parser << "Host: www.example.com\r\n"
+    parser << "Content-Length: 3\r\n\r\n"
+    parser << "bar"
+    
+    assert_equal("bar", parser.body)
+  end
+  
+  def test_rest_with_request_in_one_chunk
+    parser = HTTPTools::Parser.new
+    
+    parser << "POST /example HTTP/1.1\r\nContent-Length: 4\r\n\r\ntest"
+    
+    assert_equal("test", parser.body)
+    assert_equal("", parser.rest)
+  end
+  
+  def test_rest_with_request_and_next_in_one_chunk
+    parser = HTTPTools::Parser.new
+    
+    parser << "POST /example HTTP/1.1\r\nContent-Length: 4\r\n\r\ntestPOST /ex"
+    
+    assert_equal("test", parser.body)
+    assert_equal("POST /ex", parser.rest)
+  end
+  
+  def test_rest_size
+    parser = HTTPTools::Parser.new
+    
+    parser << "GET /foo"
+    
+    assert_equal("/foo".length, parser.rest_size)
+    
+    parser << " HTTP/1.1\r\nHost: www.example.com"
+    
+    assert_equal("www.example.com".length, parser.rest_size)
+  end
+  
   def test_not_a_http_request
     parser = HTTPTools::Parser.new
     
@@ -458,6 +508,72 @@ class ParserRequestTest < Test::Unit::TestCase
     assert_instance_of(HTTPTools::ParseError, error)
   end
   
+  def test_upgrade_websocket_hixie_76
+    parser = HTTPTools::Parser.new
+    method, path, headers = nil
+    
+    parser.add_listener(:header) do
+      method = parser.request_method
+      path = parser.path_info
+      headers = parser.header
+    end
+    
+    parser << "GET /demo HTTP/1.1\r\n"
+    parser << "Host: example.com\r\n"
+    parser << "Connection: Upgrade\r\n"
+    parser << "Sec-WebSocket-Key2: 12998 5 Y3 1  .P00\r\n"
+    parser << "Sec-WebSocket-Protocol: sample\r\n"
+    parser << "Upgrade: WebSocket\r\n"
+    parser << "Sec-WebSocket-Key1: 4 @1  46546xW%0l 1 5\r\n"
+    parser << "Origin: http://example.com\r\n\r\n^n:ds[4U"
+    
+    assert_equal("GET", method)
+    assert_equal("/demo", path)
+    assert_equal({
+      "Host" => "example.com",
+      "Connection" => "Upgrade",
+      "Sec-WebSocket-Key2" => "12998 5 Y3 1  .P00",
+      "Upgrade" => "WebSocket",
+      "Sec-WebSocket-Protocol" => "sample",
+      "Sec-WebSocket-Key1" => "4 @1  46546xW%0l 1 5",
+      "Origin" => "http://example.com"}, headers)
+    assert(parser.finished?, "Parser should be finished.")
+    assert_equal(parser.rest, "^n:ds[4U")
+  end
+  
+  def test_upgrade_websocket_hybi_09
+    parser = HTTPTools::Parser.new
+    method, path, headers = nil
+    
+    parser.add_listener(:header) do
+      method = parser.request_method
+      path = parser.path_info
+      headers = parser.header
+    end
+    
+    parser << "GET /chat HTTP/1.1\r\n"
+    parser << "Host: server.example.com\r\n"
+    parser << "Upgrade: websocket\r\n"
+    parser << "Connection: Upgrade\r\n"
+    parser << "Sec-WebSocket-Key: dGhlIHNhbXBsZSBub25jZQ==\r\n"
+    parser << "Sec-WebSocket-Origin: http://example.com\r\n"
+    parser << "Sec-WebSocket-Protocol: chat, superchat\r\n"
+    parser << "Sec-WebSocket-Version: 8\r\n\r\n"
+    
+    assert_equal("GET", method)
+    assert_equal("/chat", path)
+    assert_equal({
+      "Host" => "server.example.com",
+      "Upgrade" => "websocket",
+      "Connection" => "Upgrade",
+      "Sec-WebSocket-Key" => "dGhlIHNhbXBsZSBub25jZQ==",
+      "Sec-WebSocket-Origin" => "http://example.com",
+      "Sec-WebSocket-Protocol" => "chat, superchat",
+      "Sec-WebSocket-Version" => "8"}, headers)
+    assert(parser.finished?, "Parser should be finished.")
+    assert_equal(parser.rest, "")
+  end
+  
   def test_env
     parser = HTTPTools::Parser.new
     env = nil

data/test/parser/response_test.rb

@@ -1,3 +1,4 @@
+# encoding: utf-8
 base = File.expand_path(File.dirname(__FILE__) + '/../../lib')
 require base + '/http_tools'
 require 'test/unit'
@@ -724,6 +725,76 @@ class ParserResponseTest < Test::Unit::TestCase
     assert(parser.finished?, "parser should be finished")
   end
   
+  def test_max_chunk_size
+    parser = HTTPTools::Parser.new
+    parser.max_chunk_size = 1024
+    
+    parser << "HTTP/1.1 200 OK\r\nTransfer-Encoding: chunked\r\n\r\n"
+    
+    assert_nothing_raised(HTTPTools::ParseError) {parser << "1\r\na\r\n"}
+    assert_nothing_raised(HTTPTools::ParseError) do
+      parser << "400\r\n#{"a" * 1024}\r\n"
+    end
+    
+    assert_raise(HTTPTools::ParseError) {parser << "401\r\n"}
+  end
+  
+  def test_upgrade_websocket_hixie_76
+    parser = HTTPTools::Parser.new
+    code, message, headers = nil
+    
+    parser.add_listener(:header) do
+      code = parser.status_code
+      message = parser.message
+      headers = parser.header
+    end
+    
+    parser << "HTTP/1.1 101 WebSocket Protocol Handshake\r\n"
+    parser << "Upgrade: WebSocket\r\n"
+    parser << "Connection: Upgrade\r\n"
+    parser << "Sec-WebSocket-Origin: http://example.com\r\n"
+    parser << "Sec-WebSocket-Location: ws://example.com/demo\r\n"
+    parser << "Sec-WebSocket-Protocol: sample\r\n\r\n8jKS'y:G*Co,Wxa-"
+    
+    assert_equal(101, code)
+    assert_equal("WebSocket Protocol Handshake", message)
+    assert_equal({
+      "Upgrade" => "WebSocket",
+      "Connection" => "Upgrade",
+      "Sec-WebSocket-Origin" => "http://example.com",
+      "Sec-WebSocket-Location" => "ws://example.com/demo",
+      "Sec-WebSocket-Protocol" => "sample"}, headers)
+    assert(parser.finished?, "Parser should be finished.")
+    assert_equal(parser.rest, "8jKS'y:G*Co,Wxa-")
+  end
+  
+  def test_upgrade_websocket_hybi_09
+    parser = HTTPTools::Parser.new
+    code, message, headers = nil
+    
+    parser.add_listener(:header) do
+      code = parser.status_code
+      message = parser.message
+      headers = parser.header
+    end
+    
+    parser << "HTTP/1.1 101 Switching Protocols\r\n"
+    parser << "Upgrade: websocket\r\n"
+    parser << "Connection: Upgrade\r\n"
+    parser << "Sec-WebSocket-Accept: s3pPLMBiTxaQ9kYGzzhZRbK+xOo=\r\n"
+    parser << "Sec-WebSocket-Protocol: chat\r\n\r\n"
+    
+    assert_equal(101, code)
+    assert_equal("Switching Protocols", message)
+    assert_equal({
+      "Upgrade" => "websocket",
+      "Connection" => "Upgrade",
+      "Sec-WebSocket-Accept" => "s3pPLMBiTxaQ9kYGzzhZRbK+xOo=",
+      "Sec-WebSocket-Protocol" => "chat"}, headers)
+    assert(parser.finished?, "Parser should be finished.")
+    assert_equal(parser.rest, "")
+  end
+  
   def test_html_body_only_not_allowed
     parser = HTTPTools::Parser.new
     
@@ -789,6 +860,20 @@ class ParserResponseTest < Test::Unit::TestCase
     assert_nil(parser.body)
   end
   
+  def test_header_query
+    parser = HTTPTools::Parser.new
+    
+    parser << "HTTP/1.1 200 OK\r\n"
+    assert(!parser.header?, "Header not yet done.")
+    
+    parser << "Content-Length: 20\r\n"
+    assert(!parser.header?, "Header not yet done.")
+    
+    parser << "\r\n"
+    
+    assert(parser.header?, "Header should be done.")
+  end
+  
   def test_finished
     parser = HTTPTools::Parser.new
     code, message, remainder = nil
@@ -1052,4 +1137,4 @@ class ParserResponseTest < Test::Unit::TestCase
     assert_raise(HTTPTools::EmptyMessageError) {parser.finish}
   end
   
-end
+end

data/test/runner.rb

@@ -1 +1,10 @@
-Dir["**/*_test.rb"].each {|test| require test}
+puts
+if defined? RUBY_DESCRIPTION
+  puts RUBY_DESCRIPTION
+elsif defined? RUBY_ENGINE
+  puts "#{RUBY_ENGINE} #{RUBY_VERSION} (#{RELEASE_DATE} patchlevel #{RUBY_PATCHLEVEL}) [#{RUBY_PLATFORM}]"
+else
+  puts "ruby #{RUBY_VERSION} (#{RELEASE_DATE} patchlevel #{RUBY_PATCHLEVEL}) [#{RUBY_PLATFORM}]"
+end
+
+Dir["**/*_test.rb"].each {|test| require File.expand_path(test)}

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification 
 name: http_tools
 version: !ruby/object:Gem::Version 
-  hash: 15
+  hash: 13
   prerelease: false
   segments: 
   - 0
   - 4
-  - 0
-  version: 0.4.0
+  - 1
+  version: 0.4.1
 platform: ruby
 authors: 
 - Matthew Sadler
@@ -15,7 +15,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2011-06-12 00:00:00 +01:00
+date: 2011-06-27 00:00:00 +01:00
 default_executable: 
 dependencies: []
 
@@ -50,6 +50,8 @@ files:
 - profile/parser/response_profile.rb
 - example/http_client.rb
 - example/http_server.rb
+- example/websocket_client.rb
+- example/websocket_server.rb
 - README.rdoc
 has_rdoc: true
 homepage: http://github.com/matsadler/http_tools