rbkb-http 0.2.0

data/lib/rbkb/http/response.rb

@@ -0,0 +1,86 @@
+
+module Rbkb::Http
+  # A Response encapsulates all the entities in a HTTP response,
+  # including the status header, general headers, and body.
+  class Response < Base
+    attr_accessor :status
+
+    alias first_entity status
+    alias first_entity= status=
+
+    # Returns a raw HTTP response for this instance. Must have a status
+    # element defined at a bare minimum.
+    def to_raw(raw_body=nil)
+      raise "this response has no status" unless first_entity()
+      self.headers ||= default_headers_obj()
+      self.body = raw_body if raw_body
+
+      if do_chunked_encoding?(@headers)
+        unless @body.is_a? ChunkedBody
+          @body = ChunkedBody.new(@body, @body.opts)
+        end
+        @headers.delete_header("Content-Length")
+      elsif not opts[:ignore_content_length]
+        unless @body.is_a? BoundBody
+          @body = BoundBody.new(@body, @body.opts)
+        end
+        @headers.delete_header("Transfer-Encoding")
+      else
+        @body = Body.new(@body, @body.opts)
+      end
+      @body.base = self
+
+      yield(self) if block_given?
+
+      bstr = @body.to_raw
+      hdrs = @headers.to_raw_array.unshift(self.first_entity.to_raw)
+      return "#{hdrs.join("\r\n")}\r\n\r\n#{bstr}"
+    end
+
+    # Parses a raw HTTP response and captures data into the current instance.
+    def capture(str)
+      raise "arg 0 must be a string" unless String === str
+      hstr, bstr = str.split(/\s*\r?\n\r?\n/, 2)
+
+      capture_headers(hstr)
+
+      yield(self, bstr) if block_given?
+
+      unless @body and @body.capture_complete?
+        @body =
+          if do_chunked_encoding?
+            ChunkedBody.new {|b| b.base = self }
+          elsif content_length()
+            BoundBody.new {|b| b.base = self }
+          else
+            Body.new {|b| b.base = self }
+          end
+      end
+
+      capture_body(bstr)
+
+      return self
+    end
+
+    # Indicates whether to use chunked encoding based on presence of 
+    # the "Transfer-Encoding: chunked" header or the :ignore_chunked_encoding
+    # opts parameter.
+    def do_chunked_encoding?(hdrs=@headers)
+      ( (not @opts[:ignore_chunked_encoding]) and 
+        (hdrs.get_header_value("Transfer-Encoding").to_s =~ /(?:^|\W)chunked(?:\W|$)/) )
+    end
+
+    # Returns a new Headers object extended as ResponseHeaders. This is the 
+    # default object which will be used when composing fresh Response header
+    # entities.
+    def default_headers_obj(*args)
+      Headers.new(*args).extend(ResponseHeaders)
+    end
+
+    # Returns a new BoundBody object. This is the default object which will 
+    # be used when composing fresh Response body entities.
+    def default_body_obj(*args)
+      BoundBody.new(*args)
+    end
+  end
+end

data/lib/rbkb/http.rb

@@ -0,0 +1,22 @@
+begin ; require 'rubygems' ; rescue LoadError ; end
+
+require 'rbkb'
+
+# ???Why???? would anyone create their own HTTP implementation in ruby with 
+# so many options out there? Short answer: Net::HTTP and others just don't cut 
+# it in lots of edge cases. I needed something I could control completely.
+
+module Rbkb::Http
+  # nop
+end
+
+require 'time' # gives us Time.httpdate parser and output methods
+
+require "rbkb/http/common.rb"
+require "rbkb/http/base.rb"
+require "rbkb/http/request.rb"
+require "rbkb/http/response.rb"
+require "rbkb/http/headers.rb"
+require "rbkb/http/body.rb"
+require "rbkb/http/parameters.rb"
+

data/rbkb-http.gemspec

@@ -0,0 +1,74 @@
+# Generated by jeweler
+# DO NOT EDIT THIS FILE
+# Instead, edit Jeweler::Tasks in Rakefile, and run `rake gemspec`
+# -*- encoding: utf-8 -*-
+
+Gem::Specification.new do |s|
+  s.name = %q{rbkb-http}
+  s.version = "0.2.0"
+
+  s.required_rubygems_version = Gem::Requirement.new(">= 0") if s.respond_to? :required_rubygems_version=
+  s.authors = ["Eric Monti"]
+  s.date = %q{2009-11-12}
+  s.description = %q{HTTP libraries and tools based on and complementary to Ruby BlackBag}
+  s.email = %q{emonti@matasano.com}
+  s.extra_rdoc_files = [
+    "LICENSE",
+     "README.rdoc"
+  ]
+  s.files = [
+    ".document",
+     ".gitignore",
+     "History.txt",
+     "LICENSE",
+     "README.rdoc",
+     "Rakefile",
+     "VERSION",
+     "lib/rbkb/http.rb",
+     "lib/rbkb/http/base.rb",
+     "lib/rbkb/http/body.rb",
+     "lib/rbkb/http/common.rb",
+     "lib/rbkb/http/headers.rb",
+     "lib/rbkb/http/parameters.rb",
+     "lib/rbkb/http/request.rb",
+     "lib/rbkb/http/response.rb",
+     "rbkb-http.gemspec",
+     "spec/rbkb-http_spec.rb",
+     "spec/spec_helper.rb",
+     "test/test_helper.rb",
+     "test/test_http.rb",
+     "test/test_http_helper.rb",
+     "test/test_http_request.rb",
+     "test/test_http_response.rb"
+  ]
+  s.homepage = %q{http://github.com/emonti/rbkb-http}
+  s.rdoc_options = ["--charset=UTF-8"]
+  s.require_paths = ["lib"]
+  s.rubygems_version = %q{1.3.5}
+  s.summary = %q{HTTP protocol add-ons for Ruby BlackBag}
+  s.test_files = [
+    "spec/rbkb-http_spec.rb",
+     "spec/spec_helper.rb",
+     "test/test_helper.rb",
+     "test/test_http.rb",
+     "test/test_http_helper.rb",
+     "test/test_http_request.rb",
+     "test/test_http_response.rb"
+  ]
+
+  if s.respond_to? :specification_version then
+    current_version = Gem::Specification::CURRENT_SPECIFICATION_VERSION
+    s.specification_version = 3
+
+    if Gem::Version.new(Gem::RubyGemsVersion) >= Gem::Version.new('1.2.0') then
+      s.add_development_dependency(%q<rspec>, [">= 0"])
+      s.add_runtime_dependency(%q<rbkb>, [">= 0"])
+    else
+      s.add_dependency(%q<rspec>, [">= 0"])
+      s.add_dependency(%q<rbkb>, [">= 0"])
+    end
+  else
+    s.add_dependency(%q<rspec>, [">= 0"])
+    s.add_dependency(%q<rbkb>, [">= 0"])
+  end
+end

data/spec/rbkb-http_spec.rb

@@ -0,0 +1,7 @@
+require File.expand_path(File.dirname(__FILE__) + '/spec_helper')
+
+describe "RbkbHttp" do
+  it "fails" do
+    fail "hey buddy, you should probably rename this file and start specing for real"
+  end
+end

data/spec/spec_helper.rb

@@ -0,0 +1,9 @@
+$LOAD_PATH.unshift(File.dirname(__FILE__))
+$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
+require 'rbkb/http'
+require 'spec'
+require 'spec/autorun'
+
+Spec::Runner.configure do |config|
+  
+end

data/test/test_helper.rb

@@ -0,0 +1,9 @@
+require 'rubygems'
+require 'test/unit'
+
+$LOAD_PATH.unshift(File.dirname(__FILE__))
+$LOAD_PATH.unshift(File.join(File.dirname(__FILE__), '..', 'lib'))
+require 'rbkb/http'
+
+class Test::Unit::TestCase
+end

data/test/test_http.rb

@@ -0,0 +1,27 @@
+require File.dirname(__FILE__) + '/test_http_helper.rb'
+
+class TestHttp < Test::Unit::TestCase
+  include Rbkb::Http
+  
+  def test_names
+    assert_equal CommonInterface, Rbkb::Http::CommonInterface
+    assert_equal Base, Rbkb::Http::Base
+
+    assert_equal Response, Rbkb::Http::Response
+    assert_equal Request, Rbkb::Http::Request
+    assert_equal Parameters, Rbkb::Http::Parameters
+    assert_equal FormUrlencodedParams, Rbkb::Http::FormUrlencodedParams
+
+    assert_equal Headers, Rbkb::Http::Headers
+    assert_equal RequestHeaders, Rbkb::Http::RequestHeaders
+    assert_equal RequestAction, Rbkb::Http::RequestAction
+    assert_equal ResponseHeaders, Rbkb::Http::ResponseHeaders
+    assert_equal ResponseStatus, Rbkb::Http::ResponseStatus
+
+    assert_equal Body, Rbkb::Http::Body
+    assert_equal BoundBody, Rbkb::Http::BoundBody
+    assert_equal ChunkedBody, Rbkb::Http::ChunkedBody
+  end
+
+end
+

data/test/test_http_helper.rb

@@ -0,0 +1,59 @@
+require File.join(File.dirname(__FILE__), 'test_helper.rb')
+
+module HttpTestHelper
+  # contains various test cases to test common interface features
+  module CommonInterfaceTests
+
+    def setup()
+      raise "Helper stub called. Override setup() in TestCase"
+    end
+
+    def do_type_tests(x)
+      raise "Helper stub called. Override do_type_tests() in TestCase"
+    end
+
+    def do_type_tests(x)
+      raise "Helper stub called. Override do_capture_value_tests() in TestCase"
+    end
+
+    def test_init_parse
+      req = @obj_klass.new(@rawdat, @obj_opts)
+      do_capture_value_tests(req)
+      do_type_tests(req)
+    end
+
+    def test_parse
+      req = @obj_klass.parse(@rawdat, @obj_opts)
+      do_capture_value_tests(req)
+      do_type_tests(req)
+    end
+
+    def test_capture
+      req = @obj.capture(@rawdat)
+      do_capture_value_tests(req)
+      do_type_tests(req)
+    end
+
+    def test_back_to_raw
+      req = @obj.capture(@rawdat)
+      assert_equal @rawdat_crlf, req.to_raw
+    end
+
+    def test_capture_and_reuse_nondestructive
+      @obj.capture(@rawdat_crlf)
+      @obj.reset_capture
+      @obj.capture(@rawdat_crlf)
+      do_capture_value_tests(@obj)
+      do_type_tests(@obj)
+    end
+
+    def test_capture_and_reuse_destructive
+      @obj.capture(@rawdat_crlf)
+      @obj.reset_capture!
+      @obj.capture(@rawdat_crlf)
+      do_capture_value_tests(@obj)
+      do_type_tests(@obj)
+    end
+
+  end
+end

data/test/test_http_request.rb

@@ -0,0 +1,136 @@
+require File.dirname(__FILE__) + '/test_http_helper.rb'
+
+class TestHttpRequest < Test::Unit::TestCase
+  include HttpTestHelper::CommonInterfaceTests
+
+  include Rbkb::Http
+
+  def setup
+    @obj_klass = Request
+    @obj = @obj_klass.new
+
+    @rawdat =<<_EOF_
+GET /csi?v=3&s=webhp&action=&tran=undefined HTTP/1.1
+Host: www.google.com
+User-Agent: Mozilla/5.0
+Accept: image/png,image/*;q=0.8,*/*;q=0.5
+Accept-Language: en-us,en;q=0.5
+Accept-Encoding: gzip,deflate
+Accept-Charset: ISO-8859-1,utf-8;q=0.7,*;q=0.7
+Keep-Alive: 300
+Proxy-Connection: keep-alive
+Referer: http://www.google.com/
+Cookie: PREFID=NNNN:LM=1237996892:S=YYYYY; NID=111
+
+_EOF_
+
+    @hstr, @body = @rawdat.split(/^\n/, 2)
+    @rawdat_crlf = @hstr.gsub("\n", "\r\n") + "\r\n" + @body
+
+    @verb = "GET"
+    @uri = "/csi?v=3&s=webhp&action=&tran=undefined"
+    @path, @query = @uri.split('?',2)
+    @req_parameters = [
+      ["v", "3"], 
+      ["s", "webhp"], 
+      ["action", ""], 
+      ["tran", "undefined"]
+    ]
+    @version = "HTTP/1.1"
+
+    @headers = [
+      ["Host", "www.google.com"], 
+      ["User-Agent", "Mozilla/5.0"],
+      ["Accept", "image/png,image/*;q=0.8,*/*;q=0.5"], 
+      ["Accept-Language", "en-us,en;q=0.5"], 
+      ["Accept-Encoding", "gzip,deflate"], 
+      ["Accept-Charset", "ISO-8859-1,utf-8;q=0.7,*;q=0.7"], 
+      ["Keep-Alive", "300"], 
+      ["Proxy-Connection", "keep-alive"], 
+      ["Referer", "http://www.google.com/"], 
+      ["Cookie", "PREFID=NNNN:LM=1237996892:S=YYYYY; NID=111"],
+    ]
+  end
+
+  def do_type_tests(req)
+    assert_kind_of Request, req
+    assert_kind_of Headers, req.headers
+    assert_kind_of Body, req.body
+    assert_kind_of RequestAction, req.action
+    assert_kind_of RequestHeaders, req.headers
+  end
+
+  def do_capture_value_tests(req)
+    assert_equal @headers, req.headers
+    assert_equal @body, req.body.to_s
+    assert_equal @uri, req.action.uri.to_s
+    assert_equal @path, req.action.uri.path
+    assert_equal @query, req.action.uri.query
+    assert_equal @verb, req.action.verb
+    assert_equal @version, req.action.version
+    assert_equal @req_parameters, req.action_parameters
+  end
+
+  def test_capture_crlf_headers
+    req = @obj.capture(@rawdat_crlf)
+    do_capture_value_tests(req)
+    do_type_tests(req)
+    assert_equal @rawdat_crlf, req.to_raw
+  end
+
+  def test_captured_body_type
+    @obj.capture(@rawdat)
+    assert_kind_of Body, @obj.body
+  end
+
+end
+
+
+class TestHttpPostRequest < TestHttpRequest
+
+  def setup
+    @obj_klass = Request
+    @obj = @obj_klass.new
+
+    @rawdat =<<_EOF_
+POST /path/script.cgi?qtype=foo HTTP/1.0
+From: frog@jmarshall.com
+HasColon: this_has_a_colon:yikes
+User-Agent: HTTPTool/1.0
+Content-Type: application/x-www-form-urlencoded
+Content-Length: 32
+
+home=Cosby&favorite+flavor=flies
+_EOF_
+    @rawdat.chomp!
+
+    @hstr, @body = @rawdat.split(/^\n/, 2)
+    @rawdat_crlf = @hstr.gsub("\n", "\r\n") + "\r\n" + @body
+
+    @verb = "POST"
+    @uri = "/path/script.cgi?qtype=foo"
+    @path, @query = @uri.split('?',2)
+    @req_parameters = [
+      ["qtype", "foo"], 
+    ]
+    @version = "HTTP/1.0"
+
+    @headers = [
+      ["From", "frog@jmarshall.com"],
+      ["HasColon", "this_has_a_colon:yikes"],
+      ["User-Agent", "HTTPTool/1.0"],
+      ["Content-Type", "application/x-www-form-urlencoded"],
+      ["Content-Length", "32"],
+    ]
+
+     @post_parameters = [["home", "Cosby"], ["favorite+flavor", "flies"]]
+  end
+
+  def test_captured_body_type
+    @obj.capture(@rawdat)
+    assert_kind_of BoundBody, @obj.body
+  end
+
+
+end
+

data/test/test_http_response.rb

@@ -0,0 +1,222 @@
+require File.dirname(__FILE__) + '/test_http_helper.rb'
+
+class TestHttpResponse < Test::Unit::TestCase
+  include HttpTestHelper::CommonInterfaceTests
+
+  include Rbkb::Http
+
+  def setup
+    @obj_klass = Response
+    @obj_opts = nil
+    @obj = @obj_klass.new(nil, @obj_opts)
+
+    @rawdat =<<_EOF_
+HTTP/1.0 404 Not Found
+Date: Thu, 26 Mar 2009 01:18:52 GMT
+Server: Apache
+Content-Type: text/html; charset=iso-8859-1
+Content-Length: 190
+Connection: Keep-Alive
+
+<html><head>
+<title>Error report</title></head>
+<body><h1>HTTP Status 404</h1><HR size="1" noshade><p><u>The requested resource is not available.</u></p><HR size="1" noshade>
+</body></html>
+_EOF_
+
+    @hstr, @body = @rawdat.split(/^\n/, 2)
+    @rawdat_crlf = @hstr.gsub("\n", "\r\n") + "\r\n" + @body
+
+    @code = 404
+    @text = "Not Found"
+    @version = "HTTP/1.0"
+
+    @headers = [
+      ["Date", "Thu, 26 Mar 2009 01:18:52 GMT"],
+      ["Server", "Apache"],
+      ["Content-Type", "text/html; charset=iso-8859-1"], 
+      ["Content-Length", "190"], 
+      ["Connection", "Keep-Alive"]
+    ]
+
+    @content_length = 190
+  end
+
+  def do_type_tests(rsp)
+    assert_kind_of Response, rsp
+    assert_kind_of Headers, rsp.headers
+    assert_kind_of Body, rsp.body
+    assert_kind_of ResponseStatus, rsp.status
+    assert_kind_of ResponseHeaders, rsp.headers
+  end
+
+  def do_capture_value_tests(rsp)
+    assert_equal @headers, rsp.headers
+    assert_equal @body, rsp.body
+    assert_equal @code, rsp.status.code
+    assert_equal @text, rsp.status.text
+    assert_equal @version, rsp.status.version
+    assert_equal @content_length, rsp.content_length
+  end
+
+  def test_capture_crlf_headers
+    @obj.capture(@rawdat_crlf)
+    do_capture_value_tests(@obj)
+    do_type_tests(@obj)
+    assert_equal @rawdat_crlf, @obj.to_raw
+  end
+
+  def test_captured_body_type
+    @obj.capture(@rawdat)
+    assert_kind_of BoundBody, @obj.body
+  end
+end
+
+
+class TestHttpResponseChunked < TestHttpResponse
+  include Rbkb::Http
+
+  def setup
+    @obj_klass = Response
+    @obj_opts = {}
+    @obj = @obj_klass.new
+
+    @rawdat =<<_EOF_
+HTTP/1.1 200 OK
+Date: Thu, 26 Mar 2009 01:18:52 GMT
+Server: Apache
+Content-Type: text/html; charset=iso-8859-1
+Transfer-Encoding: chunked
+Connection: Keep-Alive
+
+20
+This is a test of a chunked-enco
+
+10
+ded HTTP request
+
+0
+_EOF_
+
+    @hstr, @rawbody = @rawdat.split(/^\n/, 2)
+    @rawdat_crlf = @rawdat.gsub("\n", "\r\n")
+    @hdrs_crlf = @hstr.gsub("\n", "\r\n")
+
+    @body = "This is a test of a chunked-encoded HTTP request"
+
+    @code = 200
+    @text = "OK"
+    @version = "HTTP/1.1"
+
+    @headers = [
+      ["Date", "Thu, 26 Mar 2009 01:18:52 GMT"],
+      ["Server", "Apache"],
+      ["Content-Type", "text/html; charset=iso-8859-1"], 
+      ["Transfer-Encoding", "chunked"], 
+      ["Connection", "Keep-Alive"]
+    ]
+
+    @content_length = nil
+    @tc_chunk_size = 0x20
+  end
+
+  def test_captured_body_type
+    @obj.capture(@rawdat)
+    assert_kind_of ChunkedBody, @obj.body
+  end
+
+  def test_back_to_raw
+    @obj.capture(@rawdat)
+    do_capture_value_tests(@obj)
+    do_type_tests(@obj)
+    @obj.body.opts[:output_chunk_size] = @tc_chunk_size
+    assert_equal @rawdat_crlf, @obj.to_raw
+  end
+
+  def test_capture_crlf_headers
+    @obj.capture(@rawdat_crlf)
+    do_capture_value_tests(@obj)
+    do_type_tests(@obj)
+    @obj.body.opts[:output_chunk_size] = @tc_chunk_size
+    assert_equal @rawdat_crlf, @obj.to_raw
+  end
+
+  def test_default_chunk_size
+    if @body.size > ChunkedBody::DEFAULT_CHUNK_SIZE
+      assert "TESTCASE ERROR!!!", "make the setup() @body < DEFAULT_CHUNK_SIZE"
+    end
+    raw_tc = "#{@hdrs_crlf}\r\n#{@body.size.to_s(16)}\r\n#{@body}\r\n\r\n0\r\n"
+    @obj.capture(@rawdat_crlf)
+    do_capture_value_tests(@obj)
+    do_type_tests(@obj)
+    assert_equal raw_tc, @obj.to_raw
+  end
+end
+
+# This test-case simulates a HTTP response to a HEAD request. This type of
+# response is special since it returns Content-Length: NNN or 
+# Transfer-Encoding: chunked headers without any actual body data. 
+# To handle this special situation, we use the 'ignore_content_length' and 
+# 'ignore_chunked_encoding' options.
+class TestHttpResponseToHead < TestHttpResponse
+  def setup
+    @obj_klass = Response
+
+    # Technically, a server should only respond to HEAD with one of 
+    # content length *or* chunked encoding. However, we ignore them both.
+    @obj_opts = {
+      :ignore_content_length => true, 
+      :ignore_chunked_encoding => true
+    }
+    @obj = @obj_klass.new(nil, @obj_opts)
+
+    # Note, our test-case includes both content length and chunked encoding.
+    # A real server probably wouldn't do this, but we want to make sure
+    # we handle both.
+    @rawdat =<<_EOF_
+HTTP/1.1 200 OK
+Cache-Control: private, max-age=0
+Date: Fri, 27 Mar 2009 04:27:27 GMT
+Expires: -1
+Content-Length: 9140
+Content-Type: text/html; charset=ISO-8859-1
+Server: Booble
+Transfer-Encoding: chunked
+
+_EOF_
+
+    @hstr, @body = @rawdat.split(/^\n/, 2)
+    @rawdat_crlf = @hstr.gsub("\n", "\r\n") + "\r\n" + @body
+
+    @code = 200
+    @text = "OK"
+    @version = "HTTP/1.1"
+
+    @headers = [
+      ["Cache-Control", "private, max-age=0"], 
+      ["Date", "Fri, 27 Mar 2009 04:27:27 GMT"], 
+      ["Expires", "-1"], 
+      ["Content-Length", "9140"], 
+      ["Content-Type", "text/html; charset=ISO-8859-1"], 
+      ["Server", "Booble"], 
+      ["Transfer-Encoding", "chunked"]
+    ]
+
+    # Content length should get ignored
+    @content_length = nil
+  end
+
+  def test_capture_crlf_headers
+    @obj.capture(@rawdat_crlf)
+    do_capture_value_tests(@obj)
+    do_type_tests(@obj)
+    assert_equal @rawdat_crlf, @obj.to_raw
+  end
+
+  def test_captured_body_type
+    @obj.capture(@rawdat)
+    assert_kind_of Body, @obj.body
+  end
+
+end
+