mechanize 2.0.pre.1 → 2.0.pre.2

data/lib/mechanize/form.rb

@@ -29,8 +29,17 @@ class Mechanize
     attr_accessor :method, :action, :name
 
     attr_reader :fields, :buttons, :file_uploads, :radiobuttons, :checkboxes
+
+    # Content-Type for form data (i.e. application/x-www-form-urlencoded)
     attr_accessor :enctype
 
+    # Character encoding of form data (i.e. UTF-8)
+    attr_accessor :encoding
+
+    # When true, character encoding errors will never be never raised on form
+    # submission.  Default is false
+    attr_accessor :ignore_encoding_error
+
     alias :elements :fields
 
     attr_reader :form_node
@@ -46,6 +55,8 @@ class Mechanize
       @page             = page
       @mech             = mech
 
+      @encoding = node['accept-charset'] || (page && page.encoding) || nil
+      @ignore_encoding_error = false
       parse
     end
 
@@ -173,7 +184,7 @@ class Mechanize
     private :proc_query
 
     def from_native_charset str
-      Util.from_native_charset(str,page && page.encoding)
+      Util.from_native_charset(str, encoding, @ignore_encoding_error, @mech && @mech.log)
     end
     private :from_native_charset
 
@@ -182,6 +193,7 @@ class Mechanize
     # be used to create a query string for this form.
     def build_query(buttons = [])
       query = []
+      @mech.log.info("form encoding: #{encoding}") if @mech && @mech.log
 
       (fields + checkboxes).sort.each do |f|
         case f
@@ -395,7 +407,7 @@ class Mechanize
       # Find all textarea tags
       form_node.search('textarea').each do |node|
         next unless node['name']
-        @fields << Field.new(node, node.inner_text)
+        @fields << Textarea.new(node, node.inner_text)
       end
 
       # Find all select tags

data/lib/mechanize/page.rb

@@ -1,34 +1,34 @@
-# = Synopsis
+##
 # This class encapsulates an HTML page.  If Mechanize finds a content
 # type of 'text/html', this class will be instantiated and returned.
 #
-# == Example
-#  require 'rubygems'
-#  require 'mechanize'
+# Example:
 #
-#  agent = Mechanize.new
-#  agent.get('http://google.com/').class  #=> Mechanize::Page
+#   require 'mechanize'
 #
+#   agent = Mechanize.new
+#   agent.get('http://google.com/').class # => Mechanize::Page
+
 class Mechanize::Page < Mechanize::File
   extend Forwardable
   extend Mechanize::ElementMatcher
 
   attr_accessor :mech
 
+  ##
+  # Possible encodings for this page based on HTTP headers and meta elements
+
+  attr_reader :encodings
+
   def initialize(uri=nil, response=nil, body=nil, code=nil, mech=nil)
     raise Mechanize::ContentTypeError, response['content-type'] unless
       response['content-type'] =~ /^(text\/html)|(application\/xhtml\+xml)/i
 
-    @bases = nil
     @encoding = nil
     @encodings = [nil]
-    @forms = nil
-    @frames = nil
-    @iframes = nil
-    @links = nil
     @mech = mech
-    @meta = nil
-    @parser = nil
+
+    reset
 
     @encodings << Mechanize::Util.detect_charset(body) if body
 
@@ -71,6 +71,8 @@ class Mechanize::Page < Mechanize::File
   end
 
   def encoding=(encoding)
+    reset
+
     @encoding = encoding
 
     if @parser
@@ -101,7 +103,9 @@ class Mechanize::Page < Mechanize::File
         break if @parser.errors.empty?
 
         break unless @parser.errors.any? do |error|
-          error.message =~ /(indicate encoding)|(Invalid char)/
+          error.message =~ /(indicate\ encoding)|
+                            (Invalid\ char)|
+                            (input\ conversion failed)/x
         end
       end
     end
@@ -111,6 +115,31 @@ class Mechanize::Page < Mechanize::File
 
   alias :root :parser
 
+  def reset
+    @bases = nil
+    @forms = nil
+    @frames = nil
+    @iframes = nil
+    @links = nil
+    @labels = nil
+    @labels_hash = nil
+    @meta = nil
+    @parser = nil
+    @title = nil
+  end
+
+  # Return the canonical URI for the page if there is a link tag
+  # with href="canonical".
+  def canonical_uri
+    link = at('link[@rel="canonical"][@href]')
+    return unless link
+    href = link['href']
+
+    URI href
+  rescue URI::InvalidURIError
+    URI Mechanize::Util.uri_escape href
+  end
+
   # Get the content type
   def content_type
     response['content-type']

data/lib/mechanize/page/link.rb

@@ -39,6 +39,16 @@ class Mechanize
         @href && URI.parse(WEBrick::HTTPUtils.escape(@href))
       end
 
+      # A list of words in the rel attribute, all lower-cased.
+      def rel
+        @rel ||= (val = attributes['rel']) ? val.downcase.split(' ') : []
+      end
+
+      # Test if the rel attribute includes +kind+.
+      def rel?(kind)
+        rel.include?(kind)
+      end
+
       # Click on this link
       def click
         @mech.click self

data/lib/mechanize/redirect_not_get_or_head_error.rb

@@ -11,7 +11,8 @@ class Mechanize
     end
 
     def to_s
-      "#{@response_code} redirect received after a #{@verb} request"
+      method = @verb.to_s.upcase
+      "#{@response_code} redirect received after a #{method} request"
     end
     alias :inspect :to_s
   end

data/lib/mechanize/robots_disallowed_error.rb

@@ -0,0 +1,29 @@
+class Mechanize
+  # Exception that is raised when an access to a resource is
+  # disallowed by robots.txt or by HTML document itself.
+  class RobotsDisallowedError < Mechanize::Error
+    def initialize(url)
+      if url.is_a?(URI)
+        @url = url.to_s
+        @uri = url
+      else
+        @url = url.to_s
+      end
+    end
+
+    # Returns the URL (string) of the resource that caused this error.
+    attr_reader :url
+
+    # Returns the URL (URI object) of the resource that caused this
+    # error.  URI::InvalidURIError may be raised if the URL happens to
+    # be invalid or not understood by the URI library.
+    def uri
+      @uri ||= URI.parse(url)
+    end
+
+    def to_s
+      "Robots access is disallowed for URL: #{url}"
+    end
+    alias :inspect :to_s
+  end
+end

data/lib/mechanize/util.rb

@@ -7,6 +7,16 @@ class Mechanize::Util
     :SJIS => "SHIFT_JIS",
     :UTF8 => "UTF-8", :UTF16 => "UTF-16", :UTF32 => "UTF-32"}
 
+  # true if RUBY_VERSION is 1.9.0 or later
+  NEW_RUBY_ENCODING = RUBY_VERSION >= '1.9.0'
+
+  # contains encoding error classes to raise
+  ENCODING_ERRORS = if NEW_RUBY_ENCODING
+                      [EncodingError]
+                    else
+                      [Iconv::InvalidEncoding, Iconv::IllegalSequence]
+                    end
+
   def self.build_query_string(parameters, enc=nil)
     parameters.map { |k,v|
       # WEBrick::HTTP.escape* has some problems about m17n on ruby-1.9.*.
@@ -15,6 +25,8 @@ class Mechanize::Util
   end
 
   def self.to_native_charset(s, code=nil)
+    location = Gem.location_of_caller.join ':'
+    warn "#{location}: Mechanize::Util::to_native_charset is deprecated and will be removed October 2011"
     if Mechanize.html_parser == Nokogiri::HTML
       return unless s
       code ||= detect_charset(s)
@@ -24,20 +36,32 @@ class Mechanize::Util
     end
   end
 
-  def self.from_native_charset(s, code)
+  # Converts string +s+ from +code+ to UTF-8.
+  def self.from_native_charset(s, code, ignore_encoding_error=false, log=nil)
     return s unless s && code
     return s unless Mechanize.html_parser == Nokogiri::HTML
 
-    if RUBY_VERSION < '1.9.2'
-      begin
-        Iconv.iconv(code.to_s, "UTF-8", s).join("")
-      rescue Iconv::InvalidEncoding, Iconv::IllegalSequence
+    begin
+      encode_to(code, s)
+    rescue *ENCODING_ERRORS => ex
+      log.debug("from_native_charset: #{ex.class}: form encoding: #{code.inspect} string: #{s}") if log
+      if ignore_encoding_error
         s
+      else
+        raise
       end
+    end
+  end
+
+  # inner convert method of Util.from_native_charset
+  def self.encode_to(encoding, str)
+    if NEW_RUBY_ENCODING
+      str.encode(encoding)
     else
-      s.encode("UTF-8") rescue s
+      Iconv.conv(encoding.to_s, "UTF-8", str)
     end
   end
+  private_class_method :encode_to
 
   def self.html_unescape(s)
     return s unless s

data/test/helper.rb

@@ -94,6 +94,12 @@ class Net::HTTP
       end
 
       res.body = PAGE_CACHE[filename]
+      case filename
+      when /\.txt$/
+        res['Content-Type'] = 'text/plain'
+      when /\.jpg$/
+        res['Content-Type'] = 'image/jpeg'
+      end
     end
 
     res['Content-Type'] ||= 'text/html'

data/test/htdocs/canonical_uri.html

@@ -0,0 +1,9 @@
+<html>
+  <head>
+    <link rel="start" title="site top" href="/" />
+    <link rel="canonical" href="http://localhost/canonical_uri" />
+  </head>
+  <body>
+    test
+  </body>
+</html>

data/test/htdocs/nofollow.html

@@ -0,0 +1,9 @@
+<html>
+  <head>
+    <title>nofollow test</title>
+    <meta name="ROBOTS" content="nofollow">
+  </head>
+  <body>
+    <p>Do not follow <a href="/robots.html">this</a> or <a href="/">this</a>!</p>
+  </body>
+</html>

data/test/htdocs/noindex.html

@@ -0,0 +1,9 @@
+<html>
+  <head>
+    <title>noindex test</title>
+    <meta name="ROBOTS" content="noarchive,noindex">
+  </head>
+  <body>
+    <p>Do not index nor archive this page!</p>
+  </body>
+</html>

data/test/htdocs/norobots.html

@@ -0,0 +1,8 @@
+<html>
+  <head>
+    <title>No Robots!</title>
+  </head>
+  <body>
+    <p>Go home, robot!</p>
+  </body>
+</html>

data/test/htdocs/rel_nofollow.html

@@ -0,0 +1,8 @@
+<html>
+  <head>
+    <title>rel_nofollow test</title>
+  </head>
+  <body>
+    <p>You can follow <a href="/robots.html">this link</a> but not <a href="/" rel="me nofollow">this</a>!</p>
+  </body>
+</html>

data/test/htdocs/robots.html

@@ -0,0 +1,8 @@
+<html>
+  <head>
+    <title>Welcome!</title>
+  </head>
+  <body>
+    <p>Welcome, robot!</p>
+  </body>
+</html>

data/test/htdocs/robots.txt

@@ -0,0 +1,2 @@
+User-Agent: *
+Disallow: /norobots

data/test/htdocs/tc_links.html

@@ -1,9 +1,9 @@
 <html>
   <body>
     <a href="thing.html"><b>Bold Dude</b></a>
-    <a href="thing.html">Dude</a>
-    <a href="thing.html">Aaron <b>James</b> Patterson</a>
-    <a href="thing.html" id="bold_aaron_link"><b>Aaron</b> Patterson</a>
+    <a href="thing.html" rel="nofollow">Dude</a>
+    <a href="thing.html" rel="me">Aaron <b>James</b> Patterson</a>
+    <a href="thing.html" id="bold_aaron_link" rel="me nofollow"><b>Aaron</b> Patterson</a>
     <a href="thing.html">Ruby <b>Rocks!</b></a>
     <!-- Testing a bug with escaped stuff in links:
       http://rubyforge.org/pipermail/mechanize-users/2006-September/000002.html

data/test/test_links.rb

@@ -94,14 +94,23 @@ class LinksMechTest < Test::Unit::TestCase
     link = page.links_with(:text => /Bold Dude/)
     assert_equal(1, link.length)
     assert_equal('Bold Dude', link.first.text)
+    assert_equal [], link.first.rel
+    assert !link.first.rel?('me')
+    assert !link.first.rel?('nofollow')
 
     link = page.links_with(:text => 'Aaron James Patterson')
     assert_equal(1, link.length)
     assert_equal('Aaron James Patterson', link.first.text)
+    assert_equal ['me'], link.first.rel
+    assert link.first.rel?('me')
+    assert !link.first.rel?('nofollow')
 
     link = page.links_with(:text => 'Aaron Patterson')
     assert_equal(1, link.length)
     assert_equal('Aaron Patterson', link.first.text)
+    assert_equal ['me', 'nofollow'], link.first.rel
+    assert link.first.rel?('me')
+    assert link.first.rel?('nofollow')
 
     link = page.links_with(:text => 'Ruby Rocks!')
     assert_equal(1, link.length)

data/test/test_mechanize.rb

@@ -32,6 +32,31 @@ class TestMechanize < Test::Unit::TestCase
                end
   end
 
+  def test_back
+    0.upto(5) do |i|
+      assert_equal(i, @agent.history.size)
+      @agent.get("http://localhost/")
+    end
+    @agent.get("http://localhost/form_test.html")
+
+    assert_equal("http://localhost/form_test.html",
+      @agent.history.last.uri.to_s)
+    assert_equal("http://localhost/",
+      @agent.history[-2].uri.to_s)
+
+    assert_equal(7, @agent.history.size)
+    @agent.back
+    assert_equal(6, @agent.history.size)
+    assert_equal("http://localhost/",
+      @agent.history.last.uri.to_s)
+  end
+
+  def test_basic_auth
+    @agent.basic_auth('user', 'pass')
+    page = @agent.get("http://localhost/basic_auth")
+    assert_equal('You are authenticated', page.body)
+  end
+
   def test_cert_key_file
     Tempfile.open 'key' do |key|
       Tempfile.open 'cert' do |cert|
@@ -61,6 +86,80 @@ class TestMechanize < Test::Unit::TestCase
     assert_equal CERT, agent.http.certificate
   end
 
+  def test_click
+    @agent.user_agent_alias = 'Mac Safari'
+    page = @agent.get("http://localhost/frame_test.html")
+    link = page.link_with(:text => "Form Test")
+    assert_not_nil(link)
+    page = @agent.click(link)
+    assert_equal("http://localhost/form_test.html",
+      @agent.history.last.uri.to_s)
+  end
+
+  def test_click_frame_hpricot_style
+    page = @agent.get("http://localhost/frame_test.html")
+
+    link = (page/"//frame[@name='frame2']").first
+    assert_not_nil(link)
+    page = @agent.click(link)
+    assert_equal("http://localhost/form_test.html",
+      @agent.history.last.uri.to_s)
+  end
+
+  def test_click_hpricot_style # HACK move to test_divide in Page
+    page = @agent.get("http://localhost/frame_test.html")
+
+    link = (page/"//a[@class='bar']").first
+    assert_not_nil(link)
+
+    page = @agent.click(link)
+
+    assert_equal("http://localhost/form_test.html",
+      @agent.history.last.uri.to_s)
+  end
+
+  def test_click_link_hpricot_style # HACK move to test_search in Page
+    page = @agent.get("http://localhost/tc_encoded_links.html")
+
+    page = @agent.click(page.search('a').first)
+
+    assert_equal("http://localhost/form_post?a=b&b=c", page.uri.to_s)
+  end
+
+  def test_click_link_query
+    page = @agent.get("http://localhost/tc_encoded_links.html")
+    link = page.links.first
+    assert_equal('/form_post?a=b&b=c', link.href)
+
+    page = @agent.click(link)
+
+    assert_equal("http://localhost/form_post?a=b&b=c", page.uri.to_s)
+  end
+
+  def test_click_link_space
+    page = @agent.get("http://localhost/tc_bad_links.html")
+
+    @agent.click page.links.first
+
+    assert_match(/alt_text.html$/, @agent.history.last.uri.to_s)
+    assert_equal(2, @agent.history.length)
+  end
+
+  def test_click_more
+    @agent.get 'http://localhost/test_click.html'
+    @agent.click 'A Button'
+    assert_equal 'http://localhost/frame_test.html?words=nil',
+      @agent.page.uri.to_s
+    @agent.back
+    @agent.click 'A Link'
+    assert_equal 'http://localhost/index.html',
+      @agent.page.uri.to_s
+    @agent.back
+    @agent.click @agent.page.link_with(:text => 'A Link')
+    assert_equal 'http://localhost/index.html',
+      @agent.page.uri.to_s
+  end
+
   def test_connection_for_file
     uri = URI.parse 'file:///nonexistent'
     conn = @agent.connection_for uri
@@ -74,6 +173,14 @@ class TestMechanize < Test::Unit::TestCase
     assert_equal @agent.http, conn
   end
 
+  def test_delete_redirect
+    page = @agent.delete('http://localhost/redirect')
+
+    assert_equal(page.uri.to_s, 'http://localhost/verb')
+
+    assert_equal 'GET', page.header['X-Request-Method']
+  end
+
   #def test_download
   #  Dir.mktmpdir do |dir|
   #    file = "#{dir}/download"
@@ -129,12 +236,293 @@ class TestMechanize < Test::Unit::TestCase
     uri = URI.parse 'file:///nonexistent'
 
     e = assert_raises Mechanize::ResponseCodeError do
-      page = @agent.send :fetch_page, uri
+      @agent.send :fetch_page, uri
     end
 
     assert_equal '404 => Net::HTTPNotFound', e.message
   end
 
+  def test_fetch_page_post_connect_hook
+    response = nil
+    @agent.post_connect_hooks << lambda { |_, res|
+      response = res
+    }
+
+    @agent.get('http://localhost/')
+    assert(response)
+  end
+
+  def test_get
+    page = @agent.get('http://localhost', { :q => 'h' }, 'http://example',
+                      { 'X-H' => 'v' })
+
+    assert_equal 'http://localhost/?q=h', page.uri.to_s
+  end
+
+  def test_get_HTTP
+    page = @agent.get('HTTP://localhost/', { :q => 'hello' })
+    assert_equal('HTTP://localhost/?q=hello', page.uri.to_s)
+  end
+
+  def test_get_anchor
+    page = @agent.get('http://localhost/?foo=bar&#34;')
+    assert_equal('http://localhost/?foo=bar%22', page.uri.to_s)
+  end
+
+  def test_get_bad_url
+    assert_raise ArgumentError do
+      @agent.get('/foo.html')
+    end
+  end
+
+  def test_get_basic_auth_bad
+    @agent.basic_auth('aaron', 'aaron')
+
+    e = assert_raises Mechanize::ResponseCodeError do
+      @agent.get("http://localhost/basic_auth")
+    end
+
+    assert_equal("401", e.response_code)
+  end
+
+  def test_get_basic_auth_none
+    e = assert_raises Mechanize::ResponseCodeError do
+      @agent.get("http://localhost/basic_auth")
+    end
+
+    assert_equal("401", e.response_code)
+  end
+
+  def test_get_digest_auth
+    block_called = false
+
+    @agent.basic_auth('user', 'pass')
+
+    @agent.pre_connect_hooks << lambda { |_, request|
+      block_called = true
+      request.to_hash.each do |k,v|
+        assert_equal(1, v.length)
+      end
+    }
+
+    page = @agent.get("http://localhost/digest_auth")
+    assert_equal('You are authenticated', page.body)
+    assert block_called
+  end
+
+  def test_get_file
+    page = @agent.get("http://localhost/frame_test.html")
+    content_length = page.header['Content-Length']
+    page_as_string = @agent.get_file("http://localhost/frame_test.html")
+    assert_equal(content_length.to_i, page_as_string.length.to_i)
+  end
+
+  def test_get_follow_meta_refresh
+    @agent.follow_meta_refresh = true
+
+    page = @agent.get('http://localhost/tc_follow_meta.html')
+
+    assert_equal(2, @agent.history.length)
+
+    assert_equal('http://localhost/tc_follow_meta.html',
+                 @agent.history.first.uri.to_s)
+    assert_equal('http://localhost/index.html', page.uri.to_s)
+    assert_equal('http://localhost/index.html', @agent.history.last.uri.to_s)
+  end
+
+  def test_get_follow_meta_refresh_disabled
+    page = @agent.get('http://localhost/tc_follow_meta.html')
+    assert_equal('http://localhost/tc_follow_meta.html', page.uri.to_s)
+    assert_equal(1, page.meta.length)
+  end
+
+  def test_get_follow_meta_refresh_empty_url
+    @agent.follow_meta_refresh = true
+
+    page = @agent.get('http://localhost/refresh_with_empty_url')
+
+    assert_equal(3, @agent.history.length)
+    assert_equal('http://localhost/refresh_with_empty_url',
+                 @agent.history[0].uri.to_s)
+    assert_equal('http://localhost/refresh_with_empty_url',
+                 @agent.history[1].uri.to_s)
+    assert_equal('http://localhost/index.html', page.uri.to_s)
+    assert_equal('http://localhost/index.html', @agent.history.last.uri.to_s)
+  end
+
+  def test_get_follow_meta_refresh_in_body
+    @agent.follow_meta_refresh = true
+    requests = []
+    @agent.pre_connect_hooks << lambda { |_, request|
+      requests << request
+    }
+
+    @agent.get('http://localhost/tc_meta_in_body.html')
+    assert_equal 1, requests.length
+  end
+
+  def test_get_follow_meta_refresh_no_url
+    @agent.follow_meta_refresh = true
+
+    page = @agent.get('http://localhost/refresh_without_url')
+
+    assert_equal(3, @agent.history.length)
+    assert_equal('http://localhost/refresh_without_url',
+                 @agent.history[0].uri.to_s)
+    assert_equal('http://localhost/refresh_without_url',
+                 @agent.history[1].uri.to_s)
+    assert_equal('http://localhost/index.html', page.uri.to_s)
+    assert_equal('http://localhost/index.html', @agent.history.last.uri.to_s)
+  end
+
+  def test_get_follow_meta_refresh_referer_not_sent
+    @agent.follow_meta_refresh = true
+
+    requests = []
+
+    @agent.pre_connect_hooks << lambda { |_, request|
+      requests << request
+    }
+
+    @agent.get('http://localhost/tc_follow_meta.html')
+
+    assert_equal 2, @agent.history.length
+    assert_nil requests.last['referer']
+  end
+
+  def test_get_gzip
+    page = @agent.get("http://localhost/gzip?file=index.html")
+
+    assert_kind_of(Mechanize::Page, page)
+
+    assert_match('Hello World', page.body)
+  end
+
+  def test_get_http_refresh
+    @agent.follow_meta_refresh = true
+    page = @agent.get('http://localhost/http_refresh?refresh_time=0')
+    assert_equal('http://localhost/index.html', page.uri.to_s)
+    assert_equal(2, @agent.history.length)
+  end
+
+  def test_get_http_refresh_delay
+    @agent.follow_meta_refresh = true
+    class << @agent
+      attr_accessor :slept
+      def sleep *args
+        @slept = args
+      end
+    end
+
+    @agent.get('http://localhost/http_refresh?refresh_time=1')
+    assert_equal [1], @agent.slept
+  end
+
+  def test_get_http_refresh_disabled
+    page = @agent.get('http://localhost/http_refresh?refresh_time=0')
+    assert_equal('http://localhost/http_refresh?refresh_time=0', page.uri.to_s)
+  end
+
+  def test_get_kcode
+    $KCODE = 'u'
+    page = @agent.get("http://localhost/?a=#{[0xd6].pack('U')}")
+    assert_not_nil(page)
+    assert_equal('http://localhost/?a=%D6', page.uri.to_s)
+    $KCODE = 'NONE'
+  end unless RUBY_VERSION >= '1.9.0'
+
+  def test_get_query
+    page = @agent.get('http://localhost/', { :q => 'hello' })
+    assert_equal('http://localhost/?q=hello', page.uri.to_s)
+  end
+
+  def test_get_redirect
+    page = @agent.get('http://localhost/redirect')
+
+    assert_equal(page.uri.to_s, 'http://localhost/verb')
+
+    assert_equal 'GET', page.header['X-Request-Method']
+  end
+
+  def test_get_redirect_found
+    page = @agent.get('http://localhost/response_code?code=302&ct=test/xml')
+
+    assert_equal('http://localhost/index.html', page.uri.to_s)
+
+    assert_equal(2, @agent.history.length)
+  end
+
+  def test_get_redirect_infinite
+    assert_raises(Mechanize::RedirectLimitReachedError) {
+      @agent.get('http://localhost/infinite_refresh')
+    }
+  end
+
+  def test_get_referer
+    request = nil
+    @agent.pre_connect_hooks << lambda { |_, req|
+      request = req
+    }
+
+    @agent.get('http://localhost/', [], 'http://tenderlovemaking.com/')
+    assert_equal 'http://tenderlovemaking.com/', request['Referer']
+  end
+
+  def test_get_referer_file
+    assert_nothing_raised do
+      @agent.get('http://localhost', [], Mechanize::File.new(URI.parse('http://tenderlovemaking.com/crossdomain.xml')))
+    end
+
+    # HACK no assertion of behavior
+  end
+
+  def test_get_referer_none
+    requests = []
+    @agent.pre_connect_hooks << lambda { |_, request|
+      requests << request
+    }
+
+    @agent.get('http://localhost/')
+    @agent.get('http://localhost/')
+    assert_equal(2, requests.length)
+    requests.each do |request|
+      assert_nil request['referer']
+    end
+  end
+
+  def test_get_scheme_unsupported
+    assert_raise(Mechanize::UnsupportedSchemeError) {
+      @agent.get('ftp://server.com/foo.html')
+    }
+  end
+
+  def test_get_space
+    page = nil
+
+    page = @agent.get("http://localhost/tc_bad_links.html ")
+
+    assert_match(/tc_bad_links.html$/, @agent.history.last.uri.to_s)
+
+    assert_equal(1, @agent.history.length)
+  end
+
+  def test_get_tilde
+    page = @agent.get('http://localhost/?foo=~2')
+    assert_equal('http://localhost/?foo=~2', page.uri.to_s)
+  end
+
+  def test_get_weird
+    assert_nothing_raised {
+      @agent.get('http://localhost/?action=bing&bang=boom=1|a=|b=|c=')
+    }
+    assert_nothing_raised {
+      @agent.get('http://localhost/?a=b&#038;b=c&#038;c=d')
+    }
+    assert_nothing_raised {
+      @agent.get("http://localhost/?a=#{[0xd6].pack('U')}")
+    }
+  end
+
   def test_get_yield
     pages = nil
 
@@ -146,6 +534,59 @@ class TestMechanize < Test::Unit::TestCase
     assert_equal('File Upload Form', pages.title)
   end
 
+  def test_head_redirect
+    page = @agent.head('http://localhost/redirect')
+
+    assert_equal(page.uri.to_s, 'http://localhost/verb')
+
+    assert_equal 'HEAD', page.header['X-Request-Method']
+  end
+
+  def test_history
+    0.upto(25) do |i|
+      assert_equal(i, @agent.history.size)
+      @agent.get("http://localhost/")
+    end
+    page = @agent.get("http://localhost/form_test.html")
+
+    assert_equal("http://localhost/form_test.html",
+      @agent.history.last.uri.to_s)
+    assert_equal("http://localhost/",
+      @agent.history[-2].uri.to_s)
+    assert_equal("http://localhost/",
+      @agent.history[-2].uri.to_s)
+
+    assert_equal(true, @agent.visited?("http://localhost/"))
+    assert_equal(true, @agent.visited?("/form_test.html"))
+    assert_equal(false, @agent.visited?("http://google.com/"))
+    assert_equal(true, @agent.visited?(page.links.first))
+
+  end
+
+  def test_history_order
+    @agent.max_history = 2
+    assert_equal(0, @agent.history.length)
+
+    @agent.get('http://localhost/form_test.html')
+    assert_equal(1, @agent.history.length)
+
+    @agent.get('http://localhost/empty_form.html')
+    assert_equal(2, @agent.history.length)
+
+    @agent.get('http://localhost/tc_checkboxes.html')
+    assert_equal(2, @agent.history.length)
+    assert_equal('http://localhost/empty_form.html', @agent.history[0].uri.to_s)
+    assert_equal('http://localhost/tc_checkboxes.html',
+                 @agent.history[1].uri.to_s)
+  end
+
+  def test_html_parser_equals
+    @agent.html_parser = {}
+    assert_raises(NoMethodError) {
+      @agent.get('http://localhost/?foo=~2').links
+    }
+  end
+
   def test_http_request_file
     uri = URI.parse 'file:///nonexistent'
     request = @agent.http_request uri, :get
@@ -168,6 +609,47 @@ class TestMechanize < Test::Unit::TestCase
     assert_equal '/', request.path
   end
 
+  def test_max_history_equals
+    @agent.max_history = 10
+    0.upto(10) do |i|
+      assert_equal(i, @agent.history.size)
+      @agent.get("http://localhost/")
+    end
+
+    0.upto(10) do |i|
+      assert_equal(10, @agent.history.size)
+      @agent.get("http://localhost/")
+    end
+  end
+
+  def test_post_basic_auth
+    class << @agent
+      alias :old_fetch_page :fetch_page
+      attr_accessor :requests
+      def fetch_page(uri, method, *args)
+        @requests ||= []
+        x = old_fetch_page(uri, method, *args)
+        @requests << method
+        x
+      end
+    end
+    @agent.basic_auth('user', 'pass')
+    page = @agent.post("http://localhost/basic_auth")
+    assert_equal('You are authenticated', page.body)
+    assert_equal(2, @agent.requests.length)
+    r1 = @agent.requests[0]
+    r2 = @agent.requests[1]
+    assert_equal(r1, r2)
+  end
+
+  def test_post_redirect
+    page = @agent.post('http://localhost/redirect')
+
+    assert_equal(page.uri.to_s, 'http://localhost/verb')
+
+    assert_equal 'GET', page.header['X-Request-Method']
+  end
+
   def test_post_connect
     @agent.post_connect_hooks << proc { |agent, response|
       assert_equal @agent, agent
@@ -192,6 +674,14 @@ class TestMechanize < Test::Unit::TestCase
     end
   end
 
+  def test_put_redirect
+    page = @agent.put('http://localhost/redirect', 'foo')
+
+    assert_equal(page.uri.to_s, 'http://localhost/verb')
+
+    assert_equal 'GET', page.header['X-Request-Method']
+  end
+
   def test_request_cookies
     uri = URI.parse 'http://host.example.com'
     Mechanize::Cookie.parse uri, 'hello=world domain=.example.com' do |cookie|
@@ -349,6 +839,22 @@ class TestMechanize < Test::Unit::TestCase
     assert_match %r%Ruby/#{ruby_version}%, @req['user-agent']
   end
 
+  def test_resolve_bad_uri
+    e = assert_raises ArgumentError do
+      @agent.resolve 'google'
+    end
+
+    assert_equal 'absolute URL needed (not google)', e.message
+  end
+
+  def test_resolve_utf8
+    uri = 'http://example?q=ü'
+
+    resolved = @agent.resolve uri
+
+    assert_equal '/?q=%C3%BC', resolved.request_uri
+  end
+
   def test_resolve_parameters_body
     input_params = { :q => 'hello' }
 
@@ -507,7 +1013,7 @@ class TestMechanize < Test::Unit::TestCase
   end
 
   def test_response_read_encoding_gzip
-    def @res.read_body()
+    def @res.read_body
       yield "\037\213\b\0002\002\225M\000\003"
       yield "+H,*\001\000\306p\017I\004\000\000\000"
     end
@@ -683,5 +1189,114 @@ class TestMechanize < Test::Unit::TestCase
     assert_equal 'UTF-8', page.encoding
   end
 
+  def test_set_proxy
+    @agent.set_proxy('www.example.com', 9001, 'joe', 'lol')
+
+    assert_equal(@agent.http.proxy_uri.host,     'www.example.com')
+    assert_equal(@agent.http.proxy_uri.port,     9001)
+    assert_equal(@agent.http.proxy_uri.user,     'joe')
+    assert_equal(@agent.http.proxy_uri.password, 'lol')
+  end
+
+  def test_submit_bad_form_method
+    page = @agent.get("http://localhost/bad_form_test.html")
+    assert_raise ArgumentError do
+      @agent.submit(page.forms.first)
+    end
+  end
+
+  def test_submit_check_one
+    page = @agent.get('http://localhost/tc_checkboxes.html')
+    form = page.forms.first
+    form.checkboxes_with(:name => 'green')[1].check
+
+    page = @agent.submit(form)
+
+    assert_equal(1, page.links.length)
+    assert_equal('green:on', page.links.first.text)
+  end
+
+  def test_submit_check_two
+    page = @agent.get('http://localhost/tc_checkboxes.html')
+    form = page.forms.first
+    form.checkboxes_with(:name => 'green')[0].check
+    form.checkboxes_with(:name => 'green')[1].check
+
+    page = @agent.submit(form)
+
+    assert_equal(2, page.links.length)
+    assert_equal('green:on', page.links[0].text)
+    assert_equal('green:on', page.links[1].text)
+  end
+
+  def test_submit_headers
+    page = @agent.get('http://localhost:2000/form_no_action.html')
+    assert form = page.forms.first
+    form.action = '/http_headers'
+    page = @agent.submit(form, nil, { 'foo' => 'bar' })
+    headers = Hash[*(
+      page.body.split("\n").map { |x| x.split('|') }.flatten
+    )]
+    assert_equal 'bar', headers['foo']
+  end
+
+  def test_submit_too_many_radiobuttons
+    page = @agent.get("http://localhost/form_test.html")
+    form = page.form_with(:name => 'post_form1')
+    form.radiobuttons.each { |r| r.checked = true }
+
+    assert_raises Mechanize::Error do
+      @agent.submit(form)
+    end
+  end
+
+  def test_transact
+    @agent.get("http://localhost/frame_test.html")
+    assert_equal(1, @agent.history.length)
+    @agent.transact { |a|
+      5.times {
+        @agent.get("http://localhost/frame_test.html")
+      }
+      assert_equal(6, @agent.history.length)
+    }
+    assert_equal(1, @agent.history.length)
+  end
+
+  def test_user_agent_alias_equals_unknown
+    assert_raises ArgumentError do
+      @agent.user_agent_alias = "Aaron's Browser"
+    end
+  end
+
+  def test_visited_eh
+    @agent.get("http://localhost/content_type_test?ct=application/pdf")
+    assert_equal(true,
+      @agent.visited?("http://localhost/content_type_test?ct=application/pdf"))
+    assert_equal(false,
+      @agent.visited?("http://localhost/content_type_test"))
+    assert_equal(false,
+      @agent.visited?("http://localhost/content_type_test?ct=text/html"))
+  end
+
+  def test_visited_eh_redirect
+    @agent.get("http://localhost/response_code?code=302")
+    assert_equal("http://localhost/index.html",
+      @agent.current_page.uri.to_s)
+    assert_equal(true,
+                 @agent.visited?('http://localhost/response_code?code=302'))
+  end
+
+  def assert_header(page, header)
+    headers = {}
+
+    page.body.split(/[\r\n]+/).each do |page_header|
+      headers.[]=(*page_header.chomp.split(/\|/))
+    end
+
+    header.each do |key, value|
+      assert(headers.has_key?(key))
+      assert_equal(value, headers[key])
+    end
+  end
 end