mechanize 2.0.1 → 2.1.pre.1

data/test/test_mechanize_page_encoding.rb

@@ -1,15 +1,15 @@
 # -*- coding: utf-8 -*-
-require 'helper'
-require 'cgi'
+require 'mechanize/test_case'
 
 # tests for Page encoding and charset and parsing
 
-class TestMechanizePageEncoding < MiniTest::Unit::TestCase
+class TestMechanizePageEncoding < Mechanize::TestCase
 
   MECH_ASCII_ENCODING = Mechanize::Util::NEW_RUBY_ENCODING ? 'US-ASCII' : 'ISO-8859-1'
 
   def setup
-    @agent = Mechanize.new
+    super
+
     @uri = URI('http://localhost/')
     @response_headers = { 'content-type' => 'text/html' }
     @body = '<title>hi</title>'
@@ -17,7 +17,7 @@ class TestMechanizePageEncoding < MiniTest::Unit::TestCase
 
   def util_page body = @body, headers = @response_headers
     body.force_encoding Encoding::BINARY if body.respond_to? :force_encoding
-    Mechanize::Page.new @uri, headers, body, 200, @agent
+    Mechanize::Page.new @uri, headers, body, 200, @mech
   end
 
   def test_page_charset
@@ -30,7 +30,7 @@ class TestMechanizePageEncoding < MiniTest::Unit::TestCase
     assert_equal 'UTF-8', charset
   end
 
-  def test_page_semicolon
+  def test_page_charset_semicolon
     charset = Mechanize::Page.charset 'text/html;charset=UTF-8;'
     assert_equal 'UTF-8', charset
   end
@@ -53,7 +53,7 @@ class TestMechanizePageEncoding < MiniTest::Unit::TestCase
   end
 
   def test_page_response_header_charset
-    headers = {'content-type' => 'text/html;charset=HEADER'}
+    headers = { 'content-type' => 'text/html;charset=HEADER' }
     charsets = Mechanize::Page.response_header_charset(headers)
 
     assert_equal ['HEADER'], charsets
@@ -71,6 +71,13 @@ class TestMechanizePageEncoding < MiniTest::Unit::TestCase
     assert_equal [], charsets
   end
 
+  def test_page_response_header_charset_wrong_header
+    headers = { 'x-content-type' => 'text/html;charset=bogus' }
+    charsets = Mechanize::Page.response_header_charset(headers)
+
+    assert_equal [], charsets
+  end
+
   def test_response_header_charset
     page = util_page nil, {'content-type' => 'text/html;charset=HEADER'}
 
@@ -90,6 +97,21 @@ class TestMechanizePageEncoding < MiniTest::Unit::TestCase
     assert_equal [], charsets
   end
 
+  def test_page_meta_charset_no_content
+    body = '<meta http-equiv="content-type">'
+
+    charsets = Mechanize::Page.meta_charset(body)
+
+    assert_empty charsets
+  end
+
+  # Test to fix issue: https://github.com/tenderlove/mechanize/issues/143
+  def test_page_meta_charset_handles_whitespace
+    body = '<meta http-equiv = "Content-Type" content = "text/html; charset=iso-8859-1">'
+    charsets = Mechanize::Page.meta_charset(body)
+    assert_equal ["iso-8859-1"], charsets
+  end
+
   def test_meta_charset
     body = '<meta http-equiv="content-type" content="text/html;charset=META">'
     page = util_page body
@@ -106,7 +128,7 @@ class TestMechanizePageEncoding < MiniTest::Unit::TestCase
   def test_encodings
     response = {'content-type' => 'text/html;charset=HEADER'}
     body = '<meta http-equiv="content-type" content="text/html;charset=META">'
-    @agent.default_encoding = 'DEFAULT'
+    @mech.default_encoding = 'DEFAULT'
     page = util_page body, response
 
     assert_equal true, page.encodings.include?('HEADER')
@@ -119,23 +141,23 @@ class TestMechanizePageEncoding < MiniTest::Unit::TestCase
     # pre test
     assert_equal false, util_page.encodings.include?('Windows-1252')
 
-    @agent.default_encoding = 'Windows-1252'
+    @mech.default_encoding = 'Windows-1252'
     page = util_page
 
     assert_equal true, page.encodings.include?('Windows-1252')
   end
 
   def test_parser_force_default_encoding
-    @agent.default_encoding = 'Windows-1252'
-    @agent.force_default_encoding = true
+    @mech.default_encoding = 'Windows-1252'
+    @mech.force_default_encoding = true
     page = util_page
 
     assert page.encodings.include? 'Windows-1252'
   end
 
   def test_parser_encoding_equals_overwrites_force_default_encoding
-    @agent.default_encoding = 'Windows-1252'
-    @agent.force_default_encoding = true
+    @mech.default_encoding = 'Windows-1252'
+    @mech.force_default_encoding = true
     page = util_page
 
     assert_equal 'Windows-1252', page.encoding
@@ -144,4 +166,17 @@ class TestMechanizePageEncoding < MiniTest::Unit::TestCase
 
     assert_equal 'ISO-8859-2', page.encoding
   end
+
+  def test_parser_encoding_when_searching_elements
+    skip "Encoding not implemented" unless Object.const_defined? :Encoding
+    
+    body = '<span id="latin1">hi</span>'
+    page = util_page body, 'content-type' => 'text/html,charset=ISO-8859-1'
+
+    result = page.search('#latin1')
+
+    assert_equal Encoding::UTF_8, result.text.encoding
+  end
+
 end
+

data/test/test_mechanize_page_frame.rb

@@ -0,0 +1,16 @@
+require 'mechanize/test_case'
+
+class TestMechanizePageFrame < Mechanize::TestCase
+
+  def test_content
+    page = page 'http://example/referer'
+    frame = node 'frame', 'name' => 'frame1', 'src' => 'http://example/'
+    frame = Mechanize::Page::Frame.new frame, @mech, page
+
+    frame.content
+
+    assert_equal 'http://example/referer', requests.first['Referer']
+  end
+
+end
+

data/test/test_mechanize_page_link.rb

@@ -1,9 +1,8 @@
 # coding: utf-8
 
-require 'helper'
-require 'cgi'
+require 'mechanize/test_case'
 
-class TestMechanizePage < MiniTest::Unit::TestCase
+class TestMechanizePageLink < Mechanize::TestCase
 
   WINDOWS_1255 = <<-HTML
 <meta http-equiv="content-type" content="text/html; charset=windows-1255">
@@ -41,7 +40,8 @@ class TestMechanizePage < MiniTest::Unit::TestCase
   ENCODING_ERROR_CLASS = Nokogiri::XML::SyntaxError
 
   def setup
-    @agent = Mechanize.new
+    super
+
     @uri = URI('http://example')
     @res = { 'content-type' => 'text/html' }
     @body = '<title>hi</title>'
@@ -49,7 +49,7 @@ class TestMechanizePage < MiniTest::Unit::TestCase
 
   def util_page body = @body, res = @res
     body.force_encoding Encoding::BINARY if body.respond_to? :force_encoding
-    Mechanize::Page.new @uri, res, body, 200, @agent
+    Mechanize::Page.new @uri, res, body, 200, @mech
   end
 
   def test_initialize_supported_content_type
@@ -91,10 +91,10 @@ class TestMechanizePage < MiniTest::Unit::TestCase
   end
 
   def test_canonical_uri
-    page = @agent.get("http://localhost/canonical_uri.html")
+    page = @mech.get("http://localhost/canonical_uri.html")
     assert_equal(URI("http://localhost/canonical_uri"), page.canonical_uri)
 
-    page = @agent.get("http://localhost/file_upload.html")
+    page = @mech.get("http://localhost/file_upload.html")
     assert_equal(nil, page.canonical_uri)
   end
 
@@ -254,7 +254,7 @@ class TestMechanizePage < MiniTest::Unit::TestCase
   end
 
   def test_frames_with
-    page = @agent.get("http://localhost/frame_test.html")
+    page = @mech.get("http://localhost/frame_test.html")
     assert_equal(3, page.frames.size)
 
     find_orig = page.frames.find_all { |f| f.name == 'frame1' }
@@ -267,7 +267,7 @@ class TestMechanizePage < MiniTest::Unit::TestCase
   end
 
   def test_links_with_dom_id
-    page = @agent.get("http://localhost/tc_links.html")
+    page = @mech.get("http://localhost/tc_links.html")
     link = page.links_with(:dom_id => 'bold_aaron_link')
     link_by_id = page.links_with(:id => 'bold_aaron_link')
     assert_equal(1, link.length)
@@ -275,34 +275,42 @@ class TestMechanizePage < MiniTest::Unit::TestCase
     assert_equal(link, link_by_id)
   end
 
+  def test_links_with_dom_class
+    page = @mech.get("http://localhost/tc_links.html")
+    link = page.links_with(:dom_class => 'thing_link')
+    link_by_class = page.links_with(:class => 'thing_link')
+    assert_equal(1, link.length)
+    assert_equal(link, link_by_class)
+  end
+
   def test_link_with_encoded_space
-    page = @agent.get("http://localhost/tc_links.html")
+    page = @mech.get("http://localhost/tc_links.html")
     link = page.link_with(:text => 'encoded space')
-    page = @agent.click link
+    page = @mech.click link
   end
 
   def test_link_with_space
-    page = @agent.get("http://localhost/tc_links.html")
+    page = @mech.get("http://localhost/tc_links.html")
     link = page.link_with(:text => 'not encoded space')
-    page = @agent.click link
+    page = @mech.click link
   end
 
   def test_link_with_unusual_characters
-    page = @agent.get("http://localhost/tc_links.html")
+    page = @mech.get("http://localhost/tc_links.html")
     link = page.link_with(:text => 'unusual characters')
 
-    @agent.click link
+    @mech.click link
 
     # HACK no assertion
   end
 
   def test_links
-    page = @agent.get("http://localhost/find_link.html")
+    page = @mech.get("http://localhost/find_link.html")
     assert_equal(18, page.links.length)
   end
 
   def test_links_with_bold
-    page = @agent.get("http://localhost/tc_links.html")
+    page = @mech.get("http://localhost/tc_links.html")
     link = page.links_with(:text => /Bold Dude/)
     assert_equal(1, link.length)
     assert_equal('Bold Dude', link.first.text)
@@ -330,7 +338,7 @@ class TestMechanizePage < MiniTest::Unit::TestCase
   end
 
   def test_meta_refresh
-    page = @agent.get("http://localhost/find_link.html")
+    page = @mech.get("http://localhost/find_link.html")
     assert_equal(3, page.meta_refresh.length)
     assert_equal(%w{
       http://www.drphil.com/
@@ -359,7 +367,7 @@ class TestMechanizePage < MiniTest::Unit::TestCase
   end
 
   def test_form
-    page  = @agent.get("http://localhost/tc_form_action.html")
+    page  = @mech.get("http://localhost/tc_form_action.html")
 
     form = page.form(:name => 'post_form1')
     assert form

data/test/test_mechanize_page_meta_refresh.rb

@@ -1,11 +1,12 @@
-require 'helper'
+require 'mechanize/test_case'
 
-class TestMechanizePageMetaRefresh < MiniTest::Unit::TestCase
+class TestMechanizePageMetaRefresh < Mechanize::TestCase
 
   def setup
+    super
+
     @MR = Mechanize::Page::MetaRefresh
 
-    @agent = Mechanize.new
     @uri = URI 'http://example/here/'
   end
 
@@ -15,7 +16,7 @@ class TestMechanizePageMetaRefresh < MiniTest::Unit::TestCase
     BODY
 
     Mechanize::Page.new(@uri, { 'content-type' => 'text/html' }, body, 200,
-                        @agent)
+                        @mech)
   end
 
   def util_meta_refresh page
@@ -24,37 +25,48 @@ class TestMechanizePageMetaRefresh < MiniTest::Unit::TestCase
   end
 
   def test_class_parse
-    delay, uri = @MR.parse "0; url=http://localhost:8080/path", @uri
+    delay, uri, link_self = @MR.parse "0; url=http://localhost:8080/path", @uri
     assert_equal "0", delay
     assert_equal "http://localhost:8080/path", uri.to_s
+    refute link_self
 
-    delay, uri = @MR.parse "100.001; url=http://localhost:8080/path", @uri
+    delay, uri, link_self =
+      @MR.parse "100.001; url=http://localhost:8080/path", @uri
     assert_equal "100.001", delay
     assert_equal "http://localhost:8080/path", uri.to_s
+    refute link_self
 
-    delay, uri = @MR.parse "0; url='http://localhost:8080/path'", @uri
+    delay, uri, link_self =
+      @MR.parse "0; url='http://localhost:8080/path'", @uri
     assert_equal "0", delay
     assert_equal "http://localhost:8080/path", uri.to_s
+    refute link_self
 
-    delay, uri = @MR.parse "0; url=\"http://localhost:8080/path\"", @uri
+    delay, uri, link_self =
+      @MR.parse "0; url=\"http://localhost:8080/path\"", @uri
     assert_equal "0", delay
     assert_equal "http://localhost:8080/path", uri.to_s
+    refute link_self
 
-    delay, uri = @MR.parse "0; url=", @uri
+    delay, uri, link_self = @MR.parse "0; url=", @uri
     assert_equal "0", delay
     assert_equal "http://example/here/", uri.to_s
+    assert link_self
 
-    delay, uri = @MR.parse "0", @uri
+    delay, uri, link_self = @MR.parse "0", @uri
     assert_equal "0", delay
     assert_equal "http://example/here/", uri.to_s
+    assert link_self
 
-    delay, uri = @MR.parse "   0;   ", @uri
+    delay, uri, link_self = @MR.parse "   0;   ", @uri
     assert_equal "0", delay
     assert_equal "http://example/here/", uri.to_s
+    assert link_self
 
-    delay, uri = @MR.parse "0; UrL=http://localhost:8080/path", @uri
+    delay, uri, link_self = @MR.parse "0; UrL=http://localhost:8080/path", @uri
     assert_equal "0", delay
     assert_equal "http://localhost:8080/path", uri.to_s
+    refute link_self
   end
 
   def test_class_from_node
@@ -95,7 +107,7 @@ class TestMechanizePageMetaRefresh < MiniTest::Unit::TestCase
     BODY
 
     page = Mechanize::Page.new(@uri, { 'content-type' => 'text/html' }, body,
-                               200, @agent)
+                               200, @mech)
 
     assert_nil util_meta_refresh page
   end
@@ -106,7 +118,7 @@ class TestMechanizePageMetaRefresh < MiniTest::Unit::TestCase
     BODY
 
     page = Mechanize::Page.new(@uri, { 'content-type' => 'text/html' }, body,
-                               200, @agent)
+                               200, @mech)
 
     assert_nil util_meta_refresh page
   end

data/test/test_mechanize_parser.rb

@@ -0,0 +1,289 @@
+require 'mechanize/test_case'
+
+class TestMechanizeParser < Mechanize::TestCase
+
+  class P
+    include Mechanize::Parser
+
+    attr_accessor :filename
+    attr_accessor :response
+    attr_accessor :uri
+
+    def initialize
+      @uri = URI 'http://example'
+      @full_path = false
+    end
+  end
+
+  def setup
+    super
+
+    @parser = P.new
+  end
+
+  def test_extract_filename
+    @parser.response = {}
+
+    assert_equal 'index.html', @parser.extract_filename
+  end
+
+  def test_extract_filename_content_disposition
+    @parser.uri = URI 'http://example/foo'
+
+    @parser.response = {
+      'content-disposition' => 'attachment; filename=genome.jpeg'
+    }
+
+    assert_equal 'genome.jpeg', @parser.extract_filename
+  end
+
+  def test_extract_filename_content_disposition_path
+    @parser.uri = URI 'http://example'
+
+    @parser.response = {
+      'content-disposition' => 'attachment; filename="../genome.jpeg"'
+    }
+
+    assert_equal 'example/genome.jpeg', @parser.extract_filename(true)
+
+    @parser.response = {
+      'content-disposition' => 'attachment; filename="foo/genome.jpeg"'
+    }
+
+    assert_equal 'example/genome.jpeg', @parser.extract_filename(true)
+  end
+
+  def test_extract_filename_content_disposition_path_windows
+    @parser.uri = URI 'http://example'
+
+    @parser.response = {
+      'content-disposition' => 'attachment; filename="..\\\\genome.jpeg"'
+    }
+
+    assert_equal 'example/genome.jpeg', @parser.extract_filename(true)
+
+    @parser.response = {
+      'content-disposition' => 'attachment; filename="foo\\\\genome.jpeg"'
+    }
+
+    assert_equal 'example/genome.jpeg', @parser.extract_filename(true)
+  end
+
+  def test_extract_filename_content_disposition_full_path
+    @parser.uri = URI 'http://example/foo'
+
+    @parser.response = {
+      'content-disposition' => 'attachment; filename=genome.jpeg'
+    }
+
+    assert_equal 'example/genome.jpeg', @parser.extract_filename(true)
+  end
+
+  def test_extract_filename_content_disposition_quoted
+    @parser.uri = URI 'http://example'
+
+    @parser.response = {
+      'content-disposition' => 'attachment; filename="some \"file\""'
+    }
+
+    assert_equal 'some__file_', @parser.extract_filename
+  end
+
+  def test_extract_filename_content_disposition_special
+    @parser.uri = URI 'http://example/foo'
+
+    @parser.response = {
+      'content-disposition' => 'attachment; filename="/\\\\<>:\\"|?*"'
+    }
+
+    assert_equal '_______', @parser.extract_filename
+
+    chars = (0..12).map { |c| c.chr }.join
+    chars += "\\\r"
+    chars += (14..31).map { |c| c.chr }.join
+
+    @parser.response = {
+      'content-disposition' => "attachment; filename=\"#{chars}\""
+    }
+
+    assert_equal '_' * 32, @parser.extract_filename
+  end
+
+  def test_extract_filename_content_disposition_windows_special
+    @parser.uri = URI 'http://example'
+
+    windows_special = %w[
+      AUX
+      COM1
+      COM2
+      COM3
+      COM4
+      COM5
+      COM6
+      COM7
+      COM8
+      COM9
+      CON
+      LPT1
+      LPT2
+      LPT3
+      LPT4
+      LPT5
+      LPT6
+      LPT7
+      LPT8
+      LPT9
+      NUL
+      PRN
+    ]
+
+    windows_special.each do |special|
+      @parser.response = {
+        'content-disposition' => "attachment; filename=#{special}"
+      }
+
+      assert_equal "_#{special}", @parser.extract_filename
+    end
+  end
+
+  def test_extract_filename_host
+    @parser.response = {}
+    @parser.uri = URI 'http://example'
+
+    assert_equal 'example/index.html', @parser.extract_filename(true)
+  end
+
+  def test_extract_filename_special_character
+    @parser.response = {}
+
+    invisible = "\t\n\v\f\r"
+
+    invisible.chars.each do |char|
+      begin
+        @parser.uri = URI "http://example/#{char}"
+
+        assert_equal 'index.html', @parser.extract_filename, char.inspect
+      rescue URI::InvalidURIError
+        # ignore
+      end
+    end
+
+    escaped = "<>\"\\|"
+
+    escaped.chars.each do |char|
+      escaped_char = CGI.escape char
+
+      @parser.uri = URI "http://example/#{escaped_char}"
+
+      assert_equal "#{escaped_char}.html", @parser.extract_filename, char
+    end
+
+    @parser.uri = URI "http://example/?"
+
+    assert_equal 'index.html_', @parser.extract_filename, 'empty query'
+
+    @parser.uri = URI "http://example/:"
+
+    assert_equal '_.html', @parser.extract_filename, 'colon'
+
+    @parser.uri = URI "http://example/*"
+
+    assert_equal '_.html', @parser.extract_filename, 'asterisk'
+  end
+
+  def test_extract_filename_uri
+    @parser.response = {}
+    @parser.uri = URI 'http://example/foo'
+
+    assert_equal 'foo.html', @parser.extract_filename
+
+    @parser.uri += '/foo.jpg'
+
+    assert_equal 'foo.jpg', @parser.extract_filename
+  end
+
+  def test_extract_filename_uri_full_path
+    @parser.response = {}
+    @parser.uri = URI 'http://example/foo'
+
+    assert_equal 'example/foo.html', @parser.extract_filename(true)
+
+    @parser.uri += '/foo.jpg'
+
+    assert_equal 'example/foo.jpg', @parser.extract_filename(true)
+  end
+
+  def test_extract_filename_uri_query
+    @parser.response = {}
+    @parser.uri = URI 'http://example/?id=5'
+
+    assert_equal 'index.html_id=5', @parser.extract_filename
+
+    @parser.uri += '/foo.html?id=5'
+
+    assert_equal 'foo.html_id=5', @parser.extract_filename
+  end
+
+  def test_extract_filename_uri_slash
+    @parser.response = {}
+    @parser.uri = URI 'http://example/foo/'
+
+    assert_equal 'example/foo/index.html', @parser.extract_filename(true)
+
+    @parser.uri += '/foo///'
+
+    assert_equal 'example/foo/index.html', @parser.extract_filename(true)
+  end
+
+  def test_extract_filename_windows_special
+    @parser.uri = URI 'http://example'
+    @parser.response = {}
+
+    windows_special = %w[
+      AUX
+      COM1
+      COM2
+      COM3
+      COM4
+      COM5
+      COM6
+      COM7
+      COM8
+      COM9
+      CON
+      LPT1
+      LPT2
+      LPT3
+      LPT4
+      LPT5
+      LPT6
+      LPT7
+      LPT8
+      LPT9
+      NUL
+      PRN
+    ]
+
+    windows_special.each do |special|
+      @parser.uri += "/#{special}"
+
+      assert_equal "_#{special}.html", @parser.extract_filename
+    end
+  end
+
+  def test_fill_header
+    @parser.fill_header 'a' => 'b'
+
+    expected = { 'a' => 'b' }
+
+    assert_equal expected, @parser.response
+  end
+
+  def test_fill_header_nil
+    @parser.fill_header nil
+
+    assert_empty @parser.response
+  end
+
+end
+