mechanize 2.7.3 → 2.7.4

data/lib/mechanize/page/image.rb

@@ -169,6 +169,8 @@ class Mechanize::Page::Image
     end
   end
 
+  alias uri url
+
   ##
   # The width attribute of the image
 

data/lib/mechanize/page/link.rb

@@ -99,5 +99,10 @@ class Mechanize::Page::Link
              end
   end
 
+  # A fully resolved URI for the #href for this link.
+  def resolved_uri
+    @mech.resolve uri
+  end
+
 end
 

data/lib/mechanize/pluggable_parsers.rb

@@ -2,6 +2,7 @@ require 'mechanize/file'
 require 'mechanize/file_saver'
 require 'mechanize/page'
 require 'mechanize/xml_file'
+require 'mime/types'
 
 ##
 # Mechanize allows different parsers for different content types.  Mechanize
@@ -68,6 +69,15 @@ class Mechanize::PluggableParser
     :xml   => ['text/xml', 'application/xml'],
   }
 
+  InvalidContentTypeError =
+    if defined?(MIME::Type::InvalidContentType)
+      # For mime-types >=2.1
+      MIME::Type::InvalidContentType
+    else
+      # For mime-types <2.1
+      MIME::InvalidContentType
+    end
+
   attr_accessor :default
 
   def initialize
@@ -97,9 +107,11 @@ class Mechanize::PluggableParser
 
     parser = @parsers[mime_type.to_s] ||
              @parsers[mime_type.simplified] ||
+             # Starting from mime-types 3.0 x-prefix is deprecated as per IANA
+             (@parsers[MIME::Type.simplified(mime_type.to_s, remove_x_prefix: true)] rescue nil) ||
              @parsers[mime_type.media_type] ||
              default
-  rescue MIME::Type::InvalidContentType
+  rescue InvalidContentTypeError
     default
   end
 

data/lib/mechanize/test_case.rb

@@ -14,6 +14,11 @@ end
 
 require 'minitest/autorun'
 
+begin
+  require 'minitest/pride'
+rescue LoadError
+end
+
 ##
 # A generic test case for testing mechanize.  Using a subclass of
 # Mechanize::TestCase for your tests will create an isolated mechanize

data/lib/mechanize/unsupported_scheme_error.rb

@@ -1,6 +1,8 @@
 class Mechanize::UnsupportedSchemeError < Mechanize::Error
-  attr_accessor :scheme
-  def initialize(scheme)
+  attr_accessor :scheme, :uri
+
+  def initialize(scheme, uri)
     @scheme = scheme
+    @uri    = uri
   end
 end

data/lib/mechanize/util.rb

@@ -2,36 +2,70 @@ require 'cgi'
 require 'nkf'
 
 class Mechanize::Util
-  CODE_DIC = {
-    NKF::JIS => "ISO-2022-JP",
-    NKF::EUC => "EUC-JP",
-    NKF::SJIS => "SHIFT_JIS",
-    NKF::UTF8 => "UTF-8",
-    NKF::UTF16 => "UTF-16",
-    NKF::UTF32 => "UTF-32",
-  }
-
-  # true if RUBY_VERSION is 1.9.0 or later
-  NEW_RUBY_ENCODING = RUBY_VERSION >= '1.9.0'
-
-  # contains encoding error classes to raise
-  ENCODING_ERRORS = if NEW_RUBY_ENCODING
-                      [EncodingError]
-                    else
-                      [Iconv::InvalidEncoding, Iconv::IllegalSequence]
-                    end
-
   # default mime type data for Page::Image#mime_type.
   # You can use another Apache-compatible mimetab.
   #   mimetab = WEBrick::HTTPUtils.load_mime_types('/etc/mime.types')
   #   Mechanize::Util::DefaultMimeTypes.replace(mimetab)
   DefaultMimeTypes = WEBrick::HTTPUtils::DefaultMimeTypes
 
-  def self.build_query_string(parameters, enc = nil)
-    parameters.map { |k,v|
-      # WEBrick::HTTP.escape* has some problems about m17n on ruby-1.9.*.
-      [CGI.escape(k.to_s), CGI.escape(v.to_s)].join("=") if k
-    }.compact.join('&')
+  class << self
+    # Builds a query string from a given enumerable object
+    # +parameters+.  This method uses Mechanize::Util.each_parameter
+    # as preprocessor, which see.
+    def build_query_string(parameters, enc = nil)
+      each_parameter(parameters).inject(nil) { |s, (k, v)|
+        # WEBrick::HTTP.escape* has some problems about m17n on ruby-1.9.*.
+        (s.nil? ? '' : s << '&') << [CGI.escape(k.to_s), CGI.escape(v.to_s)].join('=')
+      } || ''
+    end
+
+    # Parses an enumerable object +parameters+ and iterates over the
+    # key-value pairs it contains.
+    #
+    # +parameters+ may be a hash, or any enumerable object which
+    # iterates over [key, value] pairs, typically an array of arrays.
+    #
+    # If a key is paired with an array-like object, the pair is
+    # expanded into multiple occurrences of the key, one for each
+    # element of the array.  e.g. { a: [1, 2] } => [:a, 1], [:a, 2]
+    #
+    # If a key is paired with a hash-like object, the pair is expanded
+    # into hash-like multiple pairs, one for each pair of the hash.
+    # e.g. { a: { x: 1, y: 2 } } => ['a[x]', 1], ['a[y]', 2]
+    #
+    # An array-like value is allowed to be specified as hash value.
+    # e.g. { a: { q: [1, 2] } } => ['a[q]', 1], ['a[q]', 2]
+    #
+    # For a non-array-like, non-hash-like value, the key-value pair is
+    # yielded as is.
+    def each_parameter(parameters, &block)
+      return to_enum(__method__, parameters) if block.nil?
+
+      parameters.each { |key, value|
+        each_parameter_1(key, value, &block)
+      }
+    end
+
+    private
+
+    def each_parameter_1(key, value, &block)
+      return if key.nil?
+
+      case
+      when s = String.try_convert(value)
+        yield [key, s]
+      when a = Array.try_convert(value)
+        a.each { |avalue|
+          yield [key, avalue]
+        }
+      when h = Hash.try_convert(value)
+        h.each { |hkey, hvalue|
+          each_parameter_1('%s[%s]' % [key, hkey], hvalue, &block)
+        }
+      else
+        yield [key, value]
+      end
+    end
   end
 
   # Converts string +s+ from +code+ to UTF-8.
@@ -40,8 +74,8 @@ class Mechanize::Util
     return s unless Mechanize.html_parser == Nokogiri::HTML
 
     begin
-      encode_to(code, s)
-    rescue *ENCODING_ERRORS => ex
+      s.encode(code)
+    rescue EncodingError => ex
       log.debug("from_native_charset: #{ex.class}: form encoding: #{code.inspect} string: #{s}") if log
       if ignore_encoding_error
         s
@@ -51,16 +85,6 @@ class Mechanize::Util
     end
   end
 
-  # inner convert method of Util.from_native_charset
-  def self.encode_to(encoding, str)
-    if NEW_RUBY_ENCODING
-      str.encode(encoding)
-    else
-      Iconv.conv(encoding.to_s, "UTF-8", str)
-    end
-  end
-  private_class_method :encode_to
-
   def self.html_unescape(s)
     return s unless s
     s.gsub(/&(\w+|#[0-9]+);/) { |match|
@@ -75,15 +99,36 @@ class Mechanize::Util
     }
   end
 
+  case NKF::BINARY
+  when Encoding
+    def self.guess_encoding(src)
+      NKF.guess(src)
+    end
+  else
+    # Old NKF from 1.8, still bundled with JRuby and Rubinius
+    NKF_ENCODING_MAP = {
+      NKF::UNKNOWN => Encoding::US_ASCII,
+      NKF::BINARY  => Encoding::ASCII_8BIT,
+      NKF::ASCII   => Encoding::US_ASCII,
+      NKF::JIS     => Encoding::ISO_2022_JP,
+      NKF::EUC     => Encoding::EUC_JP,
+      NKF::SJIS    => Encoding::Shift_JIS,
+      NKF::UTF8    => Encoding::UTF_8,
+      NKF::UTF16   => Encoding::UTF_16BE,
+      NKF::UTF32   => Encoding::UTF_32BE,
+    }
+
+    def self.guess_encoding(src)
+      NKF_ENCODING_MAP[NKF.guess(src)]
+    end
+  end
+
   def self.detect_charset(src)
-    case enc = src && NKF.guess(src)
-    when Integer
-      # Ruby <= 1.8
-      CODE_DIC[enc]
+    if src
+      guess_encoding(src).name.upcase
     else
-      # Ruby >= 1.9
-      enc && enc.to_s.upcase
-    end || "ISO-8859-1"
+      Encoding::ISO8859_1.name
+    end
   end
 
   def self.uri_escape str, unsafe = nil

data/lib/mechanize/version.rb

@@ -0,0 +1,3 @@
+class Mechanize
+  VERSION = "2.7.4"
+end

data/mechanize.gemspec

@@ -0,0 +1,61 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'mechanize/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "mechanize"
+  spec.version       = Mechanize::VERSION
+  spec.homepage      = "http://docs.seattlerb.org/mechanize/"
+  spec.summary       = %q{The Mechanize library is used for automating interaction with websites}
+  spec.description   =
+    [
+      "The Mechanize library is used for automating interaction with websites.",
+      "Mechanize automatically stores and sends cookies, follows redirects,",
+      "and can follow links and submit forms.  Form fields can be populated and",
+      "submitted.  Mechanize also keeps track of the sites that you have visited as",
+      "a history."
+    ].join("\n")
+
+  spec.authors =
+    [
+      'Eric Hodel',
+      'Aaron Patterson',
+      'Mike Dalessio',
+      'Akinori MUSHA',
+      'Lee Jarvis'
+    ]
+  spec.email =
+    [
+      'drbrain@segment7.net',
+      'aaronp@rubyforge.org',
+      'mike.dalessio@gmail.com',
+      'knu@idaemons.org',
+      'ljjarvis@gmail.com'
+    ]
+
+  spec.license           = "MIT"
+
+  spec.require_paths = ["lib"]
+  spec.files         = `git ls-files`.split($/)
+  spec.test_files    = spec.files.grep(%r{^test/})
+
+  spec.extra_rdoc_files += Dir['*.rdoc']
+  spec.rdoc_options     = ["--main", "README.rdoc"]
+
+  spec.required_ruby_version = ">= 1.9.2"
+
+  spec.add_runtime_dependency "net-http-digest_auth", [ ">= 1.1.1", "~> 1.1" ]
+  spec.add_runtime_dependency "net-http-persistent",  [ ">= 2.5.2", "~> 2.5" ]
+  spec.add_runtime_dependency "mime-types",           [ ">= 1.17.2", "< 3" ]
+  spec.add_runtime_dependency "http-cookie",          [ "~> 1.0" ]
+  spec.add_runtime_dependency "nokogiri",             [ "~> 1.6" ]
+  spec.add_runtime_dependency "ntlm-http",            [ ">= 0.1.1", "~> 0.1"   ]
+  spec.add_runtime_dependency "webrobots",            [ "< 0.2",    ">= 0.0.9" ]
+  spec.add_runtime_dependency "domain_name",          [ ">= 0.5.1", "~> 0.5"   ]
+
+  spec.add_development_dependency "rake"
+  spec.add_development_dependency "bundler",  "~> 1.3"
+  spec.add_development_dependency "rdoc",     "~> 4.0"
+  spec.add_development_dependency "minitest", "~> 5.0"
+end

data/test/test_mechanize.rb

@@ -13,12 +13,6 @@ class TestMechanize < Mechanize::TestCase
     @res = Net::HTTPOK.allocate
     @res.instance_variable_set :@code, 200
     @res.instance_variable_set :@header, {}
-
-    @headers = if RUBY_VERSION > '1.9' then
-                 %w[accept user-agent]
-               else
-                 %w[accept]
-               end
   end
 
   def test_back
@@ -112,6 +106,22 @@ class TestMechanize < Mechanize::TestCase
     assert_equal '/index.html', requests.first.path
   end
 
+  def test_click_image_button
+    page = @mech.get("http://localhost/form_test.html")
+    get_form = page.forms.find { |f| f.name == "get_form1" }
+    image_button = get_form.buttons.first
+    new_page = @mech.click(image_button)
+    assert_equal "http://localhost/form_post?first_name=&button.x=0&button.y=0", new_page.uri.to_s
+  end
+
+  def test_click_submit_button
+    page = @mech.get("http://localhost/form_test.html")
+    get_form = page.forms.find { |f| f.name == "get_form1" }
+    submit_button = get_form.submits.first
+    new_page = @mech.click(submit_button)
+    assert_equal "http://localhost/form_post?first_name=", new_page.uri.to_s
+  end
+
   def test_click_frame
     frame = node 'frame', 'src' => '/index.html'
     frame = Mechanize::Page::Frame.new frame, @mech, fake_page
@@ -433,10 +443,6 @@ but not <a href="/" rel="me nofollow">this</a>!
 
   def test_get_follow_meta_refresh_anywhere
     @mech.follow_meta_refresh = :anywhere
-    requests = []
-    @mech.pre_connect_hooks << lambda { |_, request|
-      requests << request
-    }
 
     @mech.get('http://localhost/tc_meta_in_body.html')
     assert_equal 2, requests.length
@@ -465,10 +471,6 @@ but not <a href="/" rel="me nofollow">this</a>!
 
   def test_get_follow_meta_refresh_in_body
     @mech.follow_meta_refresh = true
-    requests = []
-    @mech.pre_connect_hooks << lambda { |_, request|
-      requests << request
-    }
 
     @mech.get('http://localhost/tc_meta_in_body.html')
     assert_equal 1, requests.length
@@ -492,12 +494,6 @@ but not <a href="/" rel="me nofollow">this</a>!
   def test_get_follow_meta_refresh_referer_not_sent
     @mech.follow_meta_refresh = true
 
-    requests = []
-
-    @mech.pre_connect_hooks << lambda { |_, request|
-      requests << request
-    }
-
     @mech.get('http://localhost/tc_follow_meta.html')
 
     assert_equal 2, @mech.history.length
@@ -578,12 +574,6 @@ but not <a href="/" rel="me nofollow">this</a>!
   def test_get_http_refresh
     @mech.follow_meta_refresh = true
 
-    requests = []
-
-    @mech.pre_connect_hooks << lambda { |_, request|
-      requests << request
-    }
-
     page = @mech.get('http://example/http_refresh?refresh_time=0')
 
     assert_equal('http://example/', page.uri.to_s)
@@ -609,18 +599,15 @@ but not <a href="/" rel="me nofollow">this</a>!
     assert_equal('http://localhost/http_refresh?refresh_time=0', page.uri.to_s)
   end
 
-  def test_get_kcode
-    $KCODE = 'u'
-    page = @mech.get("http://localhost/?a=#{[0xd6].pack('U')}")
-
-    assert_equal('http://localhost/?a=%D6', page.uri.to_s)
-
-    $KCODE = 'NONE'
-  end unless RUBY_VERSION >= '1.9.0'
-
   def test_get_query
     page = @mech.get('http://localhost/', { :q => 'hello' })
     assert_equal('http://localhost/?q=hello', page.uri.to_s)
+
+    page = @mech.get('http://localhost/', { :q => %w[hello world]})
+    assert_equal('http://localhost/?q=hello&q=world', page.uri.to_s)
+
+    page = @mech.get('http://localhost/', { :paging => { start: 1, limit: 25 } })
+    assert_equal('http://localhost/?paging%5Bstart%5D=1&paging%5Blimit%5D=25', page.uri.to_s)
   end
 
   def test_get_redirect
@@ -665,11 +652,6 @@ but not <a href="/" rel="me nofollow">this</a>!
   end
 
   def test_get_referer_none
-    requests = []
-    @mech.pre_connect_hooks << lambda { |_, request|
-      requests << request
-    }
-
     @mech.get('http://localhost/')
     @mech.get('http://localhost/')
     assert_equal(2, requests.length)
@@ -680,7 +662,13 @@ but not <a href="/" rel="me nofollow">this</a>!
 
   def test_get_scheme_unsupported
     assert_raises Mechanize::UnsupportedSchemeError do
-      @mech.get('ftp://server.com/foo.html')
+      begin
+        @mech.get('ftp://server.com/foo.html')
+      rescue Mechanize::UnsupportedSchemeError => error
+        assert_equal 'ftp', error.scheme
+        assert_equal 'ftp://server.com/foo.html', error.uri.to_s
+        raise
+      end
     end
   end
 
@@ -942,6 +930,25 @@ but not <a href="/" rel="me nofollow">this</a>!
     assert page.body.length > File.read(__FILE__).length
   end
 
+  def test_post_file_upload
+    name = File.basename(__FILE__)
+    file_upload = Mechanize::Form::FileUpload.new({'name' => 'userfile1'}, name)
+    file_upload.file_data = File.read(__FILE__)
+    file_upload.mime_type = 'application/zip'
+
+    page = @mech.post('http://localhost/file_upload', {
+      :name       => 'Some file',
+      :userfile1  => file_upload
+    })
+
+    assert_match(
+      "Content-Disposition: form-data; name=\"userfile1\"; filename=\"#{name}\"",
+      page.body
+    )
+    assert_match("Content-Type: application/zip", page.body)
+    assert page.body.length > File.read(__FILE__).length
+  end
+
   def test_post_redirect
     page = @mech.post('http://localhost/redirect')
 
@@ -970,6 +977,13 @@ but not <a href="/" rel="me nofollow">this</a>!
     assert_equal 5, @mech.read_timeout
   end
 
+  def test_timeouts_for_file_connection
+    uri = URI.parse "file://#{File.expand_path __FILE__}"
+    @mech.read_timeout = 5
+    @mech.open_timeout = 5
+    assert @mech.get(uri)
+  end
+
   def test_referer
     host_path = "localhost/tc_referer.html?t=1"
     ['http', 'https'].each { |proto|