mechanize 2.7.4 → 2.8.1

data/lib/mechanize/test_case/gzip_servlet.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 require 'stringio'
 require 'zlib'
 
@@ -13,8 +14,8 @@ class GzipServlet < WEBrick::HTTPServlet::AbstractServlet
     end
 
     if name = req.query['file'] then
-      open "#{TEST_DIR}/htdocs/#{name}" do |io|
-        string = ""
+      ::File.open("#{TEST_DIR}/htdocs/#{name}") do |io|
+        string = String.new
         zipped = StringIO.new string, 'w'
         Zlib::GzipWriter.wrap zipped do |gz|
           gz.write io.read
@@ -22,7 +23,7 @@ class GzipServlet < WEBrick::HTTPServlet::AbstractServlet
         res.body = string
       end
     else
-      res.body = ''
+      res.body = String.new
     end
 
     res['Content-Encoding'] = req['X-ResponseContentEncoding'] || 'gzip'

data/lib/mechanize/test_case/header_servlet.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 class HeaderServlet < WEBrick::HTTPServlet::AbstractServlet
   def do_GET(req, res)
     res.content_type = "text/plain"

data/lib/mechanize/test_case/http_refresh_servlet.rb

@@ -1,9 +1,9 @@
+# frozen_string_literal: true
 class HttpRefreshServlet < WEBrick::HTTPServlet::AbstractServlet
   def do_GET(req, res)
     res['Content-Type'] = req.query['ct'] || "text/html"
     refresh_time = req.query['refresh_time'] || 0
     refresh_url = req.query['refresh_url'] || '/'
-    res['Refresh'] = " #{refresh_time};url=#{refresh_url}\r\n";
+    res['Refresh'] = " #{refresh_time};url=#{refresh_url}";
   end
 end
-

data/lib/mechanize/test_case/infinite_redirect_servlet.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 class InfiniteRedirectServlet < WEBrick::HTTPServlet::AbstractServlet
   def do_GET(req, res)
     res['Content-Type'] = req.query['ct'] || "text/html"

data/lib/mechanize/test_case/infinite_refresh_servlet.rb

@@ -1,10 +1,10 @@
+# frozen_string_literal: true
 class InfiniteRefreshServlet < WEBrick::HTTPServlet::AbstractServlet
   def do_GET(req, res)
     address = "#{req.host}:#{req.port}"
     res['Content-Type'] = req.query['ct'] || "text/html"
     res.status = req.query['code'] ? req.query['code'].to_i : '302'
     number = req.query['q'] ? req.query['q'].to_i : 0
-    res['Refresh'] = "0;url=http://#{address}/infinite_refresh?q=#{number + 1}\r\n";
+    res['Refresh'] = "0;url=http://#{address}/infinite_refresh?q=#{number + 1}";
   end
 end
-

data/lib/mechanize/test_case/many_cookies_as_string_servlet.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 class ManyCookiesAsStringServlet < WEBrick::HTTPServlet::AbstractServlet
   def do_GET(req, res)
     cookies = []

data/lib/mechanize/test_case/many_cookies_servlet.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 class ManyCookiesServlet < WEBrick::HTTPServlet::AbstractServlet
   def do_GET(req, res)
     name_cookie = WEBrick::Cookie.new("name", "Aaron")

data/lib/mechanize/test_case/modified_since_servlet.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 class ModifiedSinceServlet < WEBrick::HTTPServlet::AbstractServlet
   def do_GET(req, res)
     s_time = 'Fri, 04 May 2001 00:00:38 GMT'

data/lib/mechanize/test_case/ntlm_servlet.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 class NTLMServlet < WEBrick::HTTPServlet::AbstractServlet
 
   def do_GET(req, res)

data/lib/mechanize/test_case/one_cookie_no_spaces_servlet.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 class OneCookieNoSpacesServlet < WEBrick::HTTPServlet::AbstractServlet
   def do_GET(req, res)
     cookie = WEBrick::Cookie.new("foo", "bar")

data/lib/mechanize/test_case/one_cookie_servlet.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 class OneCookieServlet < WEBrick::HTTPServlet::AbstractServlet
   def do_GET(req, res)
     cookie = WEBrick::Cookie.new("foo", "bar")

data/lib/mechanize/test_case/quoted_value_cookie_servlet.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 class QuotedValueCookieServlet < WEBrick::HTTPServlet::AbstractServlet
   def do_GET(req, res)
     cookie = WEBrick::Cookie.new("quoted", "\"value\"")

data/lib/mechanize/test_case/redirect_servlet.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 class RedirectServlet < WEBrick::HTTPServlet::AbstractServlet
   def do_GET(req, res)
     res['Content-Type'] = req.query['ct'] || 'text/html'

data/lib/mechanize/test_case/referer_servlet.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 class RefererServlet < WEBrick::HTTPServlet::AbstractServlet
   def do_GET(req, res)
     res['Content-Type'] = "text/html"

data/lib/mechanize/test_case/refresh_with_empty_url.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 class RefreshWithEmptyUrl < WEBrick::HTTPServlet::AbstractServlet
   @@count = 0
   def do_GET(req, res)

data/lib/mechanize/test_case/refresh_without_url.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 class RefreshWithoutUrl < WEBrick::HTTPServlet::AbstractServlet
   @@count = 0
   def do_GET(req, res)

data/lib/mechanize/test_case/response_code_servlet.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 class ResponseCodeServlet < WEBrick::HTTPServlet::AbstractServlet
   def do_GET(req, res)
     res['Content-Type'] = req.query['ct'] || "text/html"

data/lib/mechanize/test_case/robots_txt_servlet.rb

@@ -0,0 +1,15 @@
+# frozen_string_literal: true
+class RobotsTxtServlet < WEBrick::HTTPServlet::AbstractServlet
+  def do_GET(req, res)
+    if /301/ === req['Host'] && req.path == '/robots.txt'
+      res['Location'] = 'http://301/robots_txt'
+      res.code = 301
+    else
+      res['Content-Type'] = 'text/plain'
+      res.body = <<-'EOF'
+User-Agent: *
+Disallow: /norobots
+      EOF
+    end
+  end
+end

data/lib/mechanize/test_case/send_cookies_servlet.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 class SendCookiesServlet < WEBrick::HTTPServlet::AbstractServlet
   def do_GET(req, res)
     res.content_type = 'text/html'

data/lib/mechanize/test_case/server.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 require 'webrick'
 require 'mechanize/test_case/servlets'
 

data/lib/mechanize/test_case/servlets.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 require 'mechanize/test_case/bad_chunking_servlet'
 require 'mechanize/test_case/basic_auth_servlet'
 require 'mechanize/test_case/content_type_servlet'
@@ -21,6 +22,7 @@ require 'mechanize/test_case/referer_servlet'
 require 'mechanize/test_case/refresh_with_empty_url'
 require 'mechanize/test_case/refresh_without_url'
 require 'mechanize/test_case/response_code_servlet'
+require 'mechanize/test_case/robots_txt_servlet'
 require 'mechanize/test_case/send_cookies_servlet'
 require 'mechanize/test_case/verb_servlet'
 
@@ -49,6 +51,8 @@ MECHANIZE_TEST_CASE_SERVLETS = {
   '/refresh_with_empty_url' => RefreshWithEmptyUrl,
   '/refresh_without_url'    => RefreshWithoutUrl,
   '/response_code'          => ResponseCodeServlet,
+  '/robots.txt'             => RobotsTxtServlet,
+  '/robots_txt'             => RobotsTxtServlet,
   '/send_cookies'           => SendCookiesServlet,
   '/verb'                   => VerbServlet,
 }

data/lib/mechanize/test_case/verb_servlet.rb

@@ -1,11 +1,10 @@
+# frozen_string_literal: true
 class VerbServlet < WEBrick::HTTPServlet::AbstractServlet
   %w[HEAD GET POST PUT DELETE].each do |verb|
-    eval <<-METHOD
-      def do_#{verb}(req, res)
-        res.header['X-Request-Method'] = #{verb.dump}
-        res.body = #{verb.dump}
-      end
-    METHOD
+    define_method "do_#{verb}" do |req, res|
+      res.header['X-Request-Method'] = verb
+      res.body = verb
+    end
   end
 end
 

data/lib/mechanize/unauthorized_error.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 class Mechanize::UnauthorizedError < Mechanize::ResponseCodeError
 
   attr_reader :challenges
@@ -11,7 +12,7 @@ class Mechanize::UnauthorizedError < Mechanize::ResponseCodeError
     out = super
 
     if @challenges then
-      realms = @challenges.map { |challenge| challenge.realm_name }.join ', '
+      realms = @challenges.map(&:realm_name).join ', '
       out << " -- available realms: #{realms}"
     end
 

data/lib/mechanize/unsupported_scheme_error.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 class Mechanize::UnsupportedSchemeError < Mechanize::Error
   attr_accessor :scheme, :uri
 

data/lib/mechanize/util.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 require 'cgi'
 require 'nkf'
 
@@ -15,7 +16,7 @@ class Mechanize::Util
     def build_query_string(parameters, enc = nil)
       each_parameter(parameters).inject(nil) { |s, (k, v)|
         # WEBrick::HTTP.escape* has some problems about m17n on ruby-1.9.*.
-        (s.nil? ? '' : s << '&') << [CGI.escape(k.to_s), CGI.escape(v.to_s)].join('=')
+        (s.nil? ? String.new : s << '&') << [CGI.escape(k.to_s), CGI.escape(v.to_s)].join('=')
       } || ''
     end
 
@@ -102,10 +103,11 @@ class Mechanize::Util
   case NKF::BINARY
   when Encoding
     def self.guess_encoding(src)
-      NKF.guess(src)
+      # NKF.guess of JRuby may return nil
+      NKF.guess(src) || Encoding::US_ASCII
     end
   else
-    # Old NKF from 1.8, still bundled with JRuby and Rubinius
+    # Old NKF from 1.8, still bundled with Rubinius
     NKF_ENCODING_MAP = {
       NKF::UNKNOWN => Encoding::US_ASCII,
       NKF::BINARY  => Encoding::ASCII_8BIT,

data/lib/mechanize/version.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 class Mechanize
-  VERSION = "2.7.4"
+  VERSION = "2.8.1"
 end

data/lib/mechanize/xml_file.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 ##
 # This class encapsulates an XML file. If Mechanize finds a content-type
 # of 'text/xml' or 'application/xml' this class will be instantiated and

data/mechanize.gemspec

@@ -1,20 +1,21 @@
 # coding: utf-8
+# frozen_string_literal: true
 lib = File.expand_path('../lib', __FILE__)
 $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 require 'mechanize/version'
 
 Gem::Specification.new do |spec|
-  spec.name          = "mechanize"
-  spec.version       = Mechanize::VERSION
-  spec.homepage      = "http://docs.seattlerb.org/mechanize/"
-  spec.summary       = %q{The Mechanize library is used for automating interaction with websites}
-  spec.description   =
+  spec.name = "mechanize"
+  spec.version = Mechanize::VERSION
+  spec.homepage = "https://github.com/sparklemotion/mechanize"
+  spec.summary = 'The Mechanize library is used for automating interaction with websites'
+  spec.description =
     [
       "The Mechanize library is used for automating interaction with websites.",
       "Mechanize automatically stores and sends cookies, follows redirects,",
       "and can follow links and submit forms.  Form fields can be populated and",
       "submitted.  Mechanize also keeps track of the sites that you have visited as",
-      "a history."
+      "a history.",
     ].join("\n")
 
   spec.authors =
@@ -23,39 +24,46 @@ Gem::Specification.new do |spec|
       'Aaron Patterson',
       'Mike Dalessio',
       'Akinori MUSHA',
-      'Lee Jarvis'
+      'Lee Jarvis',
     ]
   spec.email =
     [
       'drbrain@segment7.net',
-      'aaronp@rubyforge.org',
+      'aaron.patterson@gmail.com',
       'mike.dalessio@gmail.com',
       'knu@idaemons.org',
-      'ljjarvis@gmail.com'
+      'ljjarvis@gmail.com',
     ]
 
-  spec.license           = "MIT"
+  spec.license = "MIT"
 
   spec.require_paths = ["lib"]
-  spec.files         = `git ls-files`.split($/)
-  spec.test_files    = spec.files.grep(%r{^test/})
-
-  spec.extra_rdoc_files += Dir['*.rdoc']
-  spec.rdoc_options     = ["--main", "README.rdoc"]
-
-  spec.required_ruby_version = ">= 1.9.2"
-
-  spec.add_runtime_dependency "net-http-digest_auth", [ ">= 1.1.1", "~> 1.1" ]
-  spec.add_runtime_dependency "net-http-persistent",  [ ">= 2.5.2", "~> 2.5" ]
-  spec.add_runtime_dependency "mime-types",           [ ">= 1.17.2", "< 3" ]
-  spec.add_runtime_dependency "http-cookie",          [ "~> 1.0" ]
-  spec.add_runtime_dependency "nokogiri",             [ "~> 1.6" ]
-  spec.add_runtime_dependency "ntlm-http",            [ ">= 0.1.1", "~> 0.1"   ]
-  spec.add_runtime_dependency "webrobots",            [ "< 0.2",    ">= 0.0.9" ]
-  spec.add_runtime_dependency "domain_name",          [ ">= 0.5.1", "~> 0.5"   ]
-
-  spec.add_development_dependency "rake"
-  spec.add_development_dependency "bundler",  "~> 1.3"
-  spec.add_development_dependency "rdoc",     "~> 4.0"
-  spec.add_development_dependency "minitest", "~> 5.0"
+  spec.files = %x(git ls-files).split($/)
+  spec.test_files = spec.files.grep(%r{^test/})
+
+  spec.extra_rdoc_files += Dir['*.rdoc', '*.md']
+  spec.rdoc_options = ["--main", "README.md"]
+
+  spec.required_ruby_version = ">= 2.5.0"
+
+  spec.add_runtime_dependency("addressable", "~> 2.7")
+  spec.add_runtime_dependency("domain_name", ">= 0.5.20190701", "~> 0.5")
+  spec.add_runtime_dependency("http-cookie", ">= 1.0.3", "~> 1.0")
+  spec.add_runtime_dependency("mime-types", "~> 3.0")
+  spec.add_runtime_dependency("net-http-digest_auth", ">= 1.4.1", "~> 1.4")
+
+  # careful! some folks are relying on older versions of net-http-persistent
+  # - see the socks proxy patch in use at #507 and #464
+  # - see use of retry_change_requests that was removed at #558
+  spec.add_runtime_dependency("net-http-persistent", ">= 2.5.2", "< 5.0.dev")
+
+  spec.add_runtime_dependency("nokogiri", ">= 1.11.2", "~> 1.11")
+  spec.add_runtime_dependency("rubyntlm", ">= 0.6.3", "~> 0.6")
+  spec.add_runtime_dependency("webrick", "~> 1.7")
+  spec.add_runtime_dependency("webrobots", "~> 0.1.2")
+
+  spec.add_development_dependency("minitest", "~> 5.14")
+  spec.add_development_dependency("rake", "~> 13.0")
+  spec.add_development_dependency("rdoc", "~> 6.3")
+  spec.add_development_dependency("rubocop", "~> 1.12")
 end

data/test/htdocs/dir with spaces/foo.html

@@ -1 +1,2 @@
 HELLO
+

data/test/htdocs/find_link.html

@@ -11,13 +11,10 @@
         <A HREF="http://blargle.com/">blargle</A>
         <A HREF="http://a.cpan.org/">CPAN A</A>
         <A HREF="http://b.cpan.org/">CPAN B</A>
-        <FRAME SRC="foo.html">
-        <FRAME SRC="bar.html">
         <A HREF="http://c.cpan.org/" NAME="bongo">CPAN C</A>
         <A HREF="http://d.cpan.org/">CPAN D</A>
 
         <A HREF="http://www.msnbc.com/">MSNBC</A>
-        <FRAME SRC="http://www.oreilly.com/" NAME="wongo">
         <A HREF="http://www.cnn.com/">CNN</A>
         <A HREF="http://www.bbc.co.uk/" NAME="Wilma">BBC</A>
         <A HREF="http://www.msnbc.com/">News</A>
@@ -35,7 +32,7 @@
         <!-- new stuff -->
         <A HREF="http://nowhere.org/" Name="Here">NoWhere</A>
         <A HREF="http://nowhere.org/padded" Name=" Here "> NoWhere </A>
-        <A HREF="blongo.html">Blongo!</A>
+        <A HREF="form_test.html" class="formtest">Form Test</A>
     </body>
 </html>
 

data/test/htdocs/tc_links.html

@@ -7,7 +7,7 @@
     <a href="thing.html" class="thing_link">Thing!</a>
     <a href="thing.html">Ruby <b>Rocks!</b></a>
     <!-- Testing a bug with escaped stuff in links:
-      http://rubyforge.org/pipermail/mechanize-users/2006-September/000002.html
+      http://rubygems.org/pipermail/mechanize-users/2006-September/000002.html
     -->
     <a href="link%20with%20space.html">encoded space</a>
     <a href="link with space.html">not encoded space</a>

data/test/test_mechanize.rb

@@ -92,7 +92,7 @@ class TestMechanize < Mechanize::TestCase
 
     assert_equal("http://localhost/form_test.html",
                  @mech.history.last.uri.to_s)
-  end
+  end unless RUBY_ENGINE == 'jruby'  # NekoHTML does not parse body of NOFRAMES
 
   def test_click_bogus_link_with_cookies
     @mech.cookie_jar = cookie_jar("a=b")
@@ -149,7 +149,7 @@ class TestMechanize < Mechanize::TestCase
 
     assert_equal("http://localhost/form_test.html",
                  @mech.history.last.uri.to_s)
-  end
+  end unless RUBY_ENGINE == 'jruby'  # NekoHTML does not parse body of NOFRAMES
 
   def test_click_link
     link = node 'a', 'href' => '/index.html'
@@ -345,6 +345,15 @@ but not <a href="/" rel="me nofollow">this</a>!
     end
   end
 
+  def test_download_does_not_allow_command_injection
+    skip if windows?
+    in_tmpdir do
+      @mech.download('http://example', '| ruby -rfileutils -e \'FileUtils.touch("vul.txt")\'')
+
+      refute_operator(File, :exist?, "vul.txt")
+    end
+  end
+
   def test_get
     uri = URI 'http://localhost'
 
@@ -374,6 +383,14 @@ but not <a href="/" rel="me nofollow">this</a>!
     end
   end
 
+  def test_get_auth
+    @mech.add_auth @uri, 'user', 'pass'
+
+    page = @mech.get @uri + '/basic_auth'
+
+    assert_equal 'You are authenticated', page.body
+  end
+
   def test_get_auth_bad
     @mech.add_auth(@uri, 'aaron', 'aaron')
 
@@ -392,6 +409,14 @@ but not <a href="/" rel="me nofollow">this</a>!
     assert_equal("401", e.response_code)
   end
 
+  def test_get_auth_realm
+    @mech.add_auth @uri, 'user', 'pass', 'Blah'
+
+    page = @mech.get @uri + '/basic_auth'
+
+    assert_equal 'You are authenticated', page.body
+  end
+
   def test_get_conditional
     assert_empty @mech.history
 
@@ -673,9 +698,7 @@ but not <a href="/" rel="me nofollow">this</a>!
   end
 
   def test_get_space
-    page = nil
-
-    page = @mech.get("http://localhost/tc_bad_links.html ")
+    @mech.get("http://localhost/tc_bad_links.html ")
 
     assert_match(/tc_bad_links.html$/, @mech.history.last.uri.to_s)
 
@@ -927,7 +950,26 @@ but not <a href="/" rel="me nofollow">this</a>!
       "Content-Disposition: form-data; name=\"userfile1\"; filename=\"#{name}\"",
       page.body
     )
-    assert page.body.length > File.read(__FILE__).length
+    assert_operator page.body.bytesize, :>, file_contents_without_cr(__FILE__).length
+  end
+
+  def test_post_file_upload_nonascii
+    name = 'ユーザファイル1'
+    file_upload = Mechanize::Form::FileUpload.new({'name' => 'userfile1'}, name)
+    file_upload.file_data = File.read(__FILE__)
+    file_upload.mime_type = 'application/zip'
+
+    page = @mech.post('http://localhost/file_upload', {
+      :name       => 'Some file',
+      :userfile1  => file_upload
+    })
+
+    assert_match(
+      "Content-Disposition: form-data; name=\"userfile1\"; filename=\"#{name}\"".force_encoding(Encoding::ASCII_8BIT),
+      page.body
+    )
+    assert_match("Content-Type: application/zip", page.body)
+    assert_operator page.body.bytesize, :>, file_contents_without_cr(__FILE__).length
   end
 
   def test_post_file_upload
@@ -946,7 +988,7 @@ but not <a href="/" rel="me nofollow">this</a>!
       page.body
     )
     assert_match("Content-Type: application/zip", page.body)
-    assert page.body.length > File.read(__FILE__).length
+    assert_operator page.body.bytesize, :>, file_contents_without_cr(__FILE__).length
   end
 
   def test_post_redirect
@@ -1021,6 +1063,11 @@ but not <a href="/" rel="me nofollow">this</a>!
   end
 
   def test_retry_change_requests_equals
+    unless Gem::Requirement.new("< 4.0.0").satisfied_by?(Gem::Version.new(Net::HTTP::Persistent::VERSION))
+      # see https://github.com/drbrain/net-http-persistent/pull/100
+      skip("net-http-persistent 4.0.0 and later does not support retry_change_requests")
+    end
+
     refute @mech.retry_change_requests
 
     @mech.retry_change_requests = true
@@ -1052,25 +1099,21 @@ but not <a href="/" rel="me nofollow">this</a>!
 
     assert_match(/Hello World/, @mech.current_page.body)
     refute_empty @mech.cookies
-    refute_empty Thread.current[@mech.agent.http.request_key]
 
     @mech.shutdown
 
-    assert_nil Thread.current[@mech.agent.http.request_key]
     assert_empty @mech.history
     assert_empty @mech.cookies
   end
 
   def test_start
-    body, id = nil
+    body = nil
 
     Mechanize.start do |m|
       body = m.get("http://localhost/").body
-      id = m.agent.http.request_key
     end
 
     assert_match(/Hello World/, body)
-    assert_nil Thread.current[id]
   end
 
   def test_submit_bad_form_method
@@ -1140,7 +1183,7 @@ but not <a href="/" rel="me nofollow">this</a>!
 
     page = @mech.submit(form)
 
-    contents = File.read __FILE__
+    contents = File.binread(__FILE__).gsub(/\r\n/, "\n")
     basename = File.basename __FILE__
 
     assert_match(
@@ -1160,7 +1203,7 @@ but not <a href="/" rel="me nofollow">this</a>!
 
     page = @mech.submit(form)
 
-    contents = File.read __FILE__
+    contents = File.binread __FILE__
     basename = File.basename __FILE__
     assert_match(
       "Content-Disposition: form-data; name=\"green[eggs]\"; filename=\"#{basename}\"",
@@ -1328,4 +1371,3 @@ but not <a href="/" rel="me nofollow">this</a>!
     end
   end
 end
-