mechanize 2.7.7 → 2.8.0

data/lib/mechanize/test_case/refresh_without_url.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 class RefreshWithoutUrl < WEBrick::HTTPServlet::AbstractServlet
   @@count = 0
   def do_GET(req, res)

data/lib/mechanize/test_case/response_code_servlet.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 class ResponseCodeServlet < WEBrick::HTTPServlet::AbstractServlet
   def do_GET(req, res)
     res['Content-Type'] = req.query['ct'] || "text/html"

data/lib/mechanize/test_case/robots_txt_servlet.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 class RobotsTxtServlet < WEBrick::HTTPServlet::AbstractServlet
   def do_GET(req, res)
     if /301/ === req['Host'] && req.path == '/robots.txt'

data/lib/mechanize/test_case/send_cookies_servlet.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 class SendCookiesServlet < WEBrick::HTTPServlet::AbstractServlet
   def do_GET(req, res)
     res.content_type = 'text/html'

data/lib/mechanize/test_case/server.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 require 'webrick'
 require 'mechanize/test_case/servlets'
 

data/lib/mechanize/test_case/servlets.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 require 'mechanize/test_case/bad_chunking_servlet'
 require 'mechanize/test_case/basic_auth_servlet'
 require 'mechanize/test_case/content_type_servlet'

data/lib/mechanize/test_case/verb_servlet.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 class VerbServlet < WEBrick::HTTPServlet::AbstractServlet
   %w[HEAD GET POST PUT DELETE].each do |verb|
     define_method "do_#{verb}" do |req, res|

data/lib/mechanize/unauthorized_error.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 class Mechanize::UnauthorizedError < Mechanize::ResponseCodeError
 
   attr_reader :challenges

data/lib/mechanize/unsupported_scheme_error.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 class Mechanize::UnsupportedSchemeError < Mechanize::Error
   attr_accessor :scheme, :uri
 

data/lib/mechanize/util.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 require 'cgi'
 require 'nkf'
 
@@ -15,7 +16,7 @@ class Mechanize::Util
     def build_query_string(parameters, enc = nil)
       each_parameter(parameters).inject(nil) { |s, (k, v)|
         # WEBrick::HTTP.escape* has some problems about m17n on ruby-1.9.*.
-        (s.nil? ? '' : s << '&') << [CGI.escape(k.to_s), CGI.escape(v.to_s)].join('=')
+        (s.nil? ? String.new : s << '&') << [CGI.escape(k.to_s), CGI.escape(v.to_s)].join('=')
       } || ''
     end
 

data/lib/mechanize/version.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 class Mechanize
-  VERSION = "2.7.7"
+  VERSION = "2.8.0"
 end

data/lib/mechanize/xml_file.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 ##
 # This class encapsulates an XML file. If Mechanize finds a content-type
 # of 'text/xml' or 'application/xml' this class will be instantiated and

data/mechanize.gemspec

@@ -1,20 +1,21 @@
 # coding: utf-8
+# frozen_string_literal: true
 lib = File.expand_path('../lib', __FILE__)
 $LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
 require 'mechanize/version'
 
 Gem::Specification.new do |spec|
-  spec.name          = "mechanize"
-  spec.version       = Mechanize::VERSION
-  spec.homepage      = "http://docs.seattlerb.org/mechanize/"
-  spec.summary       = %q{The Mechanize library is used for automating interaction with websites}
-  spec.description   =
+  spec.name = "mechanize"
+  spec.version = Mechanize::VERSION
+  spec.homepage = "http://docs.seattlerb.org/mechanize/"
+  spec.summary = 'The Mechanize library is used for automating interaction with websites'
+  spec.description =
     [
       "The Mechanize library is used for automating interaction with websites.",
       "Mechanize automatically stores and sends cookies, follows redirects,",
       "and can follow links and submit forms.  Form fields can be populated and",
       "submitted.  Mechanize also keeps track of the sites that you have visited as",
-      "a history."
+      "a history.",
     ].join("\n")
 
   spec.authors =
@@ -23,7 +24,7 @@ Gem::Specification.new do |spec|
       'Aaron Patterson',
       'Mike Dalessio',
       'Akinori MUSHA',
-      'Lee Jarvis'
+      'Lee Jarvis',
     ]
   spec.email =
     [
@@ -31,32 +32,38 @@ Gem::Specification.new do |spec|
       'aaron.patterson@gmail.com',
       'mike.dalessio@gmail.com',
       'knu@idaemons.org',
-      'ljjarvis@gmail.com'
+      'ljjarvis@gmail.com',
     ]
 
-  spec.license           = "MIT"
+  spec.license = "MIT"
 
   spec.require_paths = ["lib"]
-  spec.files         = `git ls-files`.split($/)
-  spec.test_files    = spec.files.grep(%r{^test/})
-
-  spec.extra_rdoc_files += Dir['*.rdoc']
-  spec.rdoc_options     = ["--main", "README.rdoc"]
-
-  spec.required_ruby_version = ">= 1.9.2"
-
-  spec.add_runtime_dependency "net-http-digest_auth", [ ">= 1.1.1", "~> 1.1" ]
-  if RUBY_VERSION >= "2.0"
-    spec.add_runtime_dependency "mime-types", [ ">= 1.17.2" ]
-    spec.add_runtime_dependency "net-http-persistent", [ ">= 2.5.2"]
-  else
-    spec.add_runtime_dependency "mime-types", [ ">= 1.17.2", "< 3" ]
-    spec.add_runtime_dependency "net-http-persistent", [ ">= 2.5.2", "~> 2.5" ]
-  end
-  spec.add_runtime_dependency "http-cookie",          [ "~> 1.0" ]
-  spec.add_runtime_dependency "nokogiri",             [ "~> 1.6" ]
-  spec.add_runtime_dependency "ntlm-http",            [ ">= 0.1.1", "~> 0.1"   ]
-  spec.add_runtime_dependency "webrobots",            [ "< 0.2",    ">= 0.0.9" ]
-  spec.add_runtime_dependency "domain_name",          [ ">= 0.5.1", "~> 0.5"   ]
-  spec.add_runtime_dependency 'webrick', "~> 1.7"
+  spec.files = %x(git ls-files).split($/)
+  spec.test_files = spec.files.grep(%r{^test/})
+
+  spec.extra_rdoc_files += Dir['*.rdoc', '*.md']
+  spec.rdoc_options = ["--main", "README.md"]
+
+  spec.required_ruby_version = ">= 2.5.0"
+
+  spec.add_runtime_dependency("addressable", "~> 2.7")
+  spec.add_runtime_dependency("domain_name", ">= 0.5.20190701", "~> 0.5")
+  spec.add_runtime_dependency("http-cookie", ">= 1.0.3", "~> 1.0")
+  spec.add_runtime_dependency("mime-types", "~> 3.0")
+  spec.add_runtime_dependency("net-http-digest_auth", ">= 1.4.1", "~> 1.4")
+
+  # careful! some folks are relying on older versions of net-http-persistent
+  # - see the socks proxy patch in use at #507 and #464
+  # - see use of retry_change_requests that was removed at #558
+  spec.add_runtime_dependency("net-http-persistent", ">= 2.5.2", "< 5.0.dev")
+
+  spec.add_runtime_dependency("nokogiri", ">= 1.11.2", "~> 1.11")
+  spec.add_runtime_dependency("rubyntlm", ">= 0.6.3", "~> 0.6")
+  spec.add_runtime_dependency("webrick", "~> 1.7")
+  spec.add_runtime_dependency("webrobots", "~> 0.1.2")
+
+  spec.add_development_dependency("minitest", "~> 5.14")
+  spec.add_development_dependency("rake", "~> 13.0")
+  spec.add_development_dependency("rdoc", "~> 6.3")
+  spec.add_development_dependency("rubocop", "~> 1.12")
 end

data/test/htdocs/dir with spaces/foo.html

@@ -1 +1,2 @@
 HELLO
+

data/test/test_mechanize.rb

@@ -346,6 +346,7 @@ but not <a href="/" rel="me nofollow">this</a>!
   end
 
   def test_download_does_not_allow_command_injection
+    skip if windows?
     in_tmpdir do
       @mech.download('http://example', '| ruby -rfileutils -e \'FileUtils.touch("vul.txt")\'')
 
@@ -949,7 +950,7 @@ but not <a href="/" rel="me nofollow">this</a>!
       "Content-Disposition: form-data; name=\"userfile1\"; filename=\"#{name}\"",
       page.body
     )
-    assert_operator page.body.bytesize, :>, File.size(__FILE__)
+    assert_operator page.body.bytesize, :>, file_contents_without_cr(__FILE__).length
   end
 
   def test_post_file_upload_nonascii
@@ -968,7 +969,7 @@ but not <a href="/" rel="me nofollow">this</a>!
       page.body
     )
     assert_match("Content-Type: application/zip", page.body)
-    assert_operator page.body.bytesize, :>, File.size(__FILE__)
+    assert_operator page.body.bytesize, :>, file_contents_without_cr(__FILE__).length
   end
 
   def test_post_file_upload
@@ -987,7 +988,7 @@ but not <a href="/" rel="me nofollow">this</a>!
       page.body
     )
     assert_match("Content-Type: application/zip", page.body)
-    assert_operator page.body.bytesize, :>, File.size(__FILE__)
+    assert_operator page.body.bytesize, :>, file_contents_without_cr(__FILE__).length
   end
 
   def test_post_redirect
@@ -1182,7 +1183,7 @@ but not <a href="/" rel="me nofollow">this</a>!
 
     page = @mech.submit(form)
 
-    contents = File.binread __FILE__
+    contents = File.binread(__FILE__).gsub(/\r\n/, "\n")
     basename = File.basename __FILE__
 
     assert_match(

data/test/test_mechanize_cookie_jar.rb

@@ -502,6 +502,7 @@ class TestMechanizeCookieJar < Mechanize::TestCase
   end
 
   def test_prevent_command_injection_when_saving
+    skip if windows?
     url = URI 'http://rubygems.org/'
     path = '| ruby -rfileutils -e \'FileUtils.touch("vul.txt")\''
 
@@ -514,6 +515,7 @@ class TestMechanizeCookieJar < Mechanize::TestCase
   end
 
   def test_prevent_command_injection_when_loading
+    skip if windows?
     url = URI 'http://rubygems.org/'
     path = '| ruby -rfileutils -e \'FileUtils.touch("vul.txt")\''
 

data/test/test_mechanize_download.rb

@@ -47,6 +47,7 @@ class TestMechanizeDownload < Mechanize::TestCase
   end
 
   def test_save_bang_does_not_allow_command_injection
+    skip if windows?
     uri = URI.parse 'http://example/foo.html'
     body_io = StringIO.new '0123456789'
 

data/test/test_mechanize_file.rb

@@ -104,6 +104,7 @@ class TestMechanizeFile < Mechanize::TestCase
   end
 
   def test_save_bang_does_not_allow_command_injection
+    skip if windows?
     uri = URI 'http://example/test.html'
     page = Mechanize::File.new uri, nil, ''
 

data/test/test_mechanize_file_connection.rb

@@ -3,19 +3,37 @@ require 'mechanize/test_case'
 class TestMechanizeFileConnection < Mechanize::TestCase
 
   def test_request
-    uri = URI.parse "file://#{File.expand_path __FILE__}"
+    file_path = File.expand_path(__FILE__)
+    uri = URI.parse "file://#{file_path}"
     conn = Mechanize::FileConnection.new
 
     body = ''
 
     conn.request uri, nil do |response|
+      assert_equal(file_path, response.file_path)
       response.read_body do |part|
         body << part
       end
     end
 
-    assert_equal File.read(__FILE__), body
+    assert_equal File.read(__FILE__), body.gsub(/\r\n/, "\n")
   end
 
-end
+  def test_request_on_uri_with_windows_drive
+    uri_string = "file://C:/path/to/file.html"
+    expected_file_path = "C:/path/to/file.html"
+
+    uri = URI.parse(uri_string)
+    conn = Mechanize::FileConnection.new
 
+    called = false
+    yielded_file_path = nil
+    conn.request(uri, nil) do |response|
+      called = true
+      yielded_file_path = response.file_path
+    end
+
+    assert(called)
+    assert_equal(expected_file_path, yielded_file_path)
+  end
+end

data/test/test_mechanize_file_response.rb

@@ -1,6 +1,11 @@
 require 'mechanize/test_case'
 
 class TestMechanizeFileResponse < Mechanize::TestCase
+  def test_file_path
+    res = Mechanize::FileResponse.new("/path/to/foo.html")
+    assert_equal("/path/to/foo.html", res.file_path)
+  end
+
   def test_content_type
     Tempfile.open %w[pi .nothtml] do |tempfile|
       res = Mechanize::FileResponse.new tempfile.path
@@ -31,6 +36,7 @@ class TestMechanizeFileResponse < Mechanize::TestCase
   end
 
   def test_read_body_does_not_allow_command_injection
+    skip if windows?
     in_tmpdir do
       FileUtils.touch('| ruby -rfileutils -e \'FileUtils.touch("vul.txt")\'')
       res = Mechanize::FileResponse.new('| ruby -rfileutils -e \'FileUtils.touch("vul.txt")\'')

data/test/test_mechanize_http_agent.rb

@@ -16,11 +16,7 @@ class TestMechanizeHttpAgent < Mechanize::TestCase
     @res.instance_variable_set :@code, 200
     @res.instance_variable_set :@header, {}
 
-    @headers = if RUBY_VERSION >= '2.0.0' then
-                 %w[accept accept-encoding user-agent]
-               else
-                 %w[accept user-agent]
-               end
+    @headers = %w[accept accept-encoding user-agent]
   end
 
   def auth_realm uri, scheme, type
@@ -205,7 +201,7 @@ class TestMechanizeHttpAgent < Mechanize::TestCase
 
     page = @agent.fetch uri
 
-    assert_equal File.read(foo), page.body
+    assert_equal File.read(foo), page.body.gsub(/\r\n/, "\n")
     assert_kind_of Mechanize::Page, page
   end
 
@@ -1508,7 +1504,8 @@ class TestMechanizeHttpAgent < Mechanize::TestCase
     headers = {
       'Range' => 'bytes=0-9999',
       'Content-Type' => 'application/x-www-form-urlencoded',
-      'Content-Length' => '9999',
+      'CONTENT-LENGTH' => '9999',
+      'content-md5' => '14758f1afd44c09b7992073ccf00b43d',
     }
 
     page = fake_page
@@ -1520,6 +1517,7 @@ class TestMechanizeHttpAgent < Mechanize::TestCase
     assert_match 'range|bytes=0-9999', page.body
     refute_match 'content-type|application/x-www-form-urlencoded', page.body
     refute_match 'content-length|9999', page.body
+    refute_match 'content-md5|14758f1afd44c09b7992073ccf00b43d', page.body
   end
 
   def test_response_redirect_malformed
@@ -1555,6 +1553,48 @@ class TestMechanizeHttpAgent < Mechanize::TestCase
     end
   end
 
+  def test_response_redirect_to_cross_site_with_credential
+    @agent.redirect_ok = true
+
+    headers = {
+      'Range' => 'bytes=0-9999',
+      'AUTHORIZATION' => 'Basic xxx',
+      'cookie' => 'name=value',
+    }
+
+    page = html_page ''
+    page = @agent.response_redirect({ 'Location' => 'http://trap/http_headers' }, :get,
+                                    page, 0, headers)
+
+    refute_includes(headers.keys, "AUTHORIZATION")
+    refute_includes(headers.keys, "cookie")
+
+    assert_match 'range|bytes=0-9999', page.body
+    refute_match("authorization|Basic xxx", page.body)
+    refute_match("cookie|name=value", page.body)
+  end
+
+  def test_response_redirect_to_same_site_with_credential
+    @agent.redirect_ok = true
+
+    headers = {
+      'Range' => 'bytes=0-9999',
+      'AUTHORIZATION' => 'Basic xxx',
+      'cookie' => 'name=value',
+    }
+
+    page = html_page ''
+    page = @agent.response_redirect({ 'Location' => '/http_headers' }, :get,
+                                    page, 0, headers)
+
+    assert_includes(headers.keys, "AUTHORIZATION")
+    assert_includes(headers.keys, "cookie")
+
+    assert_match 'range|bytes=0-9999', page.body
+    assert_match("authorization|Basic xxx", page.body)
+    assert_match("cookie|name=value", page.body)
+  end
+
   def test_response_redirect_not_ok
     @agent.redirect_ok = false
 

data/test/test_mechanize_http_content_disposition_parser.rb

@@ -30,6 +30,33 @@ class TestMechanizeHttpContentDispositionParser < Mechanize::TestCase
     assert_equal expected,     content_disposition.parameters
   end
 
+  def test_parse_date_iso8601_fallback
+    now = Time.at Time.now.to_i
+
+    content_disposition = @parser.parse \
+      'attachment;' \
+      'filename=value;' \
+      "creation-date=\"#{now.iso8601}\";" \
+      "modification-date=\"#{(now + 1).iso8601}\""
+
+    assert_equal 'attachment', content_disposition.type
+    assert_equal 'value',      content_disposition.filename
+    assert_equal now,          content_disposition.creation_date
+    assert_equal((now + 1),    content_disposition.modification_date)
+  end
+
+  def test_parse_date_invalid
+    now = Time.at Time.now.to_i
+
+    content_disposition = @parser.parse \
+      'attachment;' \
+      'filename=value;' \
+      "creation-date=\"#{now.to_s}\";" \
+      "modification-date=\"#{(now + 1).to_s}\""
+
+    assert_nil content_disposition
+  end
+
   def test_parse_header
     content_disposition = @parser.parse \
       'content-disposition: attachment;filename=value', true