mechanize 2.7.7 → 2.8.3

data/README.md

@@ -0,0 +1,77 @@
+# Mechanize
+
+* https://www.rubydoc.info/gems/mechanize/
+* https://github.com/sparklemotion/mechanize
+
+[![Test suite](https://github.com/sparklemotion/mechanize/actions/workflows/ci-test.yml/badge.svg)](https://github.com/sparklemotion/mechanize/actions/workflows/ci-test.yml)
+
+
+## Description
+
+The Mechanize library is used for automating interaction with websites. Mechanize automatically stores and sends cookies, follows redirects, and can follow links and submit forms.  Form fields can be populated and submitted. Mechanize also keeps track of the sites that you have visited as a history.
+
+
+## Dependencies
+
+* Ruby >= 2.5
+* Gems:
+  * `addressable`
+  * `domain_name`
+  * `http-cookie`
+  * `mime-types`
+  * `net-http-digest_auth`
+  * `net-http-persistent`
+  * `nokogiri`
+  * `rubyntlm`
+  * `webrick`
+  * `webrobots`
+
+
+## Support:
+
+The bug tracker is available here:
+
+* https://github.com/sparklemotion/mechanize/issues
+
+
+## Examples
+
+If you are just starting, check out [GUIDE.rdoc](https://github.com/sparklemotion/mechanize/blob/main/GUIDE.rdoc) or [EXAMPLES.rdoc](https://github.com/sparklemotion/mechanize/blob/main/EXAMPLES.rdoc).
+
+
+## Developers
+
+Use bundler to install dependencies:
+
+```
+bundle install
+```
+
+Run all tests with:
+
+```
+bundle exec rake test
+```
+
+See also Mechanize::TestCase to read about the built-in testing infrastructure.
+
+
+## Authors
+
+* Eric Hodel
+* Akinori MUSHA
+* Aaron Patterson
+* Lee Jarvis
+* Mike Dalessio
+
+
+## Acknowledgments
+
+This library was heavily influenced by its namesake in the Perl world.  A big
+thanks goes to [Andy Lester](http://petdance.com), the author of the original Perl module WWW::Mechanize which is available [here](http://search.cpan.org/dist/WWW-Mechanize/). Ruby Mechanize would not be around without you!
+
+Thank you to Michael Neumann for starting the Ruby version. Thanks to everyone who's helped out in various ways. Finally, thank you to the people using this library!
+
+## License
+
+This library is distributed under the MIT license. Please see [LICENSE.txt](https://github.com/sparklemotion/mechanize/blob/main/LICENSE.txt).

data/Rakefile

@@ -23,9 +23,9 @@ task('ssl_cert') do |p|
 end
 
 RDoc::Task.new do |rdoc|
-  rdoc.main = "README.rdoc"
+  rdoc.main = "README.md"
   rdoc.rdoc_dir = 'doc'
-  rdoc.rdoc_files.include( "CHANGELOG.rdoc", "EXAMPLES.rdoc", "GUIDE.rdoc", "LICENSE.rdoc", "README.rdoc", "lib/**/*.rb")
+  rdoc.rdoc_files.include( "CHANGELOG.md", "EXAMPLES.rdoc", "GUIDE.rdoc", "LICENSE.txt", "README.md", "lib/**/*.rb")
 end
 
 desc "Run tests"
@@ -38,4 +38,19 @@ task publish_docs: %w[rdoc] do
   sh 'rsync', '-avzO', '--delete', 'doc/', 'docs-push.seattlerb.org:/data/www/docs.seattlerb.org/mechanize/'
 end
 
-task default: :test
+desc "Run rubocop checks"
+task :rubocop => ["rubocop:security", "rubocop:frozen_string_literals"]
+
+namespace "rubocop" do
+  desc "Run rubocop security check"
+  task :security do
+    sh "rubocop lib --only Security"
+  end
+
+  desc "Run rubocop string literals check"
+  task :frozen_string_literals do
+    sh "rubocop lib --auto-correct-all --only Style/FrozenStringLiteralComment"
+  end
+end
+
+task default: [:rubocop, :test]

data/lib/mechanize/chunked_termination_error.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 ##
 # Raised when Mechanize detects the chunked transfer-encoding may be
 # incorrectly terminated.

data/lib/mechanize/content_type_error.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 ##
 # This error is raised when a pluggable parser tries to parse a content type
 # that it does not know how to handle.  For example if Mechanize::Page were to

data/lib/mechanize/cookie.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 warn 'mechanize/cookie will be deprecated.  Please migrate to the http-cookie APIs.' if $VERBOSE
 
 require 'http/cookie'
@@ -50,19 +51,6 @@ class Mechanize
 
   Cookie = ::HTTP::Cookie
 
-  # Compatibility for Ruby 1.8/1.9
-  unless Cookie.respond_to?(:prepend, true)
-    require 'mechanize/prependable'
-
-    class Cookie
-      extend Prependable
-
-      class << self
-        extend Prependable
-      end
-    end
-  end
-
   class Cookie
     prepend CookieIMethods
 

data/lib/mechanize/cookie_jar.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 warn 'mechanize/cookie_jar will be deprecated.  Please migrate to the http-cookie APIs.' if $VERBOSE
 
 require 'http/cookie_jar'
@@ -148,7 +149,7 @@ class Mechanize
       return super(input, opthash) if opthash[:format] != :yaml
 
       begin
-        data = YAML.load(input)
+        data = YAML.load(input) # rubocop:disable Security/YAMLLoad
       rescue ArgumentError
         @logger.warn "unloadable YAML cookie data discarded" if @logger
         return self
@@ -175,15 +176,6 @@ class Mechanize
     end
   end
 
-  # Compatibility for Ruby 1.8/1.9
-  unless ::HTTP::CookieJar.respond_to?(:prepend, true)
-    require 'mechanize/prependable'
-
-    class ::HTTP::CookieJar
-      extend Prependable
-    end
-  end
-
   class ::HTTP::CookieJar
     prepend CookieJarIMethods
   end

data/lib/mechanize/directory_saver.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 ##
 # Unlike Mechanize::FileSaver, the directory saver places all downloaded files
 # in a single pre-specified directory.

data/lib/mechanize/download.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 ##
 # Download is a pluggable parser for downloading files without loading them
 # into memory first.  You may subclass this class to handle content types you

data/lib/mechanize/element_matcher.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 module Mechanize::ElementMatcher
 
   def elements_with singular, plural = "#{singular}s"

data/lib/mechanize/element_not_found_error.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 ##
 # Raised when an an element was not found on the Page
 

data/lib/mechanize/file.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 ##
 # This is the base class for the Pluggable Parsers.  If Mechanize cannot find
 # an appropriate class to use for the content type, this class will be used.

data/lib/mechanize/file_connection.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 ##
 # Wrapper to make a file URI work like an http URI
 
@@ -10,8 +11,9 @@ class Mechanize::FileConnection
   end
 
   def request uri, request
-    yield Mechanize::FileResponse.new Mechanize::Util.uri_unescape uri.path
+    file_path = uri.select(:host, :path)
+                  .select { |part| part && (part.length > 0) }
+                  .join(":")
+    yield Mechanize::FileResponse.new(Mechanize::Util.uri_unescape(file_path))
   end
-
 end
-

data/lib/mechanize/file_request.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 ##
 # A wrapper for a file URI that makes a request that works like a
 # Net::HTTPRequest

data/lib/mechanize/file_response.rb

@@ -1,8 +1,11 @@
+# frozen_string_literal: true
 ##
 # Fake response for dealing with file:/// requests
 
 class Mechanize::FileResponse
 
+  attr_reader :file_path
+
   def initialize(file_path)
     @file_path = file_path
     @uri       = nil

data/lib/mechanize/file_saver.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 ##
 # This is a pluggable parser that automatically saves every file it
 # encounters.  Unlike Mechanize::DirectorySaver, the file saver saves the

data/lib/mechanize/form/button.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 ##
 # A Submit button in a Form
 

data/lib/mechanize/form/check_box.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 ##
 # This class represents a check box found in a Form.  To activate the CheckBox
 # in the Form, set the checked method to true.

data/lib/mechanize/form/field.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 ##
 # This class represents a field in a form.  It handles the following input
 # tags found in a form:

data/lib/mechanize/form/file_upload.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 # This class represents a file upload field found in a form.  To use this
 # class, set FileUpload#file_data= to the data of the file you want to upload
 # and FileUpload#mime_type= to the appropriate mime type of the file.

data/lib/mechanize/form/hidden.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 class Mechanize::Form::Hidden < Mechanize::Form::Field
 end
 

data/lib/mechanize/form/image_button.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 ##
 # This class represents an image button in a form.  Use the x and y methods to
 # set the x and y positions for where the mouse "clicked".

data/lib/mechanize/form/keygen.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 ##
 # This class represents a keygen (public / private key generator) found in a
 # Form. The field will automatically generate a key pair and compute its own

data/lib/mechanize/form/multi_select_list.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 ##
 # This class represents a select list where multiple values can be selected.
 # MultiSelectList#value= accepts an array, and those values are used as

data/lib/mechanize/form/option.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 ##
 # This class contains an option found within SelectList.  A SelectList can
 # have many Option classes associated with it.  An option can be selected by

data/lib/mechanize/form/radio_button.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 ##
 # This class represents a radio button found in a Form.  To activate the
 # RadioButton in the Form, set the checked method to true.

data/lib/mechanize/form/reset.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 class Mechanize::Form::Reset < Mechanize::Form::Button
 end
 

data/lib/mechanize/form/select_list.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 # This class represents a select list or drop down box in a Form.  Set the
 # value for the list by calling SelectList#value=.  SelectList contains a list
 # of Option that were found.  After finding the correct option, set the select

data/lib/mechanize/form/submit.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 class Mechanize::Form::Submit < Mechanize::Form::Button
 end
 

data/lib/mechanize/form/text.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 class Mechanize::Form::Text < Mechanize::Form::Field
 end
 

data/lib/mechanize/form/textarea.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 class Mechanize::Form::Textarea < Mechanize::Form::Field
 end
 

data/lib/mechanize/form.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 require 'mechanize/element_matcher'
 
 # This class encapsulates a form parsed out of an HTML page.  Each type of
@@ -643,15 +644,6 @@ class Mechanize::Form
     end
   end
 
-  unless ::String.method_defined?(:b)
-    # Define String#b for Ruby < 2.0
-    class ::String
-      def b
-        dup.force_encoding(Encoding::ASCII_8BIT)
-      end
-    end
-  end
-
   def rand_string(len = 10)
     chars = ("a".."z").to_a + ("A".."Z").to_a
     string = ::String.new

data/lib/mechanize/headers.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 class Mechanize::Headers < Hash
   def [](key)
     super(key.downcase)

data/lib/mechanize/history.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 ##
 # This class manages history for your mechanize object.
 

data/lib/mechanize/http/agent.rb

@@ -1,6 +1,6 @@
+# frozen_string_literal: true
 require 'tempfile'
 require 'net/ntlm'
-require 'kconv'
 require 'webrobots'
 
 ##
@@ -9,6 +9,9 @@ require 'webrobots'
 
 class Mechanize::HTTP::Agent
 
+  CREDENTIAL_HEADERS = ['Authorization', 'Cookie']
+  POST_HEADERS = ['Content-Length', 'Content-MD5', 'Content-Type']
+
   # :section: Headers
 
   # Disables If-Modified-Since conditional requests (enabled by default)
@@ -838,7 +841,7 @@ class Mechanize::HTTP::Agent
 
     out_io
   rescue Zlib::Error => e
-    message = "error handling content-encoding #{response['Content-Encoding']}:"
+    message = String.new("error handling content-encoding #{response['Content-Encoding']}:")
     message << " #{e.message} (#{e.class})"
     raise Mechanize::Error, message
   ensure
@@ -986,14 +989,20 @@ class Mechanize::HTTP::Agent
 
     redirect_method = method == :head ? :head : :get
 
+    new_uri = secure_resolve!(response['Location'].to_s, page)
+    @history.push(page, page.uri)
+
     # Make sure we are not copying over the POST headers from the original request
-    ['Content-Length', 'Content-MD5', 'Content-Type'].each do |key|
-      headers.delete key
+    POST_HEADERS.each do |key|
+      headers.delete_if { |h| h.casecmp?(key) }
     end
 
-    new_uri = secure_resolve! response['Location'].to_s, page
-
-    @history.push(page, page.uri)
+    # Make sure we clear credential headers if being redirected to another site
+    if new_uri.host != page.uri.host
+      CREDENTIAL_HEADERS.each do |ch|
+        headers.delete_if { |h| h.casecmp?(ch) }
+      end
+    end
 
     fetch new_uri, redirect_method, headers, [], referer, redirects + 1
   end
@@ -1278,4 +1287,3 @@ class Mechanize::HTTP::Agent
 end
 
 require 'mechanize/http/auth_store'
-

data/lib/mechanize/http/auth_challenge.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 class Mechanize::HTTP
 
   AuthChallenge = Struct.new :scheme, :params, :raw

data/lib/mechanize/http/auth_realm.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 class Mechanize::HTTP::AuthRealm
 
   attr_reader :scheme

data/lib/mechanize/http/auth_store.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 ##
 # A credential store for HTTP authentication.
 #

data/lib/mechanize/http/content_disposition_parser.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 # coding: BINARY
 
 require 'strscan'
@@ -16,6 +17,7 @@ end
 # * Missing disposition-type
 # * Multiple semicolons
 # * Whitespace around semicolons
+# * Dates in ISO 8601 format
 
 class Mechanize::HTTP::ContentDispositionParser
 
@@ -93,7 +95,17 @@ class Mechanize::HTTP::ContentDispositionParser
               when /^filename$/ then
                 rfc_2045_value
               when /^(creation|modification|read)-date$/ then
-                Time.rfc822 rfc_2045_quoted_string
+                date = rfc_2045_quoted_string
+
+                begin
+                  Time.rfc822 date
+                rescue ArgumentError
+                  begin
+                    Time.iso8601 date
+                  rescue ArgumentError
+                    nil
+                  end
+                end
               when /^size$/ then
                 rfc_2045_value.to_i(10)
               else
@@ -125,7 +137,7 @@ class Mechanize::HTTP::ContentDispositionParser
   def rfc_2045_quoted_string
     return nil unless @scanner.scan(/"/)
 
-    text = ''
+    text = String.new
 
     while true do
       chunk = @scanner.scan(/[\000-\014\016-\041\043-\133\135-\177]+/) # not \r "

data/lib/mechanize/http/www_authenticate_parser.rb

@@ -1,4 +1,4 @@
-# coding: BINARY
+# frozen_string_literal: true
 
 require 'strscan'
 
@@ -151,10 +151,10 @@ class Mechanize::HTTP::WWWAuthenticateParser
   def quoted_string
     return nil unless @scanner.scan(/"/)
 
-    text = ''
+    text = String.new
 
     while true do
-      chunk = @scanner.scan(/[\r\n \t\041\043-\176\200-\377]+/) # not "
+      chunk = @scanner.scan(/[\r\n \t\x21\x23-\x7e\u0080-\u00ff]+/) # not " which is \x22
 
       if chunk then
         text << chunk

data/lib/mechanize/http.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 ##
 # Mechanize::HTTP contains classes for communicated with HTTP servers.  All
 # API under this namespace is considered private and is subject to change at

data/lib/mechanize/image.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 ##
 # An Image holds downloaded data for an image/* response.
 

data/lib/mechanize/page/base.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 ##
 # A base element on an HTML page.  Mechanize treats base tags just like 'a'
 # tags.  Base objects will contain links, but most likely will have no text.

data/lib/mechanize/page/frame.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 # A Frame object wraps a frame HTML element.  Frame objects can be treated
 # just like Link objects.  They contain #src, the #link they refer to and a
 # #name, the name of the frame they refer to.  #src and #name are aliased to

data/lib/mechanize/page/image.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 ##
 # An image element on an HTML page
 

data/lib/mechanize/page/label.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 ##
 # A form label on an HTML page
 

data/lib/mechanize/page/link.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 ##
 # This class encapsulates links.  It contains the text and the URI for
 # 'a' tags parsed out of an HTML page.  If the link contains an image,
@@ -8,6 +9,8 @@
 #   <a href="http://example">Hello World</a>
 #   <a href="http://example"><img src="test.jpg" alt="Hello World"></a>
 
+require 'addressable/uri'
+
 class Mechanize::Page::Link
   attr_reader :node
   attr_reader :href
@@ -94,7 +97,11 @@ class Mechanize::Page::Link
                begin
                  URI.parse @href
                rescue URI::InvalidURIError
-                 URI.parse WEBrick::HTTPUtils.escape @href
+                 begin
+                   URI.parse(Addressable::URI.escape(@href))
+                 rescue Addressable::URI::InvalidURIError
+                   raise URI::InvalidURIError
+                 end
                end
              end
   end

data/lib/mechanize/page/meta_refresh.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 ##
 # This class encapsulates a meta element with a refresh http-equiv.  Mechanize
 # treats meta refresh elements just like 'a' tags.  MetaRefresh objects will

data/lib/mechanize/page.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 ##
 # This class encapsulates an HTML page.  If Mechanize finds a content
 # type of 'text/html', this class will be instantiated and returned.
@@ -103,9 +104,9 @@ class Mechanize::Page < Mechanize::File
     parser = self.parser unless parser
     return false if parser.errors.empty?
     parser.errors.any? do |error|
-      error.message =~ /(indicate\ encoding)|
-                        (Invalid\ char)|
-                        (input\ conversion\ failed)/x
+      error.message.scrub =~ /(indicate\ encoding)|
+                              (Invalid\ char)|
+                              (input\ conversion\ failed)/x
     end
   end
 

data/lib/mechanize/parser.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 ##
 # The parser module provides standard methods for accessing the headers and
 # content of a response that are shared across pluggable parsers.

data/lib/mechanize/pluggable_parsers.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 require 'mechanize/file'
 require 'mechanize/file_saver'
 require 'mechanize/page'

data/lib/mechanize/prependable.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 # Fake implementation of prepend(), which does not support overriding
 # inherited methods nor methods that are formerly overridden by
 # another invocation of prepend().

data/lib/mechanize/redirect_limit_reached_error.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 ##
 # Raised when too many redirects are sent
 

data/lib/mechanize/redirect_not_get_or_head_error.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 ##
 # Raised when a POST, PUT, or DELETE request results in a redirect
 # see RFC 2616 10.3.2, 10.3.3 http://www.ietf.org/rfc/rfc2616.txt

data/lib/mechanize/response_code_error.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 # This error is raised when Mechanize encounters a response code it does not
 # know how to handle.  Currently, this exception will be thrown if Mechanize
 # encounters response codes other than 200, 301, or 302.  Any other response
@@ -16,7 +17,7 @@ class Mechanize::ResponseCodeError < Mechanize::Error
 
   def to_s
     response_class = Net::HTTPResponse::CODE_TO_OBJ[@response_code]
-    out = "#{@response_code} => #{response_class} "
+    out = String.new("#{@response_code} => #{response_class} ")
     out << "for #{@page.uri} " if @page.respond_to? :uri # may be HTTPResponse
     out << "-- #{super}"
   end