mechanize 2.7.4 → 2.8.1

data/EXAMPLES.rdoc

@@ -24,29 +24,6 @@ example, <code>do ... end.submit</code> is the same as <code>{ ...
     end
   end
 
-== Rubyforge
-
-  require 'rubygems'
-  require 'mechanize'
-
-  a = Mechanize.new
-  a.get('http://rubyforge.org/') do |page|
-    # Click the login link
-    login_page = a.click(page.link_with(:text => /Log In/))
-
-    # Submit the login form
-    my_page = login_page.form_with(:action => '/account/login.php') do |f|
-      f.form_loginname  = ARGV[0]
-      f.form_pw         = ARGV[1]
-    end.click_button
-
-    my_page.links.each do |link|
-      text = link.text.strip
-      next unless text.length > 0
-      puts text
-    end
-  end
-
 == File Upload
 
 Upload a file to flickr.
@@ -129,7 +106,7 @@ This example also demonstrates subclassing Mechanize.
 
   class TestMech < Mechanize
     def process
-      get 'http://rubyforge.org/'
+      get 'http://rubygems.org/'
       search_form = page.forms.first
       search_form.words = 'WWW'
       submit search_form

data/Gemfile

@@ -1,3 +1,3 @@
-source 'https://rubygems.org'
+source "https://rubygems.org"
 
 gemspec

data/{LICENSE.rdoc → LICENSE.txt}

@@ -1,5 +1,9 @@
 (The MIT License)
 
+Copyright (c) 2005 by Michael Neumann (mneumann@ntecs.de)
+
+Copyright (c) 2006-2021 by Eric Hodel, Akinori MUSHA, Aaron Patterson, Lee Jarvis, Mike Dalessio
+
 Permission is hereby granted, free of charge, to any person obtaining
 a copy of this software and associated documentation files (the
 'Software'), to deal in the Software without restriction, including

data/README.md

@@ -0,0 +1,77 @@
+# Mechanize
+
+* https://www.rubydoc.info/gems/mechanize/
+* https://github.com/sparklemotion/mechanize
+
+[![Test suite](https://github.com/sparklemotion/mechanize/actions/workflows/ci-test.yml/badge.svg)](https://github.com/sparklemotion/mechanize/actions/workflows/ci-test.yml)
+
+
+## Description
+
+The Mechanize library is used for automating interaction with websites. Mechanize automatically stores and sends cookies, follows redirects, and can follow links and submit forms.  Form fields can be populated and submitted. Mechanize also keeps track of the sites that you have visited as a history.
+
+
+## Dependencies
+
+* Ruby >= 2.5
+* Gems:
+  * `addressable`
+  * `domain_name`
+  * `http-cookie`
+  * `mime-types`
+  * `net-http-digest_auth`
+  * `net-http-persistent`
+  * `nokogiri`
+  * `rubyntlm`
+  * `webrick`
+  * `webrobots`
+
+
+## Support:
+
+The bug tracker is available here:
+
+* https://github.com/sparklemotion/mechanize/issues
+
+
+## Examples
+
+If you are just starting, check out [GUIDE.rdoc](https://github.com/sparklemotion/mechanize/blob/main/GUIDE.rdoc) or [EXAMPLES.rdoc](https://github.com/sparklemotion/mechanize/blob/main/EXAMPLES.rdoc).
+
+
+## Developers
+
+Use bundler to install dependencies:
+
+```
+bundle install
+```
+
+Run all tests with:
+
+```
+bundle exec rake test
+```
+
+See also Mechanize::TestCase to read about the built-in testing infrastructure.
+
+
+## Authors
+
+* Eric Hodel
+* Akinori MUSHA
+* Aaron Patterson
+* Lee Jarvis
+* Mike Dalessio
+
+
+## Acknowledgments
+
+This library was heavily influenced by its namesake in the Perl world.  A big
+thanks goes to [Andy Lester](http://petdance.com), the author of the original Perl module WWW::Mechanize which is available [here](http://search.cpan.org/dist/WWW-Mechanize/). Ruby Mechanize would not be around without you!
+
+Thank you to Michael Neumann for starting the Ruby version. Thanks to everyone who's helped out in various ways. Finally, thank you to the people using this library!
+
+## License
+
+This library is distributed under the MIT license. Please see [LICENSE.txt](https://github.com/sparklemotion/mechanize/blob/main/LICENSE.txt).

data/Rakefile

@@ -23,9 +23,9 @@ task('ssl_cert') do |p|
 end
 
 RDoc::Task.new do |rdoc|
-  rdoc.main = "README.rdoc"
+  rdoc.main = "README.md"
   rdoc.rdoc_dir = 'doc'
-  rdoc.rdoc_files.include( "CHANGELOG.rdoc", "EXAMPLES.rdoc", "GUIDE.rdoc", "LICENSE.rdoc", "Manifest.txt", "README.rdoc", "lib/**/*.rb")
+  rdoc.rdoc_files.include( "CHANGELOG.md", "EXAMPLES.rdoc", "GUIDE.rdoc", "LICENSE.txt", "README.md", "lib/**/*.rb")
 end
 
 desc "Run tests"
@@ -38,4 +38,19 @@ task publish_docs: %w[rdoc] do
   sh 'rsync', '-avzO', '--delete', 'doc/', 'docs-push.seattlerb.org:/data/www/docs.seattlerb.org/mechanize/'
 end
 
-task default: :test
+desc "Run rubocop checks"
+task :rubocop => ["rubocop:security", "rubocop:frozen_string_literals"]
+
+namespace "rubocop" do
+  desc "Run rubocop security check"
+  task :security do
+    sh "rubocop lib --only Security"
+  end
+
+  desc "Run rubocop string literals check"
+  task :frozen_string_literals do
+    sh "rubocop lib --auto-correct-all --only Style/FrozenStringLiteralComment"
+  end
+end
+
+task default: [:rubocop, :test]

data/examples/rubygems.rb

@@ -1,4 +1,4 @@
-# This example logs a user in to rubyforge and prints out the body of the
+# This example logs a user in to rubygems and prints out the body of the
 # page after logging the user in.
 require 'rubygems'
 require 'mechanize'
@@ -9,7 +9,7 @@ mech = Mechanize.new
 mech.log = Logger.new $stderr
 mech.agent.http.debug_output = $stderr
 
-# Load the rubyforge website
+# Load the rubygems website
 page = mech.get('https://rubygems.org/')
 page = mech.click page.link_with(:text => /Sign in/) # Click the login link
 form = page.forms[1] # Select the first form

data/lib/mechanize.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 require 'mechanize/version'
 require 'fileutils'
 require 'forwardable'
@@ -175,7 +176,7 @@ class Mechanize
   # as SSL parameters or proxies:
   #
   #   agent = Mechanize.new do |a|
-  #     a.proxy_host = 'proxy.example'
+  #     a.proxy_addr = 'proxy.example'
   #     a.proxy_port = 8080
   #   end
   #
@@ -396,7 +397,7 @@ class Mechanize
     io = if io_or_filename.respond_to? :write then
            io_or_filename
          else
-           open io_or_filename, 'wb'
+           ::File.open(io_or_filename, 'wb')
          end
 
     case page

data/lib/mechanize/chunked_termination_error.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 ##
 # Raised when Mechanize detects the chunked transfer-encoding may be
 # incorrectly terminated.

data/lib/mechanize/content_type_error.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 ##
 # This error is raised when a pluggable parser tries to parse a content type
 # that it does not know how to handle.  For example if Mechanize::Page were to

data/lib/mechanize/cookie.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 warn 'mechanize/cookie will be deprecated.  Please migrate to the http-cookie APIs.' if $VERBOSE
 
 require 'http/cookie'
@@ -50,19 +51,6 @@ class Mechanize
 
   Cookie = ::HTTP::Cookie
 
-  # Compatibility for Ruby 1.8/1.9
-  unless Cookie.respond_to?(:prepend, true)
-    require 'mechanize/prependable'
-
-    class Cookie
-      extend Prependable
-
-      class << self
-        extend Prependable
-      end
-    end
-  end
-
   class Cookie
     prepend CookieIMethods
 

data/lib/mechanize/cookie_jar.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 warn 'mechanize/cookie_jar will be deprecated.  Please migrate to the http-cookie APIs.' if $VERBOSE
 
 require 'http/cookie_jar'
@@ -65,7 +66,7 @@ class Mechanize
   class CookieJar < ::HTTP::CookieJar
     def save(output, *options)
       output.respond_to?(:write) or
-        return open(output, 'w') { |io| save(io, *options) }
+        return ::File.open(output, 'w') { |io| save(io, *options) }
 
       opthash = {
         :format => :yaml,
@@ -119,7 +120,7 @@ class Mechanize
 
     def load(input, *options)
       input.respond_to?(:write) or
-        return open(input, 'r') { |io| load(io, *options) }
+        return ::File.open(input, 'r') { |io| load(io, *options) }
 
       opthash = {
         :format => :yaml,
@@ -148,7 +149,7 @@ class Mechanize
       return super(input, opthash) if opthash[:format] != :yaml
 
       begin
-        data = YAML.load(input)
+        data = YAML.load(input) # rubocop:disable Security/YAMLLoad
       rescue ArgumentError
         @logger.warn "unloadable YAML cookie data discarded" if @logger
         return self
@@ -175,15 +176,6 @@ class Mechanize
     end
   end
 
-  # Compatibility for Ruby 1.8/1.9
-  unless ::HTTP::CookieJar.respond_to?(:prepend, true)
-    require 'mechanize/prependable'
-
-    class ::HTTP::CookieJar
-      extend Prependable
-    end
-  end
-
   class ::HTTP::CookieJar
     prepend CookieJarIMethods
   end

data/lib/mechanize/directory_saver.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 ##
 # Unlike Mechanize::FileSaver, the directory saver places all downloaded files
 # in a single pre-specified directory.

data/lib/mechanize/download.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 ##
 # Download is a pluggable parser for downloading files without loading them
 # into memory first.  You may subclass this class to handle content types you
@@ -71,7 +72,7 @@ class Mechanize::Download
     dirname = File.dirname filename
     FileUtils.mkdir_p dirname
 
-    open filename, 'wb' do |io|
+    ::File.open(filename, 'wb')do |io|
       until @body_io.eof? do
         io.write @body_io.read 16384
       end

data/lib/mechanize/element_matcher.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 module Mechanize::ElementMatcher
 
   def elements_with singular, plural = "#{singular}s"
@@ -15,6 +16,9 @@ module Mechanize::ElementMatcher
               h[:dom_class] = v
             when :search, :xpath, :css
               if v
+                if method
+                  warn "multiple search selectors are given; previous selector (\#{method}: \#{selector.inspect}) is ignored."
+                end
                 selector = v
                 method = k
               end
@@ -26,7 +30,7 @@ module Mechanize::ElementMatcher
 
         f = select_#{plural}(selector, method).find_all do |thing|
           criteria.all? do |k,v|
-            v === thing.send(k)
+            v === thing.__send__(k)
           end
         end
         yield f if block_given?

data/lib/mechanize/element_not_found_error.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 ##
 # Raised when an an element was not found on the Page
 

data/lib/mechanize/file.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 ##
 # This is the base class for the Pluggable Parsers.  If Mechanize cannot find
 # an appropriate class to use for the content type, this class will be used.
@@ -82,7 +83,7 @@ class Mechanize::File
     dirname = File.dirname filename
     FileUtils.mkdir_p dirname
 
-    open filename, 'wb' do |f|
+    ::File.open(filename, 'wb')do |f|
       f.write body
     end
 

data/lib/mechanize/file_connection.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 ##
 # Wrapper to make a file URI work like an http URI
 
@@ -10,8 +11,9 @@ class Mechanize::FileConnection
   end
 
   def request uri, request
-    yield Mechanize::FileResponse.new Mechanize::Util.uri_unescape uri.path
+    file_path = uri.select(:host, :path)
+                  .select { |part| part && (part.length > 0) }
+                  .join(":")
+    yield Mechanize::FileResponse.new(Mechanize::Util.uri_unescape(file_path))
   end
-
 end
-

data/lib/mechanize/file_request.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 ##
 # A wrapper for a file URI that makes a request that works like a
 # Net::HTTPRequest

data/lib/mechanize/file_response.rb

@@ -1,8 +1,11 @@
+# frozen_string_literal: true
 ##
 # Fake response for dealing with file:/// requests
 
 class Mechanize::FileResponse
 
+  attr_reader :file_path
+
   def initialize(file_path)
     @file_path = file_path
     @uri       = nil
@@ -15,7 +18,7 @@ class Mechanize::FileResponse
     if directory?
       yield dir_body
     else
-      open @file_path, 'rb' do |io|
+      ::File.open(@file_path, 'rb') do |io|
         yield io.read
       end
     end

data/lib/mechanize/file_saver.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 ##
 # This is a pluggable parser that automatically saves every file it
 # encounters.  Unlike Mechanize::DirectorySaver, the file saver saves the

data/lib/mechanize/form.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 require 'mechanize/element_matcher'
 
 # This class encapsulates a form parsed out of an HTML page.  Each type of
@@ -16,7 +17,7 @@ require 'mechanize/element_matcher'
 #   puts form['name']
 
 class Mechanize::Form
-
+  extend Forwardable
   extend Mechanize::ElementMatcher
 
   attr_accessor :method, :action, :name
@@ -35,12 +36,13 @@ class Mechanize::Form
 
   alias :elements :fields
 
-  attr_reader :form_node
+  attr_reader :node
+  alias form_node node  # for backward compatibility
   attr_reader :page
 
   def initialize(node, mech = nil, page = nil)
     @enctype = node['enctype'] || 'application/x-www-form-urlencoded'
-    @form_node        = node
+    @node             = node
     @action           = Mechanize::Util.html_unescape(node['action'])
     @method           = (node['method'] || 'GET').upcase
     @name             = node['name']
@@ -55,24 +57,24 @@ class Mechanize::Form
 
   # Returns whether or not the form contains a field with +field_name+
   def has_field?(field_name)
-    fields.find { |f| f.name == field_name }
+    fields.any? { |f| f.name == field_name }
   end
 
   alias :has_key? :has_field?
 
   # Returns whether or not the form contains a field with +value+
   def has_value?(value)
-    fields.find { |f| f.value == value }
+    fields.any? { |f| f.value == value }
   end
 
   # Returns all field names (keys) for this form
   def keys
-    fields.map { |f| f.name }
+    fields.map(&:name)
   end
 
   # Returns all field values for this form
   def values
-    fields.map { |f| f.value }
+    fields.map(&:value)
   end
 
   # Returns all buttons of type Submit
@@ -136,7 +138,7 @@ class Mechanize::Form
   # Note that you can also use +:id+ to get to this method:
   #   page.form_with(:id => "foorm")
   def dom_id
-    form_node['id']
+    @node['id']
   end
 
   # This method is a shortcut to get form's DOM class.
@@ -144,10 +146,57 @@ class Mechanize::Form
   #   page.form_with(:dom_class => "foorm")
   # Note that you can also use +:class+ to get to this method:
   #   page.form_with(:class => "foorm")
+  # However, attribute values are compared literally as string, so
+  # form_with(class: "a") does not match a form with class="a b".
+  # Use form_with(css: "form.a") instead.
   def dom_class
-    form_node['class']
+    @node['class']
   end
 
+  ##
+  # :method: search
+  #
+  # Shorthand for +node.search+.
+  #
+  # See Nokogiri::XML::Node#search for details.
+
+  ##
+  # :method: css
+  #
+  # Shorthand for +node.css+.
+  #
+  # See also Nokogiri::XML::Node#css for details.
+
+  ##
+  # :method: xpath
+  #
+  # Shorthand for +node.xpath+.
+  #
+  # See also Nokogiri::XML::Node#xpath for details.
+
+  ##
+  # :method: at
+  #
+  # Shorthand for +node.at+.
+  #
+  # See also Nokogiri::XML::Node#at for details.
+
+  ##
+  # :method: at_css
+  #
+  # Shorthand for +node.at_css+.
+  #
+  # See also Nokogiri::XML::Node#at_css for details.
+
+  ##
+  # :method: at_xpath
+  #
+  # Shorthand for +node.at_xpath+.
+  #
+  # See also Nokogiri::XML::Node#at_xpath for details.
+
+  def_delegators :node, :search, :css, :xpath, :at, :at_css, :at_xpath
+
   # Add a field with +field_name+ and +value+
   def add_field!(field_name, value = nil)
     fields << Field.new({'name' => field_name}, value)
@@ -207,7 +256,7 @@ class Mechanize::Form
 
   # Treat form fields like accessors.
   def method_missing(meth, *args)
-    method = meth.to_s.gsub(/=$/, '')
+    (method = meth.to_s).chomp!('=')
 
     if field(method)
       return field(method).value if args.empty?
@@ -257,7 +306,7 @@ class Mechanize::Form
     successful_controls = []
 
     (fields + checkboxes).reject do |f|
-      f.node["disabled"]
+      f.node["disabled"] || f.node["name"] == ""
     end.sort.each do |f|
       case f
       when Mechanize::Form::CheckBox
@@ -278,10 +327,10 @@ class Mechanize::Form
 
     # take one radio button from each group
     radio_groups.each_value do |g|
-      checked = g.select {|f| f.checked}
+      checked = g.select(&:checked)
 
       if checked.uniq.size > 1 then
-        values = checked.map { |button| button.value }.join(', ').inspect
+        values = checked.map(&:value).join(', ').inspect
         name = checked.first.name.inspect
         raise Mechanize::Error,
               "radiobuttons #{values} are checked in the #{name} group, " \
@@ -319,7 +368,7 @@ class Mechanize::Form
   # This method adds a button to the query.  If the form needs to be
   # submitted with multiple buttons, pass each button to this method.
   def add_button_to_query(button)
-    unless button.node.document == @form_node.document then
+    unless button.node.document == @node.document then
       message =
         "#{button.inspect} does not belong to the same page as " \
         "the form #{@name.inspect} in #{@page.uri}"
@@ -337,6 +386,8 @@ class Mechanize::Form
     @clicked_buttons = []
   end
 
+  CRLF = "\r\n".freeze
+
   # This method calculates the request data to be sent back to the server
   # for this form, depending on if this is a regular post, get, or a
   # multi-part post,
@@ -348,16 +399,23 @@ class Mechanize::Form
       boundary = rand_string(20)
       @enctype = "multipart/form-data; boundary=#{boundary}"
 
-      params = query_params.map do |k,v|
-        param_to_multipart(k, v) if k
-      end.compact
+      delimiter = "--#{boundary}\r\n"
 
-      params.concat @file_uploads.map { |f| file_to_multipart(f) }
+      data = ::String.new
 
-      params.map do |part|
-        "--#{boundary}\r\n#{part.force_encoding(Encoding::ASCII_8BIT)}"
-      end.join('') +
-        "--#{boundary}--\r\n"
+      query_params.each do |k,v|
+        if k
+          data << delimiter
+          param_to_multipart(k, v, data)
+        end
+      end
+
+      @file_uploads.each do |f|
+        data << delimiter
+        file_to_multipart(f, data)
+      end
+
+      data << "--#{boundary}--\r\n"
     else
       Mechanize::Util.build_query_string(query_params)
     end
@@ -526,7 +584,7 @@ class Mechanize::Form
     @checkboxes   = []
 
     # Find all input tags
-    form_node.search('input').each do |node|
+    @node.search('input').each do |node|
       type = (node['type'] || 'text').downcase
       name = node['name']
       next if name.nil? && !%w[submit button image].include?(type)
@@ -557,13 +615,13 @@ class Mechanize::Form
     end
 
     # Find all textarea tags
-    form_node.search('textarea').each do |node|
+    @node.search('textarea').each do |node|
       next unless node['name']
       @fields << Textarea.new(node, node.inner_text)
     end
 
     # Find all select tags
-    form_node.search('select').each do |node|
+    @node.search('select').each do |node|
       next unless node['name']
       if node.has_attribute? 'multiple'
         @fields << MultiSelectList.new(node)
@@ -574,61 +632,70 @@ class Mechanize::Form
 
     # Find all submit button tags
     # FIXME: what can I do with the reset buttons?
-    form_node.search('button').each do |node|
+    @node.search('button').each do |node|
       type = (node['type'] || 'submit').downcase
       next if type == 'reset'
       @buttons << Button.new(node)
     end
 
     # Find all keygen tags
-    form_node.search('keygen').each do |node|
+    @node.search('keygen').each do |node|
       @fields << Keygen.new(node, node['value'] || '')
     end
   end
 
   def rand_string(len = 10)
     chars = ("a".."z").to_a + ("A".."Z").to_a
-    string = ""
+    string = ::String.new
     1.upto(len) { |i| string << chars[rand(chars.size-1)] }
     string
   end
 
   def mime_value_quote(str)
-    str.gsub(/(["\r\\])/){|s| '\\' + s}
+    str.b.gsub(/(["\r\\])/, '\\\\\1')
   end
 
-  def param_to_multipart(name, value)
-    return "Content-Disposition: form-data; name=\"" +
-      "#{mime_value_quote(name)}\"\r\n" +
-      "\r\n#{value}\r\n"
+  def param_to_multipart(name, value, buf = ::String.new)
+    buf <<
+      "Content-Disposition: form-data; name=\"".freeze <<
+      mime_value_quote(name) <<
+      "\"\r\n\r\n".freeze <<
+      value.b <<
+      CRLF
   end
 
-  def file_to_multipart(file)
+  def file_to_multipart(file, buf = ::String.new)
     file_name = file.file_name ? ::File.basename(file.file_name) : ''
-    body =  "Content-Disposition: form-data; name=\"" +
-      "#{mime_value_quote(file.name)}\"; " +
-      "filename=\"#{mime_value_quote(file_name)}\"\r\n" +
-      "Content-Transfer-Encoding: binary\r\n"
+
+    body = buf <<
+           "Content-Disposition: form-data; name=\"".freeze <<
+           mime_value_quote(file.name) <<
+           "\"; filename=\"".freeze <<
+           mime_value_quote(file_name) <<
+           "\"\r\nContent-Transfer-Encoding: binary\r\n".freeze
 
     if file.file_data.nil? and file.file_name
-      file.file_data = open(file.file_name, "rb") { |f| f.read }
+      file.file_data = File.binread(file.file_name)
       file.mime_type =
         WEBrick::HTTPUtils.mime_type(file.file_name,
                                      WEBrick::HTTPUtils::DefaultMimeTypes)
     end
 
     if file.mime_type
-      body << "Content-Type: #{file.mime_type}\r\n"
+      body << "Content-Type: ".freeze << file.mime_type << CRLF
     end
 
-    body <<
-      if file.file_data.respond_to? :read
-        "\r\n#{file.file_data.read}\r\n"
+    body << CRLF
+
+    if file_data = file.file_data
+      if file_data.respond_to? :read
+        body << file_data.read.force_encoding(Encoding::ASCII_8BIT)
       else
-        "\r\n#{file.file_data}\r\n"
+        body << file_data.b
       end
+    end
 
-    body
+    body << CRLF
   end
 end