mechanize 2.7.6 → 2.12.2

data/EXAMPLES.rdoc

@@ -24,29 +24,6 @@ example, <code>do ... end.submit</code> is the same as <code>{ ...
     end
   end
 
-== Rubyforge
-
-  require 'rubygems'
-  require 'mechanize'
-
-  a = Mechanize.new
-  a.get('http://rubyforge.org/') do |page|
-    # Click the login link
-    login_page = a.click(page.link_with(:text => /Log In/))
-
-    # Submit the login form
-    my_page = login_page.form_with(:action => '/account/login.php') do |f|
-      f.form_loginname  = ARGV[0]
-      f.form_pw         = ARGV[1]
-    end.click_button
-
-    my_page.links.each do |link|
-      text = link.text.strip
-      next unless text.length > 0
-      puts text
-    end
-  end
-
 == File Upload
 
 Upload a file to flickr.
@@ -129,7 +106,7 @@ This example also demonstrates subclassing Mechanize.
 
   class TestMech < Mechanize
     def process
-      get 'http://rubyforge.org/'
+      get 'http://rubygems.org/'
       search_form = page.forms.first
       search_form.words = 'WWW'
       submit search_form

data/Gemfile

@@ -1,6 +1,12 @@
-source 'https://rubygems.org'
-
-# In order to be able to install on 1.9.3 and 2.0.0
-ruby RUBY_VERSION
+source "https://rubygems.org"
 
 gemspec
+
+gem "minitest", "~> 5.14"
+gem "rake", "~> 13.0"
+gem "rdoc", "~> 6.3"
+gem "rubocop", "~> 1.12"
+unless RUBY_PLATFORM == 'java'
+  gem 'brotli', '>= 0.5'
+  gem 'zstd-ruby', '~> 1.5'
+end

data/{LICENSE.rdoc → LICENSE.txt}

@@ -1,5 +1,9 @@
 (The MIT License)
 
+Copyright (c) 2005 by Michael Neumann (mneumann@ntecs.de)
+
+Copyright (c) 2006-2021 by Eric Hodel, Akinori MUSHA, Aaron Patterson, Lee Jarvis, Mike Dalessio
+
 Permission is hereby granted, free of charge, to any person obtaining
 a copy of this software and associated documentation files (the
 'Software'), to deal in the Software without restriction, including

data/README.md

@@ -0,0 +1,77 @@
+# Mechanize
+
+* https://www.rubydoc.info/gems/mechanize/
+* https://github.com/sparklemotion/mechanize
+
+[![Test suite](https://github.com/sparklemotion/mechanize/actions/workflows/ci.yml/badge.svg)](https://github.com/sparklemotion/mechanize/actions/workflows/ci.yml)
+
+
+## Description
+
+The Mechanize library is used for automating interaction with websites. Mechanize automatically stores and sends cookies, follows redirects, and can follow links and submit forms.  Form fields can be populated and submitted. Mechanize also keeps track of the sites that you have visited as a history.
+
+
+## Dependencies
+
+* Ruby >= 2.6
+* Gems:
+  * `addressable`
+  * `domain_name`
+  * `http-cookie`
+  * `mime-types`
+  * `net-http-digest_auth`
+  * `net-http-persistent`
+  * `nokogiri`
+  * `rubyntlm`
+  * `webrick`
+  * `webrobots`
+
+
+## Support:
+
+The bug tracker is available here:
+
+* https://github.com/sparklemotion/mechanize/issues
+
+
+## Examples
+
+If you are just starting, check out [GUIDE.rdoc](https://github.com/sparklemotion/mechanize/blob/main/GUIDE.rdoc) or [EXAMPLES.rdoc](https://github.com/sparklemotion/mechanize/blob/main/EXAMPLES.rdoc).
+
+
+## Developers
+
+Use bundler to install dependencies:
+
+```
+bundle install
+```
+
+Run all tests with:
+
+```
+bundle exec rake test
+```
+
+See also Mechanize::TestCase to read about the built-in testing infrastructure.
+
+
+## Authors
+
+* Eric Hodel
+* Akinori MUSHA
+* Aaron Patterson
+* Lee Jarvis
+* Mike Dalessio
+
+
+## Acknowledgments
+
+This library was heavily influenced by its namesake in the Perl world.  A big
+thanks goes to [Andy Lester](http://petdance.com), the author of the original Perl module WWW::Mechanize which is available [here](http://search.cpan.org/dist/WWW-Mechanize/). Ruby Mechanize would not be around without you!
+
+Thank you to Michael Neumann for starting the Ruby version. Thanks to everyone who's helped out in various ways. Finally, thank you to the people using this library!
+
+## License
+
+This library is distributed under the MIT license. Please see [LICENSE.txt](https://github.com/sparklemotion/mechanize/blob/main/LICENSE.txt).

data/Rakefile

@@ -23,9 +23,9 @@ task('ssl_cert') do |p|
 end
 
 RDoc::Task.new do |rdoc|
-  rdoc.main = "README.rdoc"
+  rdoc.main = "README.md"
   rdoc.rdoc_dir = 'doc'
-  rdoc.rdoc_files.include( "CHANGELOG.rdoc", "EXAMPLES.rdoc", "GUIDE.rdoc", "LICENSE.rdoc", "README.rdoc", "lib/**/*.rb")
+  rdoc.rdoc_files.include( "CHANGELOG.md", "EXAMPLES.rdoc", "GUIDE.rdoc", "LICENSE.txt", "README.md", "lib/**/*.rb")
 end
 
 desc "Run tests"
@@ -38,4 +38,19 @@ task publish_docs: %w[rdoc] do
   sh 'rsync', '-avzO', '--delete', 'doc/', 'docs-push.seattlerb.org:/data/www/docs.seattlerb.org/mechanize/'
 end
 
-task default: :test
+desc "Run rubocop checks"
+task :rubocop => ["rubocop:security", "rubocop:frozen_string_literals"]
+
+namespace "rubocop" do
+  desc "Run rubocop security check"
+  task :security do
+    sh "rubocop lib --only Security"
+  end
+
+  desc "Run rubocop string literals check"
+  task :frozen_string_literals do
+    sh "rubocop lib test --auto-correct-all --only Style/FrozenStringLiteralComment"
+  end
+end
+
+task default: [:rubocop, :test]

data/examples/latest_user_agents.rb

@@ -0,0 +1,100 @@
+require 'mechanize'
+require 'ostruct'
+
+# LatestUAFetcher fetches latest user agents from `WhatIsMyBrowser.com`.
+# It can use to update `Mechanize::AGENT_ALIASES`.
+class LatestUAFetcher
+  attr_reader :user_agents
+
+  USER_AGENT_TYPES = OpenStruct.new(
+    linux_firefox: "Linux Firefox",
+    mac_firefox: "Mac Firefox",
+    mac_safari: "Mac Safari",
+    windows_chrome: "Windows Chrome",
+    windows_edge: "Windows Edge",
+    windows_firefox: "Windows Firefox",
+    android: "Android",
+    iphone: "iPhone",
+    ipad: "iPad",
+  )
+
+  BASE_URL = 'https://www.whatismybrowser.com/guides/the-latest-user-agent'
+
+  def initialize
+    @agent = Mechanize.new.tap { |a| a.user_agent_alias = 'Mac Firefox' }
+    @user_agents = {}
+  end
+
+  def run
+    return unless user_agents.empty?
+
+    sleep_time = 1
+
+    fetch_user_agents('chrome')
+    fetch_user_agents('firefox')
+    fetch_user_agents('safari')
+    fetch_user_agents('edge')
+  end
+
+  def ordered_user_agents
+    USER_AGENT_TYPES.to_h.values.each_with_object({}) do |type, ordered_user_agents|
+      ordered_user_agents[type] = user_agents[type]
+    end
+  end
+
+  private
+
+  def fetch_user_agents(browser_name, sleep_time = 1)
+    puts "fetch #{browser_name} UA..."
+    send(browser_name)
+    puts "sleeping... (#{sleep_time}s)"
+    sleep sleep_time
+  end
+
+  def edge
+    page = @agent.get("#{BASE_URL}/edge")
+
+    windows_dom = page.css("h2:contains('Latest Edge on Windows User Agents')")
+
+    @user_agents[USER_AGENT_TYPES.windows_edge] = windows_dom.css('+ .listing-of-useragents .code').first.text
+  end
+
+  def firefox
+    page = @agent.get("#{BASE_URL}/firefox")
+
+    desktop_dom = page.css("h2:contains('Latest Firefox on Desktop User Agents')")
+    table_dom = desktop_dom.css('+ .listing-of-useragents')
+
+    @user_agents[USER_AGENT_TYPES.linux_firefox] = table_dom.css('td:contains("Linux")').css("+ td .code:contains('Ubuntu; Linux x86_64')").text
+    @user_agents[USER_AGENT_TYPES.windows_firefox] = table_dom.css('td:contains("Windows")').css('+ td .code').text
+    @user_agents[USER_AGENT_TYPES.mac_firefox] = table_dom.css('td:contains("Macos")').css('+ td .code').text
+  end
+
+  def safari
+    page = @agent.get("#{BASE_URL}/safari")
+
+    macos_dom = page.css("h2:contains('Latest Safari on macOS User Agents')")
+    ios_dom = page.css("h2:contains('Latest Safari on iOS User Agents')")
+
+    @user_agents[USER_AGENT_TYPES.mac_safari] = macos_dom.css('+ .listing-of-useragents .code').first.text
+    @user_agents[USER_AGENT_TYPES.iphone] = ios_dom.css('+ .listing-of-useragents').css("tr:contains('Iphone') .code").text
+    @user_agents[USER_AGENT_TYPES.ipad] = ios_dom.css('+ .listing-of-useragents').css("tr:contains('Ipad') .code").text
+  end
+
+  def chrome
+    page = @agent.get("#{BASE_URL}/chrome")
+
+    windows_dom = page.css("h2:contains('Latest Chrome on Windows 10 User Agents')")
+    android_dom = page.css("h2:contains('Latest Chrome on Android User Agents')")
+
+    @user_agents[USER_AGENT_TYPES.windows_chrome] = windows_dom.css('+ .listing-of-useragents .code').first.text
+    @user_agents[USER_AGENT_TYPES.android] = android_dom.css('+ .listing-of-useragents .code').first.text
+  end
+end
+
+if $0 == __FILE__
+  agent = LatestUAFetcher.new
+  agent.run
+
+  pp agent.ordered_user_agents
+end

data/examples/rubygems.rb

@@ -1,4 +1,4 @@
-# This example logs a user in to rubyforge and prints out the body of the
+# This example logs a user in to rubygems and prints out the body of the
 # page after logging the user in.
 require 'rubygems'
 require 'mechanize'
@@ -9,7 +9,7 @@ mech = Mechanize.new
 mech.log = Logger.new $stderr
 mech.agent.http.debug_output = $stderr
 
-# Load the rubyforge website
+# Load the rubygems website
 page = mech.get('https://rubygems.org/')
 page = mech.click page.link_with(:text => /Sign in/) # Click the login link
 form = page.forms[1] # Select the first form

data/examples/wikipedia_links_to_philosophy.rb

@@ -58,10 +58,10 @@ class WikipediaLinksToPhilosophy
   # the article.
 
   def follow_first_link
-    puts @title
+    puts "#{@title} (#{@page.uri})"
 
     # > p > a rejects italics
-    links = @page.root.css('.mw-content-ltr > p > a[href^="/wiki/"]')
+    links = @page.root.css('.mw-content-ltr p > a[href^="/wiki/"]')
 
     # reject disambiguation and special pages, images and files
     links = links.reject do |link_node|
@@ -74,10 +74,9 @@ class WikipediaLinksToPhilosophy
 
     link = links.first
 
-    unless link then
-      # disambiguation page? try the first item in the list
-      link =
-        @page.root.css('.mw-content-ltr > ul > li > a[href^="/wiki/"]').first
+    if link.nil?
+      puts "Could not parse #{@page.uri}"
+      exit 1
     end
 
     # convert a Nokogiri HTML element back to a mechanize link

data/lib/mechanize/chunked_termination_error.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 ##
 # Raised when Mechanize detects the chunked transfer-encoding may be
 # incorrectly terminated.

data/lib/mechanize/content_type_error.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 ##
 # This error is raised when a pluggable parser tries to parse a content type
 # that it does not know how to handle.  For example if Mechanize::Page were to

data/lib/mechanize/cookie.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 warn 'mechanize/cookie will be deprecated.  Please migrate to the http-cookie APIs.' if $VERBOSE
 
 require 'http/cookie'
@@ -6,7 +7,7 @@ class Mechanize
   module CookieDeprecated
     def __deprecated__(to = nil)
       $VERBOSE or return
-      method = caller[0][/([^`]+)(?='$)/]
+      method = caller_locations(1,1).first.base_label
       to ||= method
       case self
       when Class
@@ -20,7 +21,7 @@ class Mechanize
         this = '%s#%s' % [klass, method]
         that = 'HTTP::%s#%s' % [lname, to]
       end
-      warn '%s: The call of %s needs to be fixed to follow the new API (%s).' % [caller[1], this, that]
+      warn '%s: The call of %s needs to be fixed to follow the new API (%s).' % [caller_locations(2,1).first, this, that]
     end
     private :__deprecated__
   end
@@ -50,19 +51,6 @@ class Mechanize
 
   Cookie = ::HTTP::Cookie
 
-  # Compatibility for Ruby 1.8/1.9
-  unless Cookie.respond_to?(:prepend, true)
-    require 'mechanize/prependable'
-
-    class Cookie
-      extend Prependable
-
-      class << self
-        extend Prependable
-      end
-    end
-  end
-
   class Cookie
     prepend CookieIMethods
 

data/lib/mechanize/cookie_jar.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 warn 'mechanize/cookie_jar will be deprecated.  Please migrate to the http-cookie APIs.' if $VERBOSE
 
 require 'http/cookie_jar'
@@ -65,7 +66,7 @@ class Mechanize
   class CookieJar < ::HTTP::CookieJar
     def save(output, *options)
       output.respond_to?(:write) or
-        return open(output, 'w') { |io| save(io, *options) }
+        return ::File.open(output, 'w') { |io| save(io, *options) }
 
       opthash = {
         :format => :yaml,
@@ -119,7 +120,7 @@ class Mechanize
 
     def load(input, *options)
       input.respond_to?(:write) or
-        return open(input, 'r') { |io| load(io, *options) }
+        return ::File.open(input, 'r') { |io| load(io, *options) }
 
       opthash = {
         :format => :yaml,
@@ -148,7 +149,7 @@ class Mechanize
       return super(input, opthash) if opthash[:format] != :yaml
 
       begin
-        data = YAML.load(input)
+        data = load_yaml(input)
       rescue ArgumentError
         @logger.warn "unloadable YAML cookie data discarded" if @logger
         return self
@@ -173,14 +174,17 @@ class Mechanize
         return self
       end
     end
-  end
 
-  # Compatibility for Ruby 1.8/1.9
-  unless ::HTTP::CookieJar.respond_to?(:prepend, true)
-    require 'mechanize/prependable'
+    private
 
-    class ::HTTP::CookieJar
-      extend Prependable
+    if YAML.name == "Psych" && Gem::Requirement.new(">= 3.1").satisfied_by?(Gem::Version.new(Psych::VERSION))
+      def load_yaml(yaml)
+        YAML.safe_load(yaml, aliases: true, permitted_classes: ["Mechanize::Cookie", "Time"])
+      end
+    else
+      def load_yaml(yaml)
+        YAML.load(yaml) # rubocop:disable Security/YAMLLoad
+      end
     end
   end
 

data/lib/mechanize/directory_saver.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 ##
 # Unlike Mechanize::FileSaver, the directory saver places all downloaded files
 # in a single pre-specified directory.

data/lib/mechanize/download.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 ##
 # Download is a pluggable parser for downloading files without loading them
 # into memory first.  You may subclass this class to handle content types you
@@ -71,7 +72,7 @@ class Mechanize::Download
     dirname = File.dirname filename
     FileUtils.mkdir_p dirname
 
-    open filename, 'wb' do |io|
+    ::File.open(filename, 'wb')do |io|
       until @body_io.eof? do
         io.write @body_io.read 16384
       end

data/lib/mechanize/element_matcher.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 module Mechanize::ElementMatcher
 
   def elements_with singular, plural = "#{singular}s"

data/lib/mechanize/element_not_found_error.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 ##
 # Raised when an an element was not found on the Page
 

data/lib/mechanize/file.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 ##
 # This is the base class for the Pluggable Parsers.  If Mechanize cannot find
 # an appropriate class to use for the content type, this class will be used.
@@ -82,7 +83,7 @@ class Mechanize::File
     dirname = File.dirname filename
     FileUtils.mkdir_p dirname
 
-    open filename, 'wb' do |f|
+    ::File.open(filename, 'wb')do |f|
       f.write body
     end
 

data/lib/mechanize/file_connection.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 ##
 # Wrapper to make a file URI work like an http URI
 
@@ -10,8 +11,9 @@ class Mechanize::FileConnection
   end
 
   def request uri, request
-    yield Mechanize::FileResponse.new Mechanize::Util.uri_unescape uri.path
+    file_path = uri.select(:host, :path)
+                  .select { |part| part && (part.length > 0) }
+                  .join(":")
+    yield Mechanize::FileResponse.new(Mechanize::Util.uri_unescape(file_path))
   end
-
 end
-

data/lib/mechanize/file_request.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 ##
 # A wrapper for a file URI that makes a request that works like a
 # Net::HTTPRequest

data/lib/mechanize/file_response.rb

@@ -1,8 +1,11 @@
+# frozen_string_literal: true
 ##
 # Fake response for dealing with file:/// requests
 
 class Mechanize::FileResponse
 
+  attr_reader :file_path
+
   def initialize(file_path)
     @file_path = file_path
     @uri       = nil
@@ -15,7 +18,7 @@ class Mechanize::FileResponse
     if directory?
       yield dir_body
     else
-      open @file_path, 'rb' do |io|
+      ::File.open(@file_path, 'rb') do |io|
         yield io.read
       end
     end

data/lib/mechanize/file_saver.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 ##
 # This is a pluggable parser that automatically saves every file it
 # encounters.  Unlike Mechanize::DirectorySaver, the file saver saves the

data/lib/mechanize/form/button.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 ##
 # A Submit button in a Form
 

data/lib/mechanize/form/check_box.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 ##
 # This class represents a check box found in a Form.  To activate the CheckBox
 # in the Form, set the checked method to true.

data/lib/mechanize/form/field.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 ##
 # This class represents a field in a form.  It handles the following input
 # tags found in a form:

data/lib/mechanize/form/file_upload.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 # This class represents a file upload field found in a form.  To use this
 # class, set FileUpload#file_data= to the data of the file you want to upload
 # and FileUpload#mime_type= to the appropriate mime type of the file.

data/lib/mechanize/form/hidden.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 class Mechanize::Form::Hidden < Mechanize::Form::Field
 end
 

data/lib/mechanize/form/image_button.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 ##
 # This class represents an image button in a form.  Use the x and y methods to
 # set the x and y positions for where the mouse "clicked".

data/lib/mechanize/form/keygen.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 ##
 # This class represents a keygen (public / private key generator) found in a
 # Form. The field will automatically generate a key pair and compute its own

data/lib/mechanize/form/multi_select_list.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 ##
 # This class represents a select list where multiple values can be selected.
 # MultiSelectList#value= accepts an array, and those values are used as
@@ -35,7 +36,7 @@ class Mechanize::Form::MultiSelectList < Mechanize::Form::Field
   #   select_list.option_with(:value => '1').value = 'foo'
 
   ##
-  # :mehtod: option_with!(criteria)
+  # :method: option_with!(criteria)
   #
   # Same as +option_with+ but raises an ElementNotFoundError if no button
   # matches +criteria+

data/lib/mechanize/form/option.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 ##
 # This class contains an option found within SelectList.  A SelectList can
 # have many Option classes associated with it.  An option can be selected by

data/lib/mechanize/form/radio_button.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 ##
 # This class represents a radio button found in a Form.  To activate the
 # RadioButton in the Form, set the checked method to true.

data/lib/mechanize/form/reset.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 class Mechanize::Form::Reset < Mechanize::Form::Button
 end
 

data/lib/mechanize/form/select_list.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 # This class represents a select list or drop down box in a Form.  Set the
 # value for the list by calling SelectList#value=.  SelectList contains a list
 # of Option that were found.  After finding the correct option, set the select

data/lib/mechanize/form/submit.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 class Mechanize::Form::Submit < Mechanize::Form::Button
 end
 

data/lib/mechanize/form/text.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 class Mechanize::Form::Text < Mechanize::Form::Field
 end
 

data/lib/mechanize/form/textarea.rb

@@ -1,3 +1,4 @@
+# frozen_string_literal: true
 class Mechanize::Form::Textarea < Mechanize::Form::Field
 end