mechanize 2.8.0 → 2.8.4

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: d7df2fdd2030e028d6107b3b54f94f788deba04b9615cb719ef2113291270d1c
-  data.tar.gz: ab12e971c446e5977662076f39711298b5311ef12fe54612855433e5e926fa46
+  metadata.gz: d8b4e424716d3f5d8fdc1fe82efca4412eee6b414c0cafee9cdced96023e632d
+  data.tar.gz: b4450930235cb304ced9e63cede8fd3abe3bc7c7f38530ef6127be8208de3c93
 SHA512:
-  metadata.gz: 50f065469eb756a860e5c02b6e2d31cd5f25e0da2db9c4e9dddd018a96fb8fe4bb209ffd70c0e51c777ede12245faff86568a9e9044dd006d61d3514cd35660c
-  data.tar.gz: 6db1eaffd62dc89449721ad9d06dbc5d0f8d15f24b49cf9be84a3f88dc1bd91a41e1774aef12bdc9933e3b8e0d5b0050cf4dbaa675db6131f7723b3713921630
+  metadata.gz: cb936d26c46330432cd5d230b5e54b3792de5a2e39d3aa3a09a4904e5cf0f4bd9547c489772f3cd8d989e171022a18943bf4d09f0420038ed36d941978605a4f
+  data.tar.gz: 3b8103cfa2759f937b43384ac13ae0b8be9d22505841199fd0acd8fc0f4c4d86e0844571bbcd930da2d55e54f2178bbe222f9876959851ee694988bd5b874668

data/.github/dependabot.yml

@@ -0,0 +1,11 @@
+# To get started with Dependabot version updates, you'll need to specify which
+# package ecosystems to update and where the package manifests are located.
+# Please see the documentation for all configuration options:
+# https://help.github.com/github/administering-a-repository/configuration-options-for-dependency-updates
+
+version: 2
+updates:
+  - package-ecosystem: "bundler"
+    directory: "/"
+    schedule:
+      interval: "weekly"

data/.github/workflows/ci-test.yml

@@ -10,36 +10,44 @@ on:
       - main
 
 jobs:
+  rubocop:
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v2
+    - uses: ruby/setup-ruby@v1
+      with:
+        ruby-version: "3.1"
+        bundler-cache: true
+    - run: bundle exec rake rubocop
+
   test:
+    needs: ["rubocop"]
     strategy:
       fail-fast: false
       matrix:
-        ruby-version: ["2.5", "2.6", "2.7", "3.0", "jruby"]
+        ruby-version: ["2.5", "2.6", "2.7", "3.0", "3.1", "head", "jruby", "truffleruby-head"]
 
     runs-on: ubuntu-latest
     steps:
     - uses: actions/checkout@v2
-    - name: Set up Ruby
-      uses: ruby/setup-ruby@v1
+    - uses: ruby/setup-ruby@v1
       with:
-        ruby-version: ${{ matrix.ruby-version }}
+        ruby-version: ${{matrix.ruby-version}}
         bundler-cache: true
-    - name: Run tests
-      run: bundle exec rake
+    - run: bundle exec rake test
 
   test-platform:
+    needs: ["rubocop"]
     strategy:
       fail-fast: false
       matrix:
         platform: ["windows-latest", "macos-latest"]
 
-    runs-on: ${{ matrix.platform }}
+    runs-on: ${{matrix.platform}}
     steps:
     - uses: actions/checkout@v2
-    - name: Set up Ruby
-      uses: ruby/setup-ruby@v1
+    - uses: ruby/setup-ruby@v1
       with:
-        ruby-version: 3.0
+        ruby-version: "3.1"
         bundler-cache: true
-    - name: Run tests
-      run: bundle exec rake
+    - run: bundle exec rake test

data/CHANGELOG.md

@@ -1,19 +1,51 @@
 # Mechanize CHANGELOG
 
+## 2.8.4 / 2022-01-17
+
+### Fix
+
+* `Mechanize::CookieJar#load` calls `Psych.safe_load` when using Psych >= 3.1
+
+
+## 2.8.3 / 2021-11-11
+
+### Update
+
+* Update the "Linux Firefox" user agent string to rev94 (#587) Thank you, @ncs1!
+
+
+## 2.8.2 / 2021-08-06
+
+### Dependencies
+
+* Update dependency on Addressable from `~>2.7` to `~>2.8`. (#584) @yidingww
+
+
+## 2.8.1 / 2021-05-09
+
+### Fix
+
+* Gracefully handle parsing errors that contain an invalid byte sequence. Previously, if libxml2 registered a parsing error that itself contained invalid encoding, an exception might be raised. (#553)
+
+
 ## 2.8.0 / 2021-04-01
 
-* Requirements
-  * Mechanize now requires Ruby 2.5 or newer.
-  * Move from `ntlm-http` to `rubyntlm` gem. (#495, #574)
+### Requirements
 
-* New Features
-  * Page::Link#uri now handles non-ASCII `href`s. (#569) @terryyin
-  * FileConnection supports Windows drive letters (#483)
-  * Credential headers 'Authorization' and 'Cookie' are deleted on cross-origin redirects. (#538) @kyoshidajp
-  * ContentDispositionParser handles ISO8601 date headers, to be robust with websites that ignore RFC2183. (#554) @reitermarkus
+* Mechanize now requires Ruby 2.5 or newer.
+* Move from `ntlm-http` to `rubyntlm` gem. (#495, #574)
+
+### New Features
+
+* Page::Link#uri now handles non-ASCII `href`s. (#569) @terryyin
+* FileConnection supports Windows drive letters (#483)
+* Credential headers 'Authorization' and 'Cookie' are deleted on cross-origin redirects. (#538) @kyoshidajp
+* ContentDispositionParser handles ISO8601 date headers, to be robust with websites that ignore RFC2183. (#554) @reitermarkus
+
+### Bug fix
+
+* POST headers 'Content-Length', 'Content-MD5', and 'Content-Type' are deleted in a case-insensitive manner on redirects. Previously these headers were treated as case-sensitive.
 
-* Bug fix
-  * POST headers 'Content-Length', 'Content-MD5', and 'Content-Type' are deleted in a case-insensitive manner on redirects. Previously these headers were treated as case-sensitive.
 
 ## 2.7.7 / 2021-02-01
 
@@ -117,8 +149,7 @@
   * Mechanize::Agent#response_read will now raise a
     Mechanize::ResponseReadError instead of an EOFError and avoid losing
     requested content. #296.
-  * Depend on http-cookie, add backwards compatible deprecations.
-    #257 Akinori MUSHA.
+  * Depend on http-cookie, add backwards compatible deprecations. #257 Akinori MUSHA.
   * Added `Download#save!` for overwriting existing files. #300 Sean Kim.
 
 * Bug fix
@@ -147,13 +178,10 @@
   * Added iPad and Android user agents.  #277 by sambit, #278 by seansay.
 
 * Bug fix
-  * Mechanize#cert and Mechanize#key now return the values set by
-    #cert= and #key=. #244, #245 (Thanks, Robert Gogolok!)
-  * Mechanize no longer submits disabled form fields.  #276 by Bogdan Gusiev,
-    #279 by Ricardo Valeriano.
+  * Mechanize#cert and Mechanize#key now return the values set by #cert= and #key=. #244, #245 (Thanks, Robert Gogolok!)
+  * Mechanize no longer submits disabled form fields.  #276 by Bogdan Gusiev, #279 by Ricardo Valeriano.
   * Mechanize::File#save now behaves like Mechanize::Download#save in
-    that it will create the parent directory before saving.
-    #272, #280 by Ryan Kowalick
+    that it will create the parent directory before saving. #272, #280 by Ryan Kowalick
   * Ensure `application/xml` is registered as an XML parser in
     `PluggableParser`, not just `text/xml`. #266 James Gregory
   * Mechanize now writes cookiestxt with a prefixed dot for wildcard domain
@@ -173,8 +201,7 @@
     In mechanize 3 the old "Mac FireFox" user-agent alias will be removed.
     Pull request #231 by Gavin Miller.
   * Mechanize now authenticates using the raw challenge, not a reconstructed
-    one, to avoid dealing with quoting rules of RFC 2617.  Fixes failures in
-    #231 due to net-http-digest_auth 1.2.1
+    one, to avoid dealing with quoting rules of RFC 2617.  Fixes failures in #231 due to net-http-digest_auth 1.2.1
   * Fixed Content-Disposition parameter parser to be case insensitive. #233
   * Fixed redirection counting in following meta refresh. #240
 
@@ -205,8 +232,7 @@
     terminate chunked transfer-encoding properly.  Issue #116
   * Mechanize no longer raises an exception when multiple identical
     radiobuttons are checked.  Issue #214 by Matthias Guenther
-  * Fixed documentation for pre_connect_hooks and post_connect_hooks.  Issue
-    #226 by Robert Poor
+  * Fixed documentation for pre_connect_hooks and post_connect_hooks.  Issue #226 by Robert Poor
   * Worked around ruby 1.8 run with -Ku and ISO-8859-1 encoded characters in
     URIs.  Issue #228 by Stanislav O.Pogrebnyak
 
@@ -272,8 +298,7 @@
   * SSL parameters and proxy may now be set at any time.  Issue #194 by
     dsisnero.
   * Improved Mechanize::Page with #image_with and #images_with and
-    Mechanize::Page::Image various img element attribute accessors, #caption,
-    #extname, #mime_type and #fetch.  Pull request #173 by kitamomonga
+    Mechanize::Page::Image various img element attribute accessors, #caption, #extname, #mime_type and #fetch.  Pull request #173 by kitamomonga
   * Added MIME type parsing for content-types in Mechanize::PluggableParser
     for fine-grained parser choices.  Parsers will be chosen based on exact
     match, simplified type or media type in that order.  See
@@ -336,8 +361,7 @@
   * SSL connections will be verified against the system certificate store by
     default.
   * Added Mechanize#retry_change_requests to allow mechanize to retry POST and
-    other non-idempotent requests when you know it is safe to do so.  Issue
-    #123
+    other non-idempotent requests when you know it is safe to do so.  Issue #123
   * Mechanize can now stream files directly to disk without loading them into
     memory first through Mechanize::Download, a pluggable parser for
     downloading files.
@@ -352,8 +376,7 @@
       agent.pluggable_parser.default = Mechanize::Download
   * Added Mechanize#content_encoding_hooks which allow handling of
     non-standard content encodings like "agzip".  Patch #125 by kitamomonga
-  * Added dom_class to elements and the element matcher like dom_id.  Patch
-    #156 by Dan Hansen.
+  * Added dom_class to elements and the element matcher like dom_id.  Patch #156 by Dan Hansen.
   * Added support for the HTML5 keygen form element.  See
     http://dev.w3.org/html5/spec/Overview.html#the-keygen-element  Patch #157
     by Victor Costan.
@@ -402,8 +425,7 @@
   * The original Referer value persists on redirection.  Issue #150
   * Do not send a referer on a Refresh header based redirection.
   * Fixed encoding error in tests when LANG=C.  Patch #142 by jinschoi.
-  * The order of items in a form submission now match the DOM order.  Patch
-    #129 by kitamomonga
+  * The order of items in a form submission now match the DOM order.  Patch #129 by kitamomonga
   * Fixed proxy example in EXAMPLE.  Issue #146 by NielsKSchjoedt
 
 ## 2.0.1 / 2011-06-28
@@ -471,8 +493,7 @@ Mechanize is now under the MIT license
   * Mechanize now implements session cookies.  GH #78
   * Mechanize now implements deflate decoding.  GH #40
   * Mechanize now allows a certificate and key to be passed directly.  GH #71
-  * Mechanize::Form::MultiSelectList now implements #option_with and
-    #options_with.  GH #42
+  * Mechanize::Form::MultiSelectList now implements #option_with and #options_with.  GH #42
   * Add Mechanize::Page::Link#rel and #rel?(kind) to read and test the rel
     attribute.
   * Add Mechanize::Page#canonical_uri to read a </tt><link

data/README.md

@@ -74,6 +74,4 @@ Thank you to Michael Neumann for starting the Ruby version. Thanks to everyone w
 
 ## License
 
-This library is distributed under the MIT license. Please see the [LICENSE](http://docs.seattlerb.org/mechanize/LICENSE_rdoc.html) file.
-
-
+This library is distributed under the MIT license. Please see [LICENSE.txt](https://github.com/sparklemotion/mechanize/blob/main/LICENSE.txt).

data/lib/mechanize/cookie_jar.rb

@@ -149,7 +149,7 @@ class Mechanize
       return super(input, opthash) if opthash[:format] != :yaml
 
       begin
-        data = YAML.load(input) # rubocop:disable Security/YAMLLoad
+        data = load_yaml(input)
       rescue ArgumentError
         @logger.warn "unloadable YAML cookie data discarded" if @logger
         return self
@@ -174,6 +174,18 @@ class Mechanize
         return self
       end
     end
+
+    private
+
+    if YAML.name == "Psych" && Gem::Requirement.new(">= 3.1").satisfied_by?(Gem::Version.new(Psych::VERSION))
+      def load_yaml(yaml)
+        YAML.safe_load(yaml, aliases: true, permitted_classes: ["Mechanize::Cookie", "Time"])
+      end
+    else
+      def load_yaml(yaml)
+        YAML.load(yaml) # rubocop:disable Security/YAMLLoad
+      end
+    end
   end
 
   class ::HTTP::CookieJar

data/lib/mechanize/page.rb

@@ -104,9 +104,9 @@ class Mechanize::Page < Mechanize::File
     parser = self.parser unless parser
     return false if parser.errors.empty?
     parser.errors.any? do |error|
-      error.message =~ /(indicate\ encoding)|
-                        (Invalid\ char)|
-                        (input\ conversion\ failed)/x
+      error.message.scrub =~ /(indicate\ encoding)|
+                              (Invalid\ char)|
+                              (input\ conversion\ failed)/x
     end
   end
 

data/lib/mechanize/version.rb

@@ -1,4 +1,4 @@
 # frozen_string_literal: true
 class Mechanize
-  VERSION = "2.8.0"
+  VERSION = "2.8.4"
 end

data/lib/mechanize.rb

@@ -115,7 +115,7 @@ class Mechanize
 
   AGENT_ALIASES = {
     'Mechanize' => "Mechanize/#{VERSION} Ruby/#{ruby_version} (http://github.com/sparklemotion/mechanize/)",
-    'Linux Firefox' => 'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:43.0) Gecko/20100101 Firefox/43.0',
+    'Linux Firefox' => 'Mozilla/5.0 (X11; Ubuntu; Linux x86_64; rv:94.0) Gecko/20100101 Firefox/94.0',
     'Linux Konqueror' => 'Mozilla/5.0 (compatible; Konqueror/3; Linux)',
     'Linux Mozilla' => 'Mozilla/5.0 (X11; U; Linux i686; en-US; rv:1.4) Gecko/20030624',
     'Mac Firefox' => 'Mozilla/5.0 (Macintosh; Intel Mac OS X 10.11; rv:43.0) Gecko/20100101 Firefox/43.0',

data/mechanize.gemspec

@@ -1,13 +1,11 @@
 # coding: utf-8
 # frozen_string_literal: true
-lib = File.expand_path('../lib', __FILE__)
-$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
-require 'mechanize/version'
+require_relative 'lib/mechanize/version'
 
 Gem::Specification.new do |spec|
   spec.name = "mechanize"
   spec.version = Mechanize::VERSION
-  spec.homepage = "http://docs.seattlerb.org/mechanize/"
+  spec.homepage = "https://github.com/sparklemotion/mechanize"
   spec.summary = 'The Mechanize library is used for automating interaction with websites'
   spec.description =
     [
@@ -35,6 +33,15 @@ Gem::Specification.new do |spec|
       'ljjarvis@gmail.com',
     ]
 
+  spec.metadata = {
+    'yard.run'          => 'yard',
+    'bug_tracker_uri'   => 'https://github.com/sparklemotion/mechanize/issues',
+    'changelog_uri'     => 'https://github.com/sparklemotion/mechanize/blob/main/CHANGELOG.md',
+    'documentation_uri' => 'https://www.rubydoc.info/gems/mechanize',
+    'homepage_uri'      => 'https://github.com/sparklemotion/mechanize',
+    'source_code_uri'   => 'https://github.com/sparklemotion/mechanize'
+  }
+
   spec.license = "MIT"
 
   spec.require_paths = ["lib"]
@@ -46,7 +53,7 @@ Gem::Specification.new do |spec|
 
   spec.required_ruby_version = ">= 2.5.0"
 
-  spec.add_runtime_dependency("addressable", "~> 2.7")
+  spec.add_runtime_dependency("addressable", "~> 2.8")
   spec.add_runtime_dependency("domain_name", ">= 0.5.20190701", "~> 0.5")
   spec.add_runtime_dependency("http-cookie", ">= 1.0.3", "~> 1.0")
   spec.add_runtime_dependency("mime-types", "~> 3.0")

data/test/test_mechanize_page_encoding.rb

@@ -183,5 +183,32 @@ class TestMechanizePageEncoding < Mechanize::TestCase
     assert_equal Encoding::UTF_8, result.text.encoding
   end
 
-end
+  def test_parser_error_message_containing_encoding_errors
+    skip if RUBY_ENGINE == 'jruby' # this is a libxml2-specific condition
+
+    # https://github.com/sparklemotion/mechanize/issues/553
+    body = <<~EOF
+      <html>
+      <body>
+      <!--
+      ## メモ
+      処理の一般化, 二重ループ, 多重ループ
+      wzxhzdk:25
+      -->
+    EOF
+    page = util_page body
+
+    # this should not raise an "invalid byte sequence in UTF-8" error while processing parsing errors
+    page.search("body")
 
+    # let's assert on the setup: a libxml2-returned parsing error itself contains an invalid character
+    # note that this problem only appears in libxml <= 2.9.10
+    error = page.parser.errors.find { |e| e.message.include?("Comment not terminated") }
+    if error
+      exception = assert_raises(ArgumentError) do
+        error.message =~ /any regex just to trigger encoding error/
+      end
+      assert_includes(exception.message, "invalid byte sequence in UTF-8")
+    end
+  end
+end

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: mechanize
 version: !ruby/object:Gem::Version
-  version: 2.8.0
+  version: 2.8.4
 platform: ruby
 authors:
 - Eric Hodel
@@ -12,7 +12,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2021-04-01 00:00:00.000000000 Z
+date: 2022-01-17 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: addressable
@@ -20,14 +20,14 @@ dependencies:
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.7'
+        version: '2.8'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
     - - "~>"
       - !ruby/object:Gem::Version
-        version: '2.7'
+        version: '2.8'
 - !ruby/object:Gem::Dependency
   name: domain_name
   requirement: !ruby/object:Gem::Requirement
@@ -261,12 +261,13 @@ email:
 executables: []
 extensions: []
 extra_rdoc_files:
-- GUIDE.rdoc
 - EXAMPLES.rdoc
-- README.md
+- GUIDE.rdoc
 - CHANGELOG.md
+- README.md
 files:
 - ".autotest"
+- ".github/dependabot.yml"
 - ".github/workflows/ci-test.yml"
 - ".gitignore"
 - ".yardopts"
@@ -474,10 +475,16 @@ files:
 - test/test_mechanize_util.rb
 - test/test_mechanize_xml_file.rb
 - test/test_multi_select.rb
-homepage: http://docs.seattlerb.org/mechanize/
+homepage: https://github.com/sparklemotion/mechanize
 licenses:
 - MIT
-metadata: {}
+metadata:
+  yard.run: yard
+  bug_tracker_uri: https://github.com/sparklemotion/mechanize/issues
+  changelog_uri: https://github.com/sparklemotion/mechanize/blob/main/CHANGELOG.md
+  documentation_uri: https://www.rubydoc.info/gems/mechanize
+  homepage_uri: https://github.com/sparklemotion/mechanize
+  source_code_uri: https://github.com/sparklemotion/mechanize
 post_install_message: 
 rdoc_options:
 - "--main"
@@ -495,7 +502,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.1.4
+rubygems_version: 3.3.5
 signing_key: 
 specification_version: 4
 summary: The Mechanize library is used for automating interaction with websites