mechanize 2.8.4 → 2.9.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: d8b4e424716d3f5d8fdc1fe82efca4412eee6b414c0cafee9cdced96023e632d
-  data.tar.gz: b4450930235cb304ced9e63cede8fd3abe3bc7c7f38530ef6127be8208de3c93
+  metadata.gz: 25fad3bc233e4efc5e99a66ccd480450e100b2bdb76e8a7e10d00ffb5386fcf0
+  data.tar.gz: 6c63c1e76044803b26f687d48b12c3f43e2009afa7e5bbd6ce183e316eaad049
 SHA512:
-  metadata.gz: cb936d26c46330432cd5d230b5e54b3792de5a2e39d3aa3a09a4904e5cf0f4bd9547c489772f3cd8d989e171022a18943bf4d09f0420038ed36d941978605a4f
-  data.tar.gz: 3b8103cfa2759f937b43384ac13ae0b8be9d22505841199fd0acd8fc0f4c4d86e0844571bbcd930da2d55e54f2178bbe222f9876959851ee694988bd5b874668
+  metadata.gz: f200b8ceaac133254e05d03cbe38344cf31b55bd7c4a2979fd53580eb5fb84ee4f04e4a76e111037cc6d1e6efdeabb14a494ff47194668dc8334d06dc26cd377
+  data.tar.gz: af78f50d64366a713f70297befb328cba35120b08850c99cf2dd09a2d670b1ffcdf5cf5e12889e12db2c7ddc7a595074707c58e7aa0c4792693cbf5cddacee3e

data/.github/workflows/ci-test.yml

@@ -1,5 +1,9 @@
 name: "ci"
 
+concurrency:
+  group: "${{github.workflow}}-${{github.ref}}"
+  cancel-in-progress: true
+
 on:
   push:
     branches:
@@ -8,15 +12,17 @@ on:
     types: [opened, synchronize]
     branches:
       - main
+  schedule:
+    - cron: "0 8 * * 5" # At 08:00 on Friday # https://crontab.guru/#0_8_*_*_5
 
 jobs:
   rubocop:
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v3
     - uses: ruby/setup-ruby@v1
       with:
-        ruby-version: "3.1"
+        ruby-version: "3.2"
         bundler-cache: true
     - run: bundle exec rake rubocop
 
@@ -25,15 +31,16 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        ruby-version: ["2.5", "2.6", "2.7", "3.0", "3.1", "head", "jruby", "truffleruby-head"]
+        ruby-version: ["2.6", "2.7", "3.0", "3.1", "3.2", "head", "jruby-9.4", "truffleruby-head"]
 
     runs-on: ubuntu-latest
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v3
     - uses: ruby/setup-ruby@v1
       with:
         ruby-version: ${{matrix.ruby-version}}
         bundler-cache: true
+        bundler: 2.3.26 # https://github.com/rubygems/rubygems/issues/6435
     - run: bundle exec rake test
 
   test-platform:
@@ -45,9 +52,10 @@ jobs:
 
     runs-on: ${{matrix.platform}}
     steps:
-    - uses: actions/checkout@v2
+    - uses: actions/checkout@v3
     - uses: ruby/setup-ruby@v1
       with:
-        ruby-version: "3.1"
+        ruby-version: "3.2"
         bundler-cache: true
+        bundler: latest
     - run: bundle exec rake test

data/CHANGELOG.md

@@ -1,5 +1,24 @@
 # Mechanize CHANGELOG
 
+## 2.9.0 / 2023-04-07
+
+### Requirements
+
+* Mechanize now requires Ruby 2.6 or newer.
+
+
+### Improvement
+
+* Mechanize can now parse frozen strings. (#610)
+
+
+## 2.8.5 / 2022-06-09
+
+### Security
+
+Fixes low-severity CVE-2022-31033, "Authorization header leak on port redirect." See [GHSA-64qm-hrgp-pgr9](https://github.com/sparklemotion/mechanize/security/advisories/GHSA-64qm-hrgp-pgr9) for more details.
+
+
 ## 2.8.4 / 2022-01-17
 
 ### Fix

data/README.md

@@ -13,7 +13,7 @@ The Mechanize library is used for automating interaction with websites. Mechaniz
 
 ## Dependencies
 
-* Ruby >= 2.5
+* Ruby >= 2.6
 * Gems:
   * `addressable`
   * `domain_name`

data/examples/wikipedia_links_to_philosophy.rb

@@ -58,10 +58,10 @@ class WikipediaLinksToPhilosophy
   # the article.
 
   def follow_first_link
-    puts @title
+    puts "#{@title} (#{@page.uri})"
 
     # > p > a rejects italics
-    links = @page.root.css('.mw-content-ltr > p > a[href^="/wiki/"]')
+    links = @page.root.css('.mw-content-ltr p > a[href^="/wiki/"]')
 
     # reject disambiguation and special pages, images and files
     links = links.reject do |link_node|
@@ -74,10 +74,9 @@ class WikipediaLinksToPhilosophy
 
     link = links.first
 
-    unless link then
-      # disambiguation page? try the first item in the list
-      link =
-        @page.root.css('.mw-content-ltr > ul > li > a[href^="/wiki/"]').first
+    if link.nil?
+      puts "Could not parse #{@page.uri}"
+      exit 1
     end
 
     # convert a Nokogiri HTML element back to a mechanize link

data/lib/mechanize/http/agent.rb

@@ -9,7 +9,8 @@ require 'webrobots'
 
 class Mechanize::HTTP::Agent
 
-  CREDENTIAL_HEADERS = ['Authorization', 'Cookie']
+  CREDENTIAL_HEADERS = ['Authorization']
+  COOKIE_HEADERS = ['Cookie']
   POST_HEADERS = ['Content-Length', 'Content-MD5', 'Content-Type']
 
   # :section: Headers
@@ -998,10 +999,14 @@ class Mechanize::HTTP::Agent
     end
 
     # Make sure we clear credential headers if being redirected to another site
-    if new_uri.host != page.uri.host
-      CREDENTIAL_HEADERS.each do |ch|
-        headers.delete_if { |h| h.casecmp?(ch) }
+    if new_uri.host == page.uri.host
+      if new_uri.port != page.uri.port
+        # https://datatracker.ietf.org/doc/html/rfc6265#section-8.5
+        # cookies are OK to be shared across ports on the same host
+        CREDENTIAL_HEADERS.each { |ch| headers.delete_if { |h| h.casecmp?(ch) } }
       end
+    else
+      (COOKIE_HEADERS + CREDENTIAL_HEADERS).each { |ch| headers.delete_if { |h| h.casecmp?(ch) } }
     end
 
     fetch new_uri, redirect_method, headers, [], referer, redirects + 1

data/lib/mechanize/page.rb

@@ -41,10 +41,6 @@ class Mechanize::Page < Mechanize::File
     @encodings.concat self.class.response_header_charset(response)
 
     if body
-      # Force the encoding to be 8BIT so we can perform regular expressions.
-      # We'll set it to the detected encoding later
-      body.force_encoding(Encoding::ASCII_8BIT)
-
       @encodings.concat self.class.meta_charset body
 
       meta_content_type = self.class.meta_content_type body

data/lib/mechanize/version.rb

@@ -1,4 +1,4 @@
 # frozen_string_literal: true
 class Mechanize
-  VERSION = "2.8.4"
+  VERSION = "2.9.0"
 end

data/mechanize.gemspec

@@ -51,7 +51,7 @@ Gem::Specification.new do |spec|
   spec.extra_rdoc_files += Dir['*.rdoc', '*.md']
   spec.rdoc_options = ["--main", "README.md"]
 
-  spec.required_ruby_version = ">= 2.5.0"
+  spec.required_ruby_version = ">= 2.6.0"
 
   spec.add_runtime_dependency("addressable", "~> 2.8")
   spec.add_runtime_dependency("domain_name", ">= 0.5.20190701", "~> 0.5")

data/test/test_mechanize_http_agent.rb

@@ -27,13 +27,10 @@ class TestMechanizeHttpAgent < Mechanize::TestCase
     realm
   end
 
-  def jruby_zlib?
+  def skip_if_jruby_zlib
     if RUBY_ENGINE == 'jruby'
       meth = caller[0][/`(\w+)/, 1]
-      warn "#{meth}: skipped because how Zlib handles error is different in JRuby"
-      true
-    else
-      false
+      skip "#{meth}: skipped because how Zlib handles error is different in JRuby"
     end
   end
 
@@ -823,7 +820,11 @@ class TestMechanizeHttpAgent < Mechanize::TestCase
     @res.instance_variable_set(:@header,
                                'www-authenticate' => ['Negotiate, NTLM'])
 
-    page = @agent.response_authenticate @res, nil, @uri, @req, {}, nil, nil
+    begin
+      page = @agent.response_authenticate @res, nil, @uri, @req, {}, nil, nil
+    rescue OpenSSL::Digest::DigestError
+      skip "It looks like OpenSSL is not configured to support MD4"
+    end
 
     assert_equal 'ok', page.body # lame test
   end
@@ -931,7 +932,7 @@ class TestMechanizeHttpAgent < Mechanize::TestCase
     body_io = StringIO.new \
       "\037\213\b\0002\002\225M\000\003+H,*\001"
 
-    skip if jruby_zlib?
+    skip_if_jruby_zlib
 
     e = assert_raises Mechanize::Error do
       @agent.response_content_encoding @res, body_io
@@ -965,7 +966,8 @@ class TestMechanizeHttpAgent < Mechanize::TestCase
 
     assert_match %r%invalid compressed data -- crc error%, log.string
   rescue IOError
-    raise unless jruby_zlib?
+    skip_if_jruby_zlib
+    raise
   end
 
   def test_response_content_encoding_gzip_checksum_corrupt_length
@@ -983,7 +985,8 @@ class TestMechanizeHttpAgent < Mechanize::TestCase
 
     assert_match %r%invalid compressed data -- length error%, log.string
   rescue IOError
-    raise unless jruby_zlib?
+    skip_if_jruby_zlib
+    raise
   end
 
   def test_response_content_encoding_gzip_checksum_truncated
@@ -1001,7 +1004,8 @@ class TestMechanizeHttpAgent < Mechanize::TestCase
 
     assert_match %r%unable to gunzip response: footer is not found%, log.string
   rescue IOError
-    raise unless jruby_zlib?
+    skip_if_jruby_zlib
+    raise
   end
 
   def test_response_content_encoding_gzip_empty
@@ -1042,7 +1046,8 @@ class TestMechanizeHttpAgent < Mechanize::TestCase
 
     assert body_io.closed?
   rescue IOError
-    raise unless jruby_zlib?
+    skip_if_jruby_zlib
+    raise
   end
 
   def test_response_content_encoding_none
@@ -1569,7 +1574,7 @@ class TestMechanizeHttpAgent < Mechanize::TestCase
     refute_includes(headers.keys, "AUTHORIZATION")
     refute_includes(headers.keys, "cookie")
 
-    assert_match 'range|bytes=0-9999', page.body
+    assert_match("range|bytes=0-9999", page.body)
     refute_match("authorization|Basic xxx", page.body)
     refute_match("cookie|name=value", page.body)
   end
@@ -1590,11 +1595,32 @@ class TestMechanizeHttpAgent < Mechanize::TestCase
     assert_includes(headers.keys, "AUTHORIZATION")
     assert_includes(headers.keys, "cookie")
 
-    assert_match 'range|bytes=0-9999', page.body
+    assert_match("range|bytes=0-9999", page.body)
     assert_match("authorization|Basic xxx", page.body)
     assert_match("cookie|name=value", page.body)
   end
 
+  def test_response_redirect_to_same_site_diff_port_with_credential
+    @agent.redirect_ok = true
+
+    headers = {
+      'Range' => 'bytes=0-9999',
+      'AUTHORIZATION' => 'Basic xxx',
+      'cookie' => 'name=value',
+    }
+
+    page = html_page ''
+    page = @agent.response_redirect({ 'Location' => 'http://example:81/http_headers' }, :get,
+                                    page, 0, headers)
+
+    refute_includes(headers.keys, "AUTHORIZATION")
+    assert_includes(headers.keys, "cookie")
+
+    assert_match("range|bytes=0-9999", page.body)
+    refute_match("authorization|Basic xxx", page.body)
+    assert_match("cookie|name=value", page.body)
+  end
+
   def test_response_redirect_not_ok
     @agent.redirect_ok = false
 

data/test/test_mechanize_page.rb

@@ -276,5 +276,19 @@ class TestMechanizePage < Mechanize::TestCase
 		assert_equal page.title, "HTML>TITLE"
 	end
 
+  def test_frozen_string_body
+    html = (<<~HTML).freeze
+      <html>
+        <head>
+          <title>Page Title</title>
+        </head>
+        <body>
+          <p>Hello World</p>
+        </body>
+      </html>
+    HTML
+
+    html_page(html) # refute_raises
+  end
 end
 

data/test/test_mechanize_page_link.rb

@@ -51,13 +51,10 @@ class TestMechanizePageLink < Mechanize::TestCase
     Mechanize::Page.new @uri, res, body && body.force_encoding(Encoding::BINARY), 200, @mech
   end
 
-  def nkf_dependency?
-    if RUBY_ENGINE == 'ruby'
-      false
-    else
+  def skip_if_nkf_dependency
+    if RUBY_ENGINE == 'jruby'
       meth = caller[0][/`(\w+)/, 1]
-      warn "#{meth}: skipped because this feature currently depends on NKF"
-      true
+      skip "#{meth}: skipped because this feature currently depends on NKF"
     end
   end
 
@@ -112,7 +109,7 @@ class TestMechanizePageLink < Mechanize::TestCase
   end
 
   def test_encoding_charset_after_title_bad
-    return if nkf_dependency?
+    skip_if_nkf_dependency
 
     page = util_page UTF8
 
@@ -122,7 +119,7 @@ class TestMechanizePageLink < Mechanize::TestCase
   end
 
   def test_encoding_charset_after_title_double_bad
-    return if nkf_dependency?
+    skip_if_nkf_dependency
 
     page = util_page SJIS_BAD_AFTER_TITLE
 
@@ -132,7 +129,7 @@ class TestMechanizePageLink < Mechanize::TestCase
   end
 
   def test_encoding_charset_bad
-    return if nkf_dependency?
+    skip_if_nkf_dependency
 
     page = util_page "<title>#{UTF8_TITLE}</title>"
     page.encodings.replace %w[

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: mechanize
 version: !ruby/object:Gem::Version
-  version: 2.8.4
+  version: 2.9.0
 platform: ruby
 authors:
 - Eric Hodel
@@ -12,7 +12,7 @@ authors:
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2022-01-17 00:00:00.000000000 Z
+date: 2023-04-07 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: addressable
@@ -495,14 +495,14 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: 2.5.0
+      version: 2.6.0
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.3.5
+rubygems_version: 3.4.10
 signing_key: 
 specification_version: 4
 summary: The Mechanize library is used for automating interaction with websites