mechanize 2.10.1 → 2.12.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: d5d4ec84463a2df7945cede673dde4ea74d9f03270c923cd7f11d2fbc56aadcd
-  data.tar.gz: c5803f307bc7924da28191b6cfee4a9ce2ed4e9f5daeeee3e3cc9e38bd405cfc
+  metadata.gz: 991701dfb3f8c11d731e22733ddb6bd158303ff68b3701d8fdcd780228cbc4be
+  data.tar.gz: 53d0f50ac55ec64c3accc35f69c828477ba2c5efe36921711e8adea1c459da05
 SHA512:
-  metadata.gz: 87f463a89c2158edfce7b556e3b824a6215646c6581947c434df8e7289f2e150a986db1750d5336b9266b7aee768f4000903271472575edcb90008862071818b
-  data.tar.gz: 5787421c7c331be9f0adc25ae01587a88ae8c78786420113aa813a9bbddc4ea755655501add2a7afc0400c03d5f9bdfb041958c5ebc61815981c0dac084296fb
+  metadata.gz: edde84f61a68ae2461b621b02e20dc479cab2258f8d005a3a6304f0c18d8829ae9e40c01c4d3f2eef0f24d08cc046035c1432a8b16202095a46f55ed67788fc5
+  data.tar.gz: 2bdbed0e937a610002095df60ad1bb476f5ddfe521b97044a4dd4102558ddf5a890a5a335fd3f6cb89be1403e91461f7db97ec0f924b3d6c21c95bf837834fd6

data/CHANGELOG.md

@@ -1,5 +1,15 @@
 # Mechanize CHANGELOG
 
+## 2.12.0 / 2024-07-29
+
+* Introduce experimental support for handling Brotli-compressed responses (CRuby only). (#650) @weshatheleopard
+
+
+## 2.11.0 / 2024-07-18
+
+* The `accept-charset` header is no longer sent. In early versions of Mechanize, circa 2007, this was a common header but now no modern browser sends it, and servers are instructed to ignore it. See #646 for an example of a server that is confused by its presence. (#647) @flavorjones
+
+
 ## 2.10.1 / 2024-06-12
 
 * Improve page encoding error recovery on pages with broken encoding when used with libxml2 >= 2.12.0. (#644) @flavorjones

data/Gemfile

@@ -1,3 +1,9 @@
 source "https://rubygems.org"
 
 gemspec
+
+gem "minitest", "~> 5.14"
+gem "rake", "~> 13.0"
+gem "rdoc", "~> 6.3"
+gem "rubocop", "~> 1.12"
+gem "brotli", ">= 0.5" unless RUBY_PLATFORM == "java"

data/lib/mechanize/http/agent.rb

@@ -495,6 +495,34 @@ class Mechanize::HTTP::Agent
     body_io.close
   end
 
+  ##
+  # Decodes a Brotli-encoded +body_io+
+  #
+  # (Experimental, CRuby only) Although Mechanize will never request a Brotli-encoded response via
+  # `accept-encoding`, buggy servers may return brotli-encoded responses anyway. Let's try to handle
+  # that case if the Brotli gem is loaded.
+  #
+  # If you need to handle Brotli-encoded responses, install the 'brotli' gem and require it in your
+  # application. If the `Brotli` constant is defined, Mechanize will attempt to use it to inflate
+  # the response.
+  #
+  def content_encoding_brotli(body_io)
+    log.debug('deflate brotli body') if log
+
+    unless defined?(::Brotli)
+      raise Mechanize::Error, "cannot deflate brotli-encoded response. Please install and require the 'brotli' gem."
+    end
+
+    begin
+      return StringIO.new(Brotli.inflate(body_io.read))
+    rescue Brotli::Error
+      log.error("unable to brotli-inflate response") if log
+      raise Mechanize::Error, "error inflating brotli-encoded response."
+    end
+  ensure
+    body_io.close
+  end
+
   def disable_keep_alive request
     request['connection'] = 'close' unless @keep_alive
   end
@@ -581,7 +609,6 @@ class Mechanize::HTTP::Agent
   end
 
   def request_language_charset request
-    request['accept-charset']  = 'ISO-8859-1,utf-8;q=0.7,*;q=0.7'
     request['accept-language'] = 'en-us,en;q=0.5'
   end
 
@@ -832,6 +859,8 @@ class Mechanize::HTTP::Agent
                content_encoding_inflate body_io
              when 'gzip', 'x-gzip' then
                content_encoding_gunzip body_io
+             when 'br' then
+               content_encoding_brotli body_io
              else
                raise Mechanize::Error,
                  "unsupported content-encoding: #{response['Content-Encoding']}"

data/lib/mechanize/version.rb

@@ -1,4 +1,4 @@
 # frozen_string_literal: true
 class Mechanize
-  VERSION = "2.10.1"
+  VERSION = "2.12.0"
 end

data/mechanize.gemspec

@@ -71,9 +71,4 @@ Gem::Specification.new do |spec|
   spec.add_runtime_dependency("rubyntlm", ">= 0.6.3", "~> 0.6")
   spec.add_runtime_dependency("base64") # removed from bundled gems in 3.4, and needed by rubyntlm (which doesn't declare this dependency)
   spec.add_runtime_dependency("nkf") # removed from bundled gems in 3.4
-
-  spec.add_development_dependency("minitest", "~> 5.14")
-  spec.add_development_dependency("rake", "~> 13.0")
-  spec.add_development_dependency("rdoc", "~> 6.3")
-  spec.add_development_dependency("rubocop", "~> 1.12")
 end

data/test/test_mechanize_http_agent.rb

@@ -2,6 +2,7 @@
 # frozen_string_literal: true
 
 require 'mechanize/test_case'
+require "brotli" unless RUBY_PLATFORM == "java"
 
 class TestMechanizeHttpAgent < Mechanize::TestCase
 
@@ -576,8 +577,8 @@ class TestMechanizeHttpAgent < Mechanize::TestCase
   def test_request_language_charset
     @agent.request_language_charset @req
 
-    assert_equal 'en-us,en;q=0.5', @req['accept-language']
-    assert_equal 'ISO-8859-1,utf-8;q=0.7,*;q=0.7', @req['accept-charset']
+    assert_equal('en-us,en;q=0.5', @req['accept-language'])
+    assert_nil(@req['accept-charset'])
   end
 
   def test_request_referer
@@ -924,6 +925,46 @@ class TestMechanizeHttpAgent < Mechanize::TestCase
     assert_equal 'part', body.read
   end
 
+  def test_response_content_encoding_brotli_when_brotli_not_loaded
+    skip("only test this on jruby which doesn't have brotli support") unless RUBY_ENGINE == 'jruby'
+
+    @res.instance_variable_set :@header, 'content-encoding' => %w[br]
+    body_io = StringIO.new("content doesn't matter for this test")
+
+    e = assert_raises(Mechanize::Error) do
+      @agent.response_content_encoding(@res, body_io)
+    end
+    assert_includes(e.message, "cannot deflate brotli-encoded response")
+
+    assert(body_io.closed?)
+  end
+
+  def test_response_content_encoding_brotli
+    skip("jruby does not have brotli support") if RUBY_ENGINE == 'jruby'
+
+    @res.instance_variable_set :@header, 'content-encoding' => %w[br]
+    body_io = StringIO.new(Brotli.deflate("this is compressed by brotli"))
+
+    body = @agent.response_content_encoding(@res, body_io)
+
+    assert_equal("this is compressed by brotli", body.read)
+    assert(body_io.closed?)
+  end
+
+  def test_response_content_encoding_brotli_corrupt
+    skip("jruby does not have brotli support") if RUBY_ENGINE == 'jruby'
+
+    @res.instance_variable_set :@header, 'content-encoding' => %w[br]
+    body_io = StringIO.new("not a brotli payload")
+
+    e = assert_raises(Mechanize::Error) do
+      @agent.response_content_encoding(@res, body_io)
+    end
+    assert_includes(e.message, "error inflating brotli-encoded response")
+    assert_kind_of(Brotli::Error, e.cause)
+    assert(body_io.closed?)
+  end
+
   def test_response_content_encoding_gzip_corrupt
     log = StringIO.new
     logger = Logger.new log

metadata

@@ -1,7 +1,7 @@
 --- !ruby/object:Gem::Specification
 name: mechanize
 version: !ruby/object:Gem::Version
-  version: 2.10.1
+  version: 2.12.0
 platform: ruby
 authors:
 - Eric Hodel
@@ -12,7 +12,7 @@ authors:
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2024-06-12 00:00:00.000000000 Z
+date: 2024-07-29 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: addressable
@@ -218,62 +218,6 @@ dependencies:
     - - ">="
       - !ruby/object:Gem::Version
         version: '0'
-- !ruby/object:Gem::Dependency
-  name: minitest
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '5.14'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '5.14'
-- !ruby/object:Gem::Dependency
-  name: rake
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '13.0'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '13.0'
-- !ruby/object:Gem::Dependency
-  name: rdoc
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '6.3'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '6.3'
-- !ruby/object:Gem::Dependency
-  name: rubocop
-  requirement: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '1.12'
-  type: :development
-  prerelease: false
-  version_requirements: !ruby/object:Gem::Requirement
-    requirements:
-    - - "~>"
-      - !ruby/object:Gem::Version
-        version: '1.12'
 description: |-
   The Mechanize library is used for automating interaction with websites.
   Mechanize automatically stores and sends cookies, follows redirects,