rack-utf8_sanitizer 1.8.0 → 1.9.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 045740a7d869543a26c071de26ee6585d00e6193eaa2a5a02bfe09142cfe11c1
-  data.tar.gz: '050977cbbb72a835dea65e4df6bd75d6837c216b2a2f68eecd83701f1153e7ff'
+  metadata.gz: 9893353fe731d6c942263ff57af911b10d1c9f88ba2c2c2b56bb7a76586935e5
+  data.tar.gz: eeeaca39f55d680abe2707fa1aaadfe34eed06de2dd848506d184ae4ff326a8d
 SHA512:
-  metadata.gz: 611f078bdbe5f0247eac8ba3258a174eb70a1d3263b49ca0d4d261a8fd3de1b260da9defc36bc644eb2e6805211038adddf4396a9f7b0bb48fbd932e41991f97
-  data.tar.gz: 1bc7f43fbd004ac010a7829cb8077b2bdb245500670d2fc4d22372ccc44c959db47a3853aa68f1eb4c1533b5202ee6fdb44d368d722be7fa9439dc6049ddbef2
+  metadata.gz: 28cfbadcc1fbbf0678db3b51259618a9c9897c0cb78bcd8b8f5d87c91eddb44873a7a77fb21b9db47bcc704cb7df17d942c50436475858d0d9fcc17c2a8b59ce
+  data.tar.gz: 7e781931df60e405d533f5eb832416510a431a1c6fa74baeabffb9499533d9eba2f97ffa24f89ed79a193de5f6715c0461fff4fb0403c863a3643e4aaabbcf0a

data/.github/workflows/ci.yml

@@ -10,7 +10,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        ruby: ["2.5", "2.6", "2.7", "3.0", "3.1", ruby-head, jruby-9.2, jruby-9.3, jruby-head]
+        ruby: ["2.5", "2.6", "2.7", "3.0", "3.1", "3.2", ruby-head, jruby-9.2, jruby-9.3, jruby-head]
 
     steps:
     - uses: actions/checkout@v3

data/lib/rack/utf8_sanitizer.rb

@@ -7,6 +7,9 @@ module Rack
   class UTF8Sanitizer
     StringIO = ::StringIO
     BAD_REQUEST = [400, { "Content-Type" => "text/plain" }, ["Bad Request"]]
+    NULL_BYTE_REGEX = /\x00/.freeze
+
+    class NullByteInString < StandardError; end
 
     # options[:sanitizable_content_types] Array
     # options[:additional_content_types] Array
@@ -17,6 +20,7 @@ module Rack
       @sanitizable_content_types ||= SANITIZABLE_CONTENT_TYPES + (options[:additional_content_types] || [])
       @only = Array(options[:only]).flatten
       @except = Array(options[:except]).flatten
+      @sanitize_null_bytes = options.fetch(:sanitize_null_bytes, false)
     end
 
     def call(env)
@@ -29,21 +33,27 @@ module Rack
     end
 
     DEFAULT_STRATEGIES = {
-      replace: lambda do |input|
+      replace: lambda do |input, sanitize_null_bytes: false|
+        if sanitize_null_bytes
+          input = input.gsub(NULL_BYTE_REGEX, "")
+        end
         input.
           force_encoding(Encoding::ASCII_8BIT).
           encode!(Encoding::UTF_8,
                   invalid: :replace,
                   undef:   :replace)
       end,
-      exception: lambda do |input|
+      exception: lambda do |input, sanitize_null_bytes: false|
+        if sanitize_null_bytes && input =~ NULL_BYTE_REGEX
+          raise NullByteInString
+        end
         input.
           force_encoding(Encoding::ASCII_8BIT).
           encode!(Encoding::UTF_8)
       end
     }.freeze
 
-    # http://rack.rubyforge.org/doc/SPEC.html
+    # https://github.com/rack/rack/blob/main/SPEC.rdoc
     URI_FIELDS  = %w(
         SCRIPT_NAME
         REQUEST_PATH REQUEST_URI PATH_INFO
@@ -207,7 +217,8 @@ module Rack
 
     # This regexp matches all 'unreserved' characters from RFC3986 (2.3),
     # plus all multibyte UTF-8 characters.
-    UNRESERVED_OR_UTF8 = /[A-Za-z0-9\-._~\x80-\xFF]/
+    UNRESERVED_OR_UTF8 = /[A-Za-z0-9\-._~\x80-\xFF]/.freeze
+    UNRESERVED_OR_UTF8_OR_NULL = /[A-Za-z0-9\-._~\x00\x80-\xFF]/.freeze
 
     # RFC3986, 2.2 states that the characters from 'reserved' group must be
     # protected during normalization (which is what UTF8Sanitizer does).
@@ -218,7 +229,8 @@ module Rack
       input.gsub(/%([a-f\d]{2})/i) do |encoded|
         decoded = $1.hex.chr
 
-        if decoded =~ UNRESERVED_OR_UTF8
+        decodable_regex = @sanitize_null_bytes ? UNRESERVED_OR_UTF8_OR_NULL : UNRESERVED_OR_UTF8
+        if decoded =~ decodable_regex
           decoded
         else
           encoded
@@ -244,10 +256,10 @@ module Rack
       if input.is_a? String
         input = input.dup.force_encoding(Encoding::UTF_8)
 
-        if input.valid_encoding?
+        if input.valid_encoding? && !(@sanitize_null_bytes && input =~ NULL_BYTE_REGEX)
           input
         else
-          @strategy.call(input)
+          @strategy.call(input, sanitize_null_bytes: @sanitize_null_bytes)
         end
       else
         input

data/rack-utf8_sanitizer.gemspec

@@ -2,7 +2,7 @@
 
 Gem::Specification.new do |gem|
   gem.name          = "rack-utf8_sanitizer"
-  gem.version       = '1.8.0'
+  gem.version       = '1.9.0'
   gem.authors       = ["whitequark"]
   gem.license       = "MIT"
   gem.email         = ["whitequark@whitequark.org"]

data/test/test_utf8_sanitizer.rb

@@ -337,6 +337,61 @@ describe Rack::UTF8Sanitizer do
         @response_env["CONTENT_LENGTH"].should == sanitized_input.bytesize.to_s
       end
     end
+
+    it "does not sanitize null bytes by default" do
+      input =  "foo=bla&quux=bar%00"
+      @rack_input = StringIO.new input
+
+      sanitize_form_data do |sanitized_input|
+        sanitized_input.encoding.should == Encoding::UTF_8
+        sanitized_input.should.be.valid_encoding
+        sanitized_input.should == input
+      end
+    end
+
+    it "optionally sanitizes null bytes with the replace strategy" do
+      @app = Rack::UTF8Sanitizer.new(-> env { env }, sanitize_null_bytes: true)
+      input =  "foo=bla&quux=bar\x00"
+      @rack_input = StringIO.new input
+
+      sanitize_form_data do |sanitized_input|
+        sanitized_input.encoding.should == Encoding::UTF_8
+        sanitized_input.should.be.valid_encoding
+        sanitized_input.should == "foo=bla&quux=bar"
+      end
+    end
+
+    it "optionally sanitizes encoded null bytes with the replace strategy" do
+      @app = Rack::UTF8Sanitizer.new(-> env { env }, sanitize_null_bytes: true)
+      input =  "foo=bla&quux=bar%00"
+      @rack_input = StringIO.new input
+
+      sanitize_form_data do |sanitized_input|
+        sanitized_input.encoding.should == Encoding::UTF_8
+        sanitized_input.should.be.valid_encoding
+        sanitized_input.should == "foo=bla&quux=bar"
+      end
+    end
+
+    it "optionally raises on null bytes with the exception strategy" do
+      @app = Rack::UTF8Sanitizer.new(-> env { env }, sanitize_null_bytes: true, strategy: :exception)
+      input =  "foo=bla&quux=bar\x00"
+      @rack_input = StringIO.new input
+
+      should.raise(Rack::UTF8Sanitizer::NullByteInString) do
+        sanitize_form_data
+      end
+    end
+
+    it "optionally raises on encoded null bytes with the exception strategy" do
+      @app = Rack::UTF8Sanitizer.new(-> env { env }, sanitize_null_bytes: true, strategy: :exception)
+      input =  "foo=bla&quux=bar%00"
+      @rack_input = StringIO.new input
+
+      should.raise(Rack::UTF8Sanitizer::NullByteInString) do
+        sanitize_form_data
+      end
+    end
   end
 
   describe "with custom content-type" do
@@ -552,7 +607,10 @@ describe Rack::UTF8Sanitizer do
     end
 
     it "accepts a proc as a strategy" do
-      truncate = -> input { 'replace'.force_encoding(Encoding::UTF_8) }
+      truncate = -> (input, sanitize_null_bytes:) do
+        sanitize_null_bytes.should == false
+        'replace'.force_encoding(Encoding::UTF_8)
+      end
 
       @app = Rack::UTF8Sanitizer.new(-> env { env }, strategy: truncate)
 
@@ -566,5 +624,24 @@ describe Rack::UTF8Sanitizer do
         sanitized_input.should == 'replace'
       end
     end
+
+    it "accepts a proc as a strategy and passes along sanitize_null_bytes" do
+      truncate = -> (input, sanitize_null_bytes:) do
+        sanitize_null_bytes.should == true
+        'replace'.force_encoding(Encoding::UTF_8)
+      end
+
+      @app = Rack::UTF8Sanitizer.new(-> env { env }, sanitize_null_bytes: true, strategy: truncate)
+      input = "foo=bla&quux=bar\x00"
+
+      @rack_input = StringIO.new input
+
+      env = request_env
+      sanitize_data(env) do |sanitized_input|
+        sanitized_input.encoding.should == Encoding::UTF_8
+        sanitized_input.should.be.valid_encoding
+        sanitized_input.should == 'replace'
+      end
+    end
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rack-utf8_sanitizer
 version: !ruby/object:Gem::Version
-  version: 1.8.0
+  version: 1.9.0
 platform: ruby
 authors:
 - whitequark
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2022-10-25 00:00:00.000000000 Z
+date: 2023-07-05 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rack
@@ -97,7 +97,7 @@ homepage: http://github.com/whitequark/rack-utf8_sanitizer
 licenses:
 - MIT
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -113,7 +113,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubygems_version: 3.2.5
-signing_key: 
+signing_key:
 specification_version: 4
 summary: Rack::UTF8Sanitizer is a Rack middleware which cleans up invalid UTF8 characters
   in request URI and headers.