rack-utf8_sanitizer 1.8.0 → 1.9.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 045740a7d869543a26c071de26ee6585d00e6193eaa2a5a02bfe09142cfe11c1
-  data.tar.gz: '050977cbbb72a835dea65e4df6bd75d6837c216b2a2f68eecd83701f1153e7ff'
+  metadata.gz: 7825c2fec2176e38043c4d7a3c1fcbe1cf112bcc7a17a7ef42b249fab30118c4
+  data.tar.gz: 5090e3c92af9a74377d559be48685d343b29315e5a9ce0f76faf36a8b96437ee
 SHA512:
-  metadata.gz: 611f078bdbe5f0247eac8ba3258a174eb70a1d3263b49ca0d4d261a8fd3de1b260da9defc36bc644eb2e6805211038adddf4396a9f7b0bb48fbd932e41991f97
-  data.tar.gz: 1bc7f43fbd004ac010a7829cb8077b2bdb245500670d2fc4d22372ccc44c959db47a3853aa68f1eb4c1533b5202ee6fdb44d368d722be7fa9439dc6049ddbef2
+  metadata.gz: e20607b2c412ecfb3d2ba719a7d0aeb381cc4f685e08c6a7801fb2b60a0993c71cc5c219b9ff17cedb497f7d5d0ee907da94ab91476960143f25c704058f1ebc
+  data.tar.gz: 7df7e1d357a6d3b12f089c1d7fea0a55eeb31d2d8f7e3d2b2e2e8729c1ae21c6260a9eb370e8c54e6886344986b1dd436d55b404f7321263d6bbf120115d1788

data/.github/workflows/ci.yml

@@ -10,7 +10,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        ruby: ["2.5", "2.6", "2.7", "3.0", "3.1", ruby-head, jruby-9.2, jruby-9.3, jruby-head]
+        ruby: ["2.5", "2.6", "2.7", "3.0", "3.1", "3.2", ruby-head, jruby-9.2, jruby-9.3, jruby-head]
 
     steps:
     - uses: actions/checkout@v3

data/README.md

@@ -113,7 +113,7 @@ config.middleware.insert 0, Rack::UTF8Sanitizer, strategy: :exception
 ```
 
 ```ruby
-replace_string = lambda do |_invalid|
+replace_string = lambda do |_invalid, sanitize_null_bytes: false|
   Rails.logger.warn('Replacing invalid string')
 
   '<Bad Encoding>'.freeze
@@ -122,6 +122,10 @@ end
 config.middleware.insert 0, Rack::UTF8Sanitizer, strategy: replace_string
 ```
 
+### Sanitizing Null Bytes
+
+While null bytes are valid UTF-8, it can be useful to further restrict the valid character set to exclude null bytes. For example, PostgreSQL text columns do not allow storing null bytes. Passing `sanitize_null_bytes: true` in the configuration hash enables sanitizing null bytes, and the two built-in strategies both support this feature. Custom strategies should accept a keyword argument `sanitize_null_bytes` containing this configuration value.
+
 ## Contributing
 
 1. Fork it

data/lib/rack/utf8_sanitizer.rb

@@ -7,6 +7,9 @@ module Rack
   class UTF8Sanitizer
     StringIO = ::StringIO
     BAD_REQUEST = [400, { "Content-Type" => "text/plain" }, ["Bad Request"]]
+    NULL_BYTE_REGEX = /\x00/.freeze
+
+    class NullByteInString < StandardError; end
 
     # options[:sanitizable_content_types] Array
     # options[:additional_content_types] Array
@@ -17,6 +20,7 @@ module Rack
       @sanitizable_content_types ||= SANITIZABLE_CONTENT_TYPES + (options[:additional_content_types] || [])
       @only = Array(options[:only]).flatten
       @except = Array(options[:except]).flatten
+      @sanitize_null_bytes = options.fetch(:sanitize_null_bytes, false)
     end
 
     def call(env)
@@ -29,21 +33,29 @@ module Rack
     end
 
     DEFAULT_STRATEGIES = {
-      replace: lambda do |input|
+      replace: lambda do |input, sanitize_null_bytes: false|
         input.
           force_encoding(Encoding::ASCII_8BIT).
           encode!(Encoding::UTF_8,
                   invalid: :replace,
                   undef:   :replace)
+        if sanitize_null_bytes
+          input = input.gsub(NULL_BYTE_REGEX, "")
+        end
+        input
       end,
-      exception: lambda do |input|
+      exception: lambda do |input, sanitize_null_bytes: false|
         input.
           force_encoding(Encoding::ASCII_8BIT).
           encode!(Encoding::UTF_8)
+        if sanitize_null_bytes && input =~ NULL_BYTE_REGEX
+          raise NullByteInString
+        end
+        input
       end
     }.freeze
 
-    # http://rack.rubyforge.org/doc/SPEC.html
+    # https://github.com/rack/rack/blob/main/SPEC.rdoc
     URI_FIELDS  = %w(
         SCRIPT_NAME
         REQUEST_PATH REQUEST_URI PATH_INFO
@@ -207,7 +219,8 @@ module Rack
 
     # This regexp matches all 'unreserved' characters from RFC3986 (2.3),
     # plus all multibyte UTF-8 characters.
-    UNRESERVED_OR_UTF8 = /[A-Za-z0-9\-._~\x80-\xFF]/
+    UNRESERVED_OR_UTF8 = /[A-Za-z0-9\-._~\x80-\xFF]/.freeze
+    UNRESERVED_OR_UTF8_OR_NULL = /[A-Za-z0-9\-._~\x00\x80-\xFF]/.freeze
 
     # RFC3986, 2.2 states that the characters from 'reserved' group must be
     # protected during normalization (which is what UTF8Sanitizer does).
@@ -218,7 +231,8 @@ module Rack
       input.gsub(/%([a-f\d]{2})/i) do |encoded|
         decoded = $1.hex.chr
 
-        if decoded =~ UNRESERVED_OR_UTF8
+        decodable_regex = @sanitize_null_bytes ? UNRESERVED_OR_UTF8_OR_NULL : UNRESERVED_OR_UTF8
+        if decoded =~ decodable_regex
           decoded
         else
           encoded
@@ -244,10 +258,10 @@ module Rack
       if input.is_a? String
         input = input.dup.force_encoding(Encoding::UTF_8)
 
-        if input.valid_encoding?
+        if input.valid_encoding? && !(@sanitize_null_bytes && input =~ NULL_BYTE_REGEX)
           input
         else
-          @strategy.call(input)
+          @strategy.call(input, sanitize_null_bytes: @sanitize_null_bytes)
         end
       else
         input

data/rack-utf8_sanitizer.gemspec

@@ -2,7 +2,7 @@
 
 Gem::Specification.new do |gem|
   gem.name          = "rack-utf8_sanitizer"
-  gem.version       = '1.8.0'
+  gem.version       = '1.9.1'
   gem.authors       = ["whitequark"]
   gem.license       = "MIT"
   gem.email         = ["whitequark@whitequark.org"]

data/test/test_utf8_sanitizer.rb

@@ -337,6 +337,71 @@ describe Rack::UTF8Sanitizer do
         @response_env["CONTENT_LENGTH"].should == sanitized_input.bytesize.to_s
       end
     end
+
+    it "does not sanitize null bytes by default" do
+      input =  "foo=bla&quux=bar%00"
+      @rack_input = StringIO.new input
+
+      sanitize_form_data do |sanitized_input|
+        sanitized_input.encoding.should == Encoding::UTF_8
+        sanitized_input.should.be.valid_encoding
+        sanitized_input.should == input
+      end
+    end
+
+    it "optionally sanitizes null bytes with the replace strategy" do
+      @app = Rack::UTF8Sanitizer.new(-> env { env }, sanitize_null_bytes: true)
+      input = "foo=bla\xED&quux=bar\x00"
+      @rack_input = StringIO.new input
+
+      sanitize_form_data do |sanitized_input|
+        sanitized_input.encoding.should == Encoding::UTF_8
+        sanitized_input.should.be.valid_encoding
+        sanitized_input.should == "foo=bla%EF%BF%BD&quux=bar"
+      end
+    end
+
+    it "optionally sanitizes encoded null bytes with the replace strategy" do
+      @app = Rack::UTF8Sanitizer.new(-> env { env }, sanitize_null_bytes: true)
+      input = "foo=bla%ED&quux=bar%00"
+      @rack_input = StringIO.new input
+
+      sanitize_form_data do |sanitized_input|
+        sanitized_input.encoding.should == Encoding::UTF_8
+        sanitized_input.should.be.valid_encoding
+        sanitized_input.should == "foo=bla%EF%BF%BD&quux=bar"
+      end
+    end
+
+    it "optionally raises on null bytes with the exception strategy" do
+      @app = Rack::UTF8Sanitizer.new(-> env { env }, sanitize_null_bytes: true, strategy: :exception)
+      input =  "foo=bla&quux=bar\x00"
+      @rack_input = StringIO.new input
+
+      should.raise(Rack::UTF8Sanitizer::NullByteInString) do
+        sanitize_form_data
+      end
+    end
+
+    it "optionally raises on encoded null bytes with the exception strategy" do
+      @app = Rack::UTF8Sanitizer.new(-> env { env }, sanitize_null_bytes: true, strategy: :exception)
+      input =  "foo=bla&quux=bar%00"
+      @rack_input = StringIO.new input
+
+      should.raise(Rack::UTF8Sanitizer::NullByteInString) do
+        sanitize_form_data
+      end
+    end
+
+    it "gives precedence to encoding errors with the exception strategy and null byte sanitisation" do
+      @app = Rack::UTF8Sanitizer.new(-> env { env }, sanitize_null_bytes: true, strategy: :exception)
+      input = "foo=bla\x00&quux=bar\xED"
+      @rack_input = StringIO.new input
+
+      should.raise(EncodingError) do
+        sanitize_form_data
+      end
+    end
   end
 
   describe "with custom content-type" do
@@ -552,7 +617,10 @@ describe Rack::UTF8Sanitizer do
     end
 
     it "accepts a proc as a strategy" do
-      truncate = -> input { 'replace'.force_encoding(Encoding::UTF_8) }
+      truncate = -> (input, sanitize_null_bytes:) do
+        sanitize_null_bytes.should == false
+        'replace'.force_encoding(Encoding::UTF_8)
+      end
 
       @app = Rack::UTF8Sanitizer.new(-> env { env }, strategy: truncate)
 
@@ -566,5 +634,24 @@ describe Rack::UTF8Sanitizer do
         sanitized_input.should == 'replace'
       end
     end
+
+    it "accepts a proc as a strategy and passes along sanitize_null_bytes" do
+      truncate = -> (input, sanitize_null_bytes:) do
+        sanitize_null_bytes.should == true
+        'replace'.force_encoding(Encoding::UTF_8)
+      end
+
+      @app = Rack::UTF8Sanitizer.new(-> env { env }, sanitize_null_bytes: true, strategy: truncate)
+      input = "foo=bla&quux=bar\x00"
+
+      @rack_input = StringIO.new input
+
+      env = request_env
+      sanitize_data(env) do |sanitized_input|
+        sanitized_input.encoding.should == Encoding::UTF_8
+        sanitized_input.should.be.valid_encoding
+        sanitized_input.should == 'replace'
+      end
+    end
   end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rack-utf8_sanitizer
 version: !ruby/object:Gem::Version
-  version: 1.8.0
+  version: 1.9.1
 platform: ruby
 authors:
 - whitequark
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2022-10-25 00:00:00.000000000 Z
+date: 2023-08-30 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rack
@@ -97,7 +97,7 @@ homepage: http://github.com/whitequark/rack-utf8_sanitizer
 licenses:
 - MIT
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -112,8 +112,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.2.5
-signing_key: 
+rubygems_version: 3.3.15
+signing_key:
 specification_version: 4
 summary: Rack::UTF8Sanitizer is a Rack middleware which cleans up invalid UTF8 characters
   in request URI and headers.