rack-utf8_sanitizer 1.9.0 → 1.10.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 9893353fe731d6c942263ff57af911b10d1c9f88ba2c2c2b56bb7a76586935e5
-  data.tar.gz: eeeaca39f55d680abe2707fa1aaadfe34eed06de2dd848506d184ae4ff326a8d
+  metadata.gz: c33079dde3e7e3efb8c46742a50303be620b316243b5e11c5f026a89ce29bf7d
+  data.tar.gz: 537ff74a7f0c3edfe1bc3904540ae3c95c4ed08f15198a8872bf25a055376673
 SHA512:
-  metadata.gz: 28cfbadcc1fbbf0678db3b51259618a9c9897c0cb78bcd8b8f5d87c91eddb44873a7a77fb21b9db47bcc704cb7df17d942c50436475858d0d9fcc17c2a8b59ce
-  data.tar.gz: 7e781931df60e405d533f5eb832416510a431a1c6fa74baeabffb9499533d9eba2f97ffa24f89ed79a193de5f6715c0461fff4fb0403c863a3643e4aaabbcf0a
+  metadata.gz: 787fbd5b17de52dbd26bcef3e64acb359d026eb6eeb3f5900fa0fe641980235f795c0fa2067693ab2630b2781e8d8a9c03fda141e7ad2fbca6c65f663b571795
+  data.tar.gz: be57486cc8be56299013bf3b101363702a8bc2af26746293e6661dd762b11737a5ae76c1f14373d5200772a3e229b19c7687f3c1855fc1950256cf4e750e3ca8

data/.github/workflows/ci.yml

@@ -10,10 +10,10 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        ruby: ["2.5", "2.6", "2.7", "3.0", "3.1", "3.2", ruby-head, jruby-9.2, jruby-9.3, jruby-head]
+        ruby: ["2.5", "2.6", "2.7", "3.0", "3.1", "3.2", "3.3", ruby-head, jruby-9.2, jruby-9.3, jruby-head]
 
     steps:
-    - uses: actions/checkout@v3
+    - uses: actions/checkout@v4
     - name: Set up Ruby
       uses: ruby/setup-ruby@v1
       with:

data/Gemfile

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 source 'https://rubygems.org'
 
 # Specify your gem's dependencies in rack-utf8_sanitizer.gemspec

data/README.md

@@ -113,7 +113,7 @@ config.middleware.insert 0, Rack::UTF8Sanitizer, strategy: :exception
 ```
 
 ```ruby
-replace_string = lambda do |_invalid|
+replace_string = lambda do |_invalid, sanitize_null_bytes: false|
   Rails.logger.warn('Replacing invalid string')
 
   '<Bad Encoding>'.freeze
@@ -122,6 +122,10 @@ end
 config.middleware.insert 0, Rack::UTF8Sanitizer, strategy: replace_string
 ```
 
+### Sanitizing Null Bytes
+
+While null bytes are valid UTF-8, it can be useful to further restrict the valid character set to exclude null bytes. For example, PostgreSQL text columns do not allow storing null bytes. Passing `sanitize_null_bytes: true` in the configuration hash enables sanitizing null bytes, and the two built-in strategies both support this feature. Custom strategies should accept a keyword argument `sanitize_null_bytes` containing this configuration value.
+
 ## Contributing
 
 1. Fork it

data/Rakefile

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require "bundler/gem_tasks"
 
 task :default => :spec

data/lib/rack/utf8_sanitizer.rb

@@ -1,12 +1,13 @@
 # encoding: ascii-8bit
+# frozen_string_literal: true
 
 require 'uri'
 require 'stringio'
+require 'rack/request'
 
 module Rack
   class UTF8Sanitizer
     StringIO = ::StringIO
-    BAD_REQUEST = [400, { "Content-Type" => "text/plain" }, ["Bad Request"]]
     NULL_BYTE_REGEX = /\x00/.freeze
 
     class NullByteInString < StandardError; end
@@ -27,29 +28,31 @@ module Rack
       begin
         env = sanitize(env)
       rescue EOFError
-        return BAD_REQUEST
+        return [400, { "Content-Type" => "text/plain" }, ["Bad Request"]]
       end
       @app.call(env)
     end
 
     DEFAULT_STRATEGIES = {
       replace: lambda do |input, sanitize_null_bytes: false|
-        if sanitize_null_bytes
-          input = input.gsub(NULL_BYTE_REGEX, "")
-        end
         input.
           force_encoding(Encoding::ASCII_8BIT).
           encode!(Encoding::UTF_8,
                   invalid: :replace,
                   undef:   :replace)
+        if sanitize_null_bytes
+          input = input.gsub(NULL_BYTE_REGEX, "")
+        end
+        input
       end,
       exception: lambda do |input, sanitize_null_bytes: false|
-        if sanitize_null_bytes && input =~ NULL_BYTE_REGEX
-          raise NullByteInString
-        end
         input.
           force_encoding(Encoding::ASCII_8BIT).
           encode!(Encoding::UTF_8)
+        if sanitize_null_bytes && NULL_BYTE_REGEX.match?(input)
+          raise NullByteInString
+        end
+        input
       end
     }.freeze
 
@@ -62,20 +65,20 @@ module Rack
         ORIGINAL_FULLPATH
         ORIGINAL_SCRIPT_NAME
         SERVER_NAME
-    ).map(&:freeze).freeze
+    ).freeze
 
     SANITIZABLE_CONTENT_TYPES = %w(
       text/plain
       application/x-www-form-urlencoded
       application/json
       text/javascript
-    ).map(&:freeze).freeze
+    ).freeze
 
     URI_ENCODED_CONTENT_TYPES = %w(
       application/x-www-form-urlencoded
-    ).map(&:freeze).freeze
+    ).freeze
 
-    HTTP_ = 'HTTP_'.freeze
+    HTTP_ = 'HTTP_'
 
     def sanitize(env)
       sanitize_rack_input(env)
@@ -113,17 +116,17 @@ module Rack
     end
 
     def sanitize_rack_input(env)
-      # https://github.com/rack/rack/blob/master/lib/rack/request.rb#L42
-      # Logic borrowed from Rack::Request#media_type,#media_type_params,#content_charset
-      # Ignoring charset in content type.
-      content_type   = env['CONTENT_TYPE']
-      content_type &&= content_type.split(/\s*[;,]\s*/, 2).first
-      content_type &&= content_type.downcase
+      request = Rack::Request.new(env)
+      content_type = request.media_type
       return unless @sanitizable_content_types.any? {|type| content_type == type }
+
+      charset = request.content_charset
+      return if charset && charset.downcase != 'utf-8'
+
       uri_encoded = URI_ENCODED_CONTENT_TYPES.any? {|type| content_type == type}
 
       if env['rack.input']
-        sanitized_input = sanitize_io(env['rack.input'], uri_encoded)
+        sanitized_input = sanitize_io(env['rack.input'], uri_encoded, env['CONTENT_LENGTH']&.to_i)
 
         env['rack.input'] = sanitized_input
         env['CONTENT_LENGTH'] &&= sanitized_input.size.to_s
@@ -165,8 +168,12 @@ module Rack
       end
     end
 
-    def sanitize_io(io, uri_encoded = false)
-      input = io.read
+    def sanitize_io(io, uri_encoded = false, content_length = nil)
+      input = if content_length && content_length >= 0
+        io.read(content_length)
+      else
+        io.read
+      end
       sanitized_input = sanitize_string(strip_byte_order_mark(input))
       if uri_encoded
         sanitized_input = sanitize_uri_encoded_string(sanitized_input).
@@ -249,7 +256,7 @@ module Rack
     # Performs the reverse function of `unescape_unreserved`. Unlike
     # the previous function, we can reuse the logic in URI#encode
     def escape_unreserved(input)
-      URI::DEFAULT_PARSER.escape(input, UNSAFE)
+      URI::RFC2396_PARSER.escape(input, UNSAFE)
     end
 
     def sanitize_string(input)
@@ -274,7 +281,7 @@ module Rack
       end
     end
 
-    UTF8_BOM = "\xef\xbb\xbf".force_encoding(Encoding::BINARY).freeze
+    UTF8_BOM = "\xef\xbb\xbf".dup.force_encoding(Encoding::BINARY).freeze
     UTF8_BOM_SIZE = UTF8_BOM.bytesize
 
     def strip_byte_order_mark(input)

data/rack-utf8_sanitizer.gemspec

@@ -1,21 +1,22 @@
 # -*- encoding: utf-8 -*-
+# frozen_string_literal: true
 
 Gem::Specification.new do |gem|
   gem.name          = "rack-utf8_sanitizer"
-  gem.version       = '1.9.0'
-  gem.authors       = ["whitequark"]
+  gem.version       = '1.10.0'
+  gem.authors       = ["Catherine"]
   gem.license       = "MIT"
   gem.email         = ["whitequark@whitequark.org"]
-  gem.description   = %{Rack::UTF8Sanitizer is a Rack middleware which cleans up } <<
-                      %{invalid UTF8 characters in request URI and headers.}
+  gem.description   = "Rack::UTF8Sanitizer is a Rack middleware which cleans up " \
+                      "invalid UTF8 characters in request URI and headers."
   gem.summary       = gem.description
-  gem.homepage      = "http://github.com/whitequark/rack-utf8_sanitizer"
+  gem.homepage      = "https://github.com/whitequark/rack-utf8_sanitizer"
 
   gem.files         = `git ls-files`.split($/)
   gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
   gem.require_paths = ["lib"]
 
-  gem.required_ruby_version = '>= 1.9.3'
+  gem.required_ruby_version = '>= 2.3'
 
   gem.add_dependency             "rack", '>= 1.0', '< 4.0'
 

data/test/test_utf8_sanitizer.rb

@@ -1,4 +1,5 @@
 # encoding:ascii-8bit
+# frozen_string_literal: true
 
 require 'bacon/colored_output'
 require 'cgi'
@@ -31,7 +32,7 @@ describe Rack::UTF8Sanitizer do
 
   describe "with invalid host input" do
     it "sanitizes host entity (SERVER_NAME)" do
-      host   = "host\xD0".force_encoding('UTF-8')
+      host   = "host\xD0".dup.force_encoding('UTF-8')
       env    = @app.({ "SERVER_NAME" => host })
       result = env["SERVER_NAME"]
 
@@ -42,8 +43,8 @@ describe Rack::UTF8Sanitizer do
 
   describe "with invalid UTF-8 input" do
     before do
-      @plain_input = "foo\xe0".force_encoding('UTF-8')
-      @uri_input   = "http://bar/foo%E0".force_encoding('UTF-8')
+      @plain_input = "foo\xe0".dup.force_encoding('UTF-8')
+      @uri_input   = "http://bar/foo%E0".dup.force_encoding('UTF-8')
     end
 
     behaves_like :does_sanitize_plain
@@ -52,7 +53,7 @@ describe Rack::UTF8Sanitizer do
 
   describe "with invalid, incorrectly percent-encoded UTF-8 URI input" do
     before do
-      @uri_input   = "http://bar/foo%E0\xe0".force_encoding('UTF-8')
+      @uri_input   = "http://bar/foo%E0\xe0".dup.force_encoding('UTF-8')
     end
 
     behaves_like :does_sanitize_uri
@@ -100,8 +101,8 @@ describe Rack::UTF8Sanitizer do
 
   describe "with valid UTF-8 input" do
     before do
-      @plain_input = "foo bar лол".force_encoding('UTF-8')
-      @uri_input   = "http://bar/foo+bar+%D0%BB%D0%BE%D0%BB".force_encoding('UTF-8')
+      @plain_input = "foo bar лол".dup.force_encoding('UTF-8')
+      @uri_input   = "http://bar/foo+bar+%D0%BB%D0%BE%D0%BB".dup.force_encoding('UTF-8')
     end
 
     behaves_like :identity_plain
@@ -109,7 +110,7 @@ describe Rack::UTF8Sanitizer do
 
     describe "with URI characters from reserved range" do
       before do
-        @uri_input   = "http://bar/foo+%2F%3A+bar+%D0%BB%D0%BE%D0%BB".force_encoding('UTF-8')
+        @uri_input   = "http://bar/foo+%2F%3A+bar+%D0%BB%D0%BE%D0%BB".dup.force_encoding('UTF-8')
       end
 
       behaves_like :identity_uri
@@ -118,7 +119,7 @@ describe Rack::UTF8Sanitizer do
 
   describe "with valid, not percent-encoded UTF-8 URI input" do
     before do
-      @uri_input   = "http://bar/foo+bar+лол".force_encoding('UTF-8')
+      @uri_input   = "http://bar/foo+bar+лол".dup.force_encoding('UTF-8')
       @encoded     = "http://bar/foo+bar+#{CGI.escape("лол")}"
     end
 
@@ -152,8 +153,8 @@ describe Rack::UTF8Sanitizer do
 
   describe "with frozen strings" do
     before do
-      @plain_input = "bar baz".freeze
-      @uri_input   = "http://bar/bar+baz".freeze
+      @plain_input = "bar baz"
+      @uri_input   = "http://bar/bar+baz"
     end
 
     it "preserves the frozen? status of input" do
@@ -165,9 +166,24 @@ describe Rack::UTF8Sanitizer do
     end
   end
 
+  describe "with mutable strings" do
+    before do
+      @plain_input = "bar baz".dup
+      @uri_input   = "http://bar/bar+baz".dup
+    end
+
+    it "preserves the frozen? status of input" do
+      env  = @app.({ "HTTP_USER_AGENT" => @plain_input,
+                     "REQUEST_PATH" => @uri_input })
+
+      env["HTTP_USER_AGENT"].should.not.be.frozen
+      env["REQUEST_PATH"].should.not.be.frozen
+    end
+  end
+
   describe "with symbols in the env" do
     before do
-      @uri_input = "http://bar/foo%E0\xe0".force_encoding('UTF-8')
+      @uri_input = "http://bar/foo%E0\xe0".dup.force_encoding('UTF-8')
     end
 
     it "sanitizes REQUEST_PATH with invalid UTF-8 URI input" do
@@ -183,7 +199,7 @@ describe Rack::UTF8Sanitizer do
 
   describe "with form data" do
     def request_env
-      @plain_input = "foo bar лол".force_encoding('UTF-8')
+      @plain_input = "foo bar лол".dup.force_encoding('UTF-8')
       {
          "REQUEST_METHOD" => "POST",
          "CONTENT_TYPE" => "application/x-www-form-urlencoded;foo=bar",
@@ -193,7 +209,7 @@ describe Rack::UTF8Sanitizer do
     end
 
     def sanitize_form_data(request_env = request_env())
-      @uri_input = "http://bar/foo+%2F%3A+bar+%D0%BB%D0%BE%D0%BB".force_encoding('UTF-8')
+      @uri_input = "http://bar/foo+%2F%3A+bar+%D0%BB%D0%BE%D0%BB".dup.force_encoding('UTF-8')
       @response_env = @app.(request_env)
       sanitized_input = @response_env['rack.input'].read
 
@@ -219,6 +235,16 @@ describe Rack::UTF8Sanitizer do
       @response_env.should == [400, {"Content-Type"=>"text/plain"}, ["Bad Request"]]
     end
 
+    it "Bad Request response can safety be mutated" do
+      @rack_input = BrokenIO.new
+      response_env = @app.(request_env)
+      response_env.should == [400, {"Content-Type"=>"text/plain"}, ["Bad Request"]]
+      response_env[1]["Set-Cookie"] = "you_are_admin"
+
+      response_env = @app.(request_env)
+      response_env[1]["Set-Cookie"].should == nil
+    end
+
     it "sanitizes StringIO rack.input" do
       input = "foo=bla&quux=bar"
       @rack_input = StringIO.new input
@@ -252,6 +278,18 @@ describe Rack::UTF8Sanitizer do
       end
     end
 
+    it "sanitizes the rack body if the charset is present and utf-8" do
+      input =  "name=#{CGI.escape("まつもと")}"
+      @rack_input = StringIO.new input
+
+      env = request_env.update('CONTENT_TYPE' => "application/x-www-form-urlencoded; charset=utf-8")
+      sanitize_form_data(env) do |sanitized_input|
+        sanitized_input.encoding.should == Encoding::UTF_8
+        sanitized_input.should.be.valid_encoding
+        sanitized_input.should == input
+      end
+    end
+
     it "strip UTF-8 BOM from StringIO rack.input" do
       input = %(\xef\xbb\xbf{"Hello": "World"})
       @rack_input = StringIO.new input
@@ -327,6 +365,18 @@ describe Rack::UTF8Sanitizer do
       end
     end
 
+    it "does not sanitize the rack body if the charset is present and not utf-8" do
+      input = "name=".encode("Shift_JIS") + CGI.escape("まつもと".encode("Shift_JIS", "UTF-8"))
+      @rack_input = StringIO.new input
+
+      env = request_env.update('CONTENT_TYPE' => "application/x-www-form-urlencoded; charset=Shift_JIS")
+      sanitize_form_data(env) do |sanitized_input|
+        sanitized_input.encoding.should == Encoding::SHIFT_JIS
+        sanitized_input.should.be.valid_encoding
+        sanitized_input.should == input
+      end
+    end
+
     it "adjusts content-length when replacing input" do
       input =  "foo=bla&quux=bar\xED"
       @rack_input = StringIO.new input
@@ -351,25 +401,25 @@ describe Rack::UTF8Sanitizer do
 
     it "optionally sanitizes null bytes with the replace strategy" do
       @app = Rack::UTF8Sanitizer.new(-> env { env }, sanitize_null_bytes: true)
-      input =  "foo=bla&quux=bar\x00"
+      input = "foo=bla\xED&quux=bar\x00"
       @rack_input = StringIO.new input
 
       sanitize_form_data do |sanitized_input|
         sanitized_input.encoding.should == Encoding::UTF_8
         sanitized_input.should.be.valid_encoding
-        sanitized_input.should == "foo=bla&quux=bar"
+        sanitized_input.should == "foo=bla%EF%BF%BD&quux=bar"
       end
     end
 
     it "optionally sanitizes encoded null bytes with the replace strategy" do
       @app = Rack::UTF8Sanitizer.new(-> env { env }, sanitize_null_bytes: true)
-      input =  "foo=bla&quux=bar%00"
+      input = "foo=bla%ED&quux=bar%00"
       @rack_input = StringIO.new input
 
       sanitize_form_data do |sanitized_input|
         sanitized_input.encoding.should == Encoding::UTF_8
         sanitized_input.should.be.valid_encoding
-        sanitized_input.should == "foo=bla&quux=bar"
+        sanitized_input.should == "foo=bla%EF%BF%BD&quux=bar"
       end
     end
 
@@ -392,6 +442,16 @@ describe Rack::UTF8Sanitizer do
         sanitize_form_data
       end
     end
+
+    it "gives precedence to encoding errors with the exception strategy and null byte sanitisation" do
+      @app = Rack::UTF8Sanitizer.new(-> env { env }, sanitize_null_bytes: true, strategy: :exception)
+      input = "foo=bla\x00&quux=bar\xED"
+      @rack_input = StringIO.new input
+
+      should.raise(EncodingError) do
+        sanitize_form_data
+      end
+    end
   end
 
   describe "with custom content-type" do
@@ -424,7 +484,7 @@ describe Rack::UTF8Sanitizer do
 
   describe "with custom content-type" do
     def request_env
-      @plain_input = "foo bar лол".force_encoding('UTF-8')
+      @plain_input = "foo bar лол".dup.force_encoding('UTF-8')
       {
           "REQUEST_METHOD" => "POST",
           "CONTENT_TYPE" => "application/vnd.api+json",
@@ -434,7 +494,7 @@ describe Rack::UTF8Sanitizer do
     end
 
     def sanitize_data(request_env = request_env())
-      @uri_input = "http://bar/foo+%2F%3A+bar+%D0%BB%D0%BE%D0%BB".force_encoding('UTF-8')
+      @uri_input = "http://bar/foo+%2F%3A+bar+%D0%BB%D0%BE%D0%BB".dup.force_encoding('UTF-8')
       @response_env = @app.(request_env)
       sanitized_input = @response_env['rack.input'].read
 
@@ -508,7 +568,7 @@ describe Rack::UTF8Sanitizer do
 
   describe "with only and/or except options" do
     before do
-      @plain_input = "foo\xe0".force_encoding('UTF-8')
+      @plain_input = "foo\xe0".dup.force_encoding('UTF-8')
     end
 
     def request_env
@@ -565,7 +625,7 @@ describe Rack::UTF8Sanitizer do
 
   describe "with custom strategy" do
     def request_env
-      @plain_input = "foo bar лол".force_encoding('UTF-8')
+      @plain_input = "foo bar лол".dup.force_encoding('UTF-8')
       {
           "REQUEST_METHOD" => "POST",
           "CONTENT_TYPE" => "application/json",
@@ -575,7 +635,7 @@ describe Rack::UTF8Sanitizer do
     end
 
     def sanitize_data(request_env = request_env())
-      @uri_input = "http://bar/foo+%2F%3A+bar+%D0%BB%D0%BE%D0%BB".force_encoding('UTF-8')
+      @uri_input = "http://bar/foo+%2F%3A+bar+%D0%BB%D0%BE%D0%BB".dup.force_encoding('UTF-8')
       @response_env = @app.(request_env)
       sanitized_input = @response_env['rack.input'].read
 
@@ -609,7 +669,7 @@ describe Rack::UTF8Sanitizer do
     it "accepts a proc as a strategy" do
       truncate = -> (input, sanitize_null_bytes:) do
         sanitize_null_bytes.should == false
-        'replace'.force_encoding(Encoding::UTF_8)
+        "replace".dup.force_encoding(Encoding::UTF_8)
       end
 
       @app = Rack::UTF8Sanitizer.new(-> env { env }, strategy: truncate)
@@ -628,7 +688,7 @@ describe Rack::UTF8Sanitizer do
     it "accepts a proc as a strategy and passes along sanitize_null_bytes" do
       truncate = -> (input, sanitize_null_bytes:) do
         sanitize_null_bytes.should == true
-        'replace'.force_encoding(Encoding::UTF_8)
+        "replace".dup.force_encoding(Encoding::UTF_8)
       end
 
       @app = Rack::UTF8Sanitizer.new(-> env { env }, sanitize_null_bytes: true, strategy: truncate)

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rack-utf8_sanitizer
 version: !ruby/object:Gem::Version
-  version: 1.9.0
+  version: 1.10.0
 platform: ruby
 authors:
-- whitequark
+- Catherine
 autorequire:
 bindir: bin
 cert_chain: []
-date: 2023-07-05 00:00:00.000000000 Z
+date: 2025-01-08 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rack
@@ -93,7 +93,7 @@ files:
 - lib/rack/utf8_sanitizer.rb
 - rack-utf8_sanitizer.gemspec
 - test/test_utf8_sanitizer.rb
-homepage: http://github.com/whitequark/rack-utf8_sanitizer
+homepage: https://github.com/whitequark/rack-utf8_sanitizer
 licenses:
 - MIT
 metadata: {}
@@ -105,14 +105,14 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: 1.9.3
+      version: '2.3'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubygems_version: 3.2.5
+rubygems_version: 3.3.15
 signing_key:
 specification_version: 4
 summary: Rack::UTF8Sanitizer is a Rack middleware which cleans up invalid UTF8 characters