rack-utf8_sanitizer 1.3.1 → 1.11.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
-SHA1:
-  metadata.gz: a21608edab6e71fde026d8c9b5fdf6afaeb831ab
-  data.tar.gz: d3648a488dd74a3ea6357d39cc87b1d486af2c1c
+SHA256:
+  metadata.gz: 14d5b3232eb52814e46f5960b435851d0b0a6bdd6d18cfb1220adc08c5ecdb1e
+  data.tar.gz: 40d9329c056a74dd0328ac7af2fa4183799cb451cd1795630f6754f24718324a
 SHA512:
-  metadata.gz: 8c8c7e485b0ca9584951b2689b090feaf4441eb96c1fb5ee3fd8f7c79bddcb4bf4d005b69e56a1fd0675b1e96de07dca21906ccca4b255e3893007278e819afe
-  data.tar.gz: b8dcdd43af94a1277b978184ba8e5b26c782f269f646805839d102cf216f7e8d1e375a4104ace50357c23c3761afd00acaef441ac13bcc05ae1555e719e71079
+  metadata.gz: 11ffb7389bcbe67ef49ce1d29829de920d72c44799c623924dd0b93e578d89414de07bb4ef315d50d13e5463825b1d4727f999c10faafb1d507e52875caa09be
+  data.tar.gz: 4b9115c9909dc1cfd8378451f0974808f5d834bd7daaaca012d04413e48ed80cc5f8bcb10f6307300b982cf5f56080dafada5963126b5563df18ff83772dcfd8

data/.editorconfig

@@ -0,0 +1,17 @@
+root = true
+
+[*]
+indent_style = space
+indent_size = 2
+end_of_line = lf
+charset = utf-8
+trim_trailing_whitespace = true
+insert_final_newline = true
+
+[*.md]
+indent_style = space
+indent_size = 2
+
+[*.y{a,}ml]
+indent_style = space
+indent_size = 2

data/.github/dependabot.yml

@@ -0,0 +1,6 @@
+version: 2
+updates:
+  - package-ecosystem: "github-actions"
+    directory: "/"
+    schedule:
+      interval: "weekly"

data/.github/workflows/ci.yml

@@ -0,0 +1,40 @@
+name: CI
+
+on: [push, pull_request]
+
+jobs:
+  test:
+
+    runs-on: ${{ matrix.os }}
+
+    strategy:
+      fail-fast: false
+      matrix:
+        os: [ubuntu-latest]
+        ruby:
+          - 2.5
+          - 2.6
+          - 2.7
+          - "3.0"
+          - 3.1
+          - 3.2
+          - 3.3
+          - 3.4
+          - ruby-head
+          - jruby-9.3
+          - jruby-9.4
+          - jruby-10.0
+          - jruby-head
+        include:
+          - os: ubuntu-22.04
+            ruby: jruby-9.2
+
+    steps:
+    - uses: actions/checkout@v6
+    - name: Set up Ruby
+      uses: ruby/setup-ruby@v1
+      with:
+        bundler-cache: true # 'bundle install' and cache gems
+        ruby-version: ${{ matrix.ruby }}
+    - name: Run tests
+      run: bundle exec rake

data/CHANGELOG.md

@@ -1,7 +1,119 @@
 Changelog
 =========
 
-Master
+v1.11.0 (2025-12-04)
+-------------------------
+
+Bugs fixed:
+
+  * Return HTTP 400 when Content-Length is too large (Benjamin Quorning, #103)
+
+v1.10.1 (2025-01-10)
+-------------------------
+
+Bugs fixed:
+
+  * Fix `URI::RFC2396_PARSER` issue with older Rubies (Tekin Süleyman, #94)
+
+
+v1.10.0 (2025-01-08)
+-------------------------
+
+Changes:
+
+  * Require Ruby 2.3.0+.  (Jean Boussier, #80)
+
+Bugs fixed:
+
+  * Skip sanitizing the request body if the charset is non-utf-8 (#84)
+  * Don't use a mutable constant as Rack response (Jean Boussier, #86)
+
+Chores:
+
+  * Add the `frozen_string_literal` header (Benjamin Quorning, #90)
+  * Avoid deprecation warming by switching from `URI::DEFAULT_PARSER` to `URI::RFC2396_PARSER` (Roman Gaufman, #92)
+
+Performance:
+
+  * Use Content-Length to read the request body if available (Jean Boussier, #80)
+  * Avoid 2nd degree polynomial regexp for sanitizing content type (Jean Boussier, #82)
+  * Use `Regexp#match?` over `String#=~` when testing for null bytes (Geoff Harcourt, #85)
+
+v1.9.1 (2023-08-31)
+-------------------------
+
+Bugs fixed:
+
+  * Fix null byte sanitisation (Szymon Madeja, #78)
+
+v1.9.0 (2023-07-06)
+-------------------------
+
+  * Optionally sanitize null bytes (James Coleman, #75)
+  * CI: add Ruby 3.2 (Peter Goldstein, #71)
+
+v1.8.0 (2022-10-25)
+-------------------------
+
+Bugs fixed:
+
+  * Handle EOFError (Kir Shatrov, #57)
+
+Features implemented:
+
+  * Allow Rack version 3 (Alexander Popov, #66)
+  * Various CI chores (Olle Jonsson)
+  * Move to GitHub Actions, configure Dependabot (Peter Goldstein, #62, #64)
+
+v1.7.0 (2020-05-05)
+-------------------------
+
+  * Resolve Ruby warnings about `URI.escape` (Alexander Popov, #53)
+  * README: better reflect that this also can sanitize text bodies (Zach McCormick, #47)
+  * Update documentation on exception strategy handler (Josh Frankel, #52)
+
+v1.6.0 (2018-06-06)
+-------------------------
+
+Bugs fixed:
+
+  * Add sanitation of cookie header (John Hager, #45)
+
+v1.5.0 (2018-02-16)
+-------------------------
+
+Bugs fixed:
+
+  * Sanitize `nil` in `sanitize_uri_encoded_string` (David Čepelík, #44)
+
+Features implemented:
+
+  * Add `:only` and `:except` options (John Hager, #43)
+  * Add strategies to rack-utf8_sanitizer (John Hager, #41)
+
+    ```rb
+    # Example usage in Rails config/application.rb:
+    config.middleware.insert(0, Rack::UTF8Sanitizer, strategy: :exception)
+    ```
+
+v1.4.0 (2016-03-07)
+-------------------------
+
+Performance:
+
+  * Use more performant `%char` decoding `.hex.chr` (Martin Emde, #36)
+  * Make `HTTP_` a constant to avoid creating the string every loop (Martin Emde, #35)
+
+Features implemented:
+
+  * Add SERVER_NAME to list of sanitization (Denis Lysenko, 9644371)
+
+Chores:
+
+  * Add license to gemspec (Robert Reiz, #38)
+
+
+v1.3.2 (2015-12-23)
 -------------------------
 
 API modifications:
@@ -10,6 +122,9 @@ Features implemented:
 
 Bugs fixed:
 
+  * Strip UTF-8 Byte Order Mark from the request body (Jean Boussier, #29)
+  * Add options to #initialize to allow configurable sanitizable content types (Shelby Switzer, #30)
+
 v1.3.1 (2015-07-09)
 -------------------------
 

data/Gemfile

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 source 'https://rubygems.org'
 
 # Specify your gem's dependencies in rack-utf8_sanitizer.gemspec

data/README.md

@@ -1,6 +1,9 @@
 # Rack::UTF8Sanitizer
 
-Rack::UTF8Sanitizer is a Rack middleware which cleans up invalid UTF8 characters in request URI and headers.
+Rack::UTF8Sanitizer is a Rack middleware which cleans up invalid UTF8 characters in request URI and headers. Additionally,
+it cleans up invalid UTF8 characters in the request body (depending on the configurable content type filters) by reading
+the input into a string, sanitizing the string, then replacing the Rack input stream with a rewindable input stream backed
+by the sanitized string.
 
 ## Installation
 
@@ -43,6 +46,90 @@ For fields with "raw data", the algorithm is applied once and the (UTF-8 encoded
 
 For fields with "percent-encoded data", the algorithm is applied twice to catch both invalid characters appearing as-is and invalid characters appearing in the percent encoding. The percent encoded, ASCII-8BIT encoded result is left in the environment.
 
+### Sanitizable content types
+
+The default content types to be sanitized are 'text/plain', 'application/x-www-form-urlencoded', 'application/json', 'text/javascript'. You may wish to modify this, for example if your app accepts specific or custom media types in the CONTENT_TYPE header. If you want to change the sanitizable content types, you can pass options when using Rack::UTF8Sanitizer.
+
+To add sanitizable content types to the list of defaults, pass the `additional_content_types` options when using Rack::UTF8Sanitizer, e.g.
+
+``` ruby
+config.middleware.insert 0, Rack::UTF8Sanitizer, additional_content_types: ['application/vnd.api+json']
+```
+
+To explicitly set sanitizable content types and override the defaults, use the `sanitizable_content_types` option:
+
+``` ruby
+config.middleware.insert 0, Rack::UTF8Sanitizer, sanitizable_content_types: ['application/vnd.api+json']
+```
+
+### Whitelist/Blacklist Rack Env Keys
+
+Using the `:only` and `:except` keys you can skip sanitation of values in the Rack Env. `:only` and `:except` are arrays that can contain strings or regular expressions.
+
+Only sanitize the body, query string, and url of a request.
+
+```ruby
+config.middleware.insert 0, Rack::UTF8Sanitizer, only: ['rack.input', 'PATH_INFO', 'QUERY_STRING']
+```
+
+Sanitize everything except HTTP headers.
+
+```ruby
+config.middleware.insert 0, Rack::UTF8Sanitizer, except: [/HTTP_.+/]
+```
+
+### Strategies
+
+There are two built in strategies for handling invalid characters. The default strategy is `:replace`, which will cause any invalid characters to be replaces with the unicode replacement character (�). The second built in strategy is `:exception` which will cause an `EncodingError` exception to be raised if invalid characters are found (the exception can then be handled by another Rack middleware).
+
+This is an example of handling the `:exception` strategy with additional middleware:
+
+```ruby
+require "./your/middleware/directory/utf8_sanitizer_exception_handler.rb"
+
+config.middleware.insert 0, Rack::UTF8SanitizerExceptionHandler
+config.middleware.insert_after Rack::UTF8SanitizerExceptionHandler, Rack::UTF8Sanitizer, strategy: :exception
+```
+
+Note: The exception handling middleware must be inserted before `Rack::UTF8Sanitizer`
+
+```ruby
+module Rack
+  class UTF8SanitizerExceptionHandler
+    def initialize(app)
+      @app = app
+    end
+
+    def call(env)
+      @app.call(env)
+    rescue EncodingError => exception
+      # OPTIONAL: Add error logging service of your choice here
+      return [400, {}, ["Bad Request"]]
+    end
+  end
+end
+```
+
+An object that responds to `#call` and accepts the offending string with invalid characters as an argument can also be passed as a `:strategy`. This is how you can define custom strategies.
+
+```ruby
+config.middleware.insert 0, Rack::UTF8Sanitizer, strategy: :exception
+```
+
+```ruby
+replace_string = lambda do |_invalid, sanitize_null_bytes: false|
+  Rails.logger.warn('Replacing invalid string')
+
+  '<Bad Encoding>'.freeze
+end
+
+config.middleware.insert 0, Rack::UTF8Sanitizer, strategy: replace_string
+```
+
+### Sanitizing Null Bytes
+
+While null bytes are valid UTF-8, it can be useful to further restrict the valid character set to exclude null bytes. For example, PostgreSQL text columns do not allow storing null bytes. Passing `sanitize_null_bytes: true` in the configuration hash enables sanitizing null bytes, and the two built-in strategies both support this feature. Custom strategies should accept a keyword argument `sanitize_null_bytes` containing this configuration value.
+
 ## Contributing
 
 1. Fork it
@@ -50,3 +137,5 @@ For fields with "percent-encoded data", the algorithm is applied twice to catch
 3. Commit your changes (`git commit -am 'Add some feature'`)
 4. Push to the branch (`git push origin my-new-feature`)
 5. Create new Pull Request
+
+To run the tests, run `rake spec` in the project directory.

data/Rakefile

@@ -1,3 +1,5 @@
+# frozen_string_literal: true
+
 require "bundler/gem_tasks"
 
 task :default => :spec

data/lib/rack/utf8_sanitizer.rb

@@ -1,21 +1,63 @@
 # encoding: ascii-8bit
+# frozen_string_literal: true
 
 require 'uri'
 require 'stringio'
+require 'rack/request'
 
 module Rack
   class UTF8Sanitizer
     StringIO = ::StringIO
+    NULL_BYTE_REGEX = /\x00/.freeze
 
-    def initialize(app)
+    class InvalidStream < IOError; end
+    class NullByteInString < StandardError; end
+
+    # options[:sanitizable_content_types] Array
+    # options[:additional_content_types] Array
+    def initialize(app, options={})
       @app = app
+      @strategy = build_strategy(options)
+      @sanitizable_content_types = options[:sanitizable_content_types]
+      @sanitizable_content_types ||= SANITIZABLE_CONTENT_TYPES + (options[:additional_content_types] || [])
+      @only = Array(options[:only]).flatten
+      @except = Array(options[:except]).flatten
+      @sanitize_null_bytes = options.fetch(:sanitize_null_bytes, false)
     end
 
     def call(env)
-      @app.call(sanitize(env))
+      begin
+        env = sanitize(env)
+      rescue EOFError, InvalidStream
+        return [400, { "Content-Type" => "text/plain" }, ["Bad Request"]]
+      end
+      @app.call(env)
     end
 
-    # http://rack.rubyforge.org/doc/SPEC.html
+    DEFAULT_STRATEGIES = {
+      replace: lambda do |input, sanitize_null_bytes: false|
+        input.
+          force_encoding(Encoding::ASCII_8BIT).
+          encode!(Encoding::UTF_8,
+                  invalid: :replace,
+                  undef:   :replace)
+        if sanitize_null_bytes
+          input = input.gsub(NULL_BYTE_REGEX, "")
+        end
+        input
+      end,
+      exception: lambda do |input, sanitize_null_bytes: false|
+        input.
+          force_encoding(Encoding::ASCII_8BIT).
+          encode!(Encoding::UTF_8)
+        if sanitize_null_bytes && NULL_BYTE_REGEX.match?(input)
+          raise NullByteInString
+        end
+        input
+      end
+    }.freeze
+
+    # https://github.com/rack/rack/blob/main/SPEC.rdoc
     URI_FIELDS  = %w(
         SCRIPT_NAME
         REQUEST_PATH REQUEST_URI PATH_INFO
@@ -23,26 +65,32 @@ module Rack
         HTTP_REFERER
         ORIGINAL_FULLPATH
         ORIGINAL_SCRIPT_NAME
-    )
+        SERVER_NAME
+    ).freeze
 
     SANITIZABLE_CONTENT_TYPES = %w(
       text/plain
       application/x-www-form-urlencoded
       application/json
       text/javascript
-    )
+    ).freeze
 
     URI_ENCODED_CONTENT_TYPES = %w(
       application/x-www-form-urlencoded
-    )
+    ).freeze
+
+    HTTP_ = 'HTTP_'
 
     def sanitize(env)
       sanitize_rack_input(env)
+      sanitize_cookies(env)
       env.each do |key, value|
+        next if skip?(key)
+
         if URI_FIELDS.include?(key)
           env[key] = transfer_frozen(value,
               sanitize_uri_encoded_string(value))
-        elsif key.to_s.start_with?("HTTP_")
+        elsif key.to_s.start_with?(HTTP_)
           # Just sanitize the headers and leave them in UTF-8. There is
           # no reason to have UTF-8 in headers, but if it's valid, let it be.
           env[key] = transfer_frozen(value,
@@ -53,18 +101,33 @@ module Rack
 
     protected
 
+    def skip?(rack_env_key)
+      return true if !@except.empty? && @except.any? { |matcher| rack_env_key[matcher] }
+      return true if !@only.empty? && @only.none? { |matcher| rack_env_key[matcher] }
+
+      false
+    end
+
+    def build_strategy(options)
+      strategy = options.fetch(:strategy) { :replace }
+
+      return strategy unless DEFAULT_STRATEGIES.key?(strategy)
+
+      DEFAULT_STRATEGIES[strategy]
+    end
+
     def sanitize_rack_input(env)
-      # https://github.com/rack/rack/blob/master/lib/rack/request.rb#L42
-      # Logic borrowed from Rack::Request#media_type,#media_type_params,#content_charset
-      # Ignoring charset in content type.
-      content_type   = env['CONTENT_TYPE']
-      content_type &&= content_type.split(/\s*[;,]\s*/, 2).first
-      content_type &&= content_type.downcase
-      return unless SANITIZABLE_CONTENT_TYPES.any? {|type| content_type == type }
+      request = Rack::Request.new(env)
+      content_type = request.media_type
+      return unless @sanitizable_content_types.any? {|type| content_type == type }
+
+      charset = request.content_charset
+      return if charset && charset.downcase != 'utf-8'
+
       uri_encoded = URI_ENCODED_CONTENT_TYPES.any? {|type| content_type == type}
 
-      if env["rack.input"]
-        sanitized_input = sanitize_io(env['rack.input'], uri_encoded)
+      if env['rack.input']
+        sanitized_input = sanitize_io(env['rack.input'], uri_encoded, env['CONTENT_LENGTH']&.to_i)
 
         env['rack.input'] = sanitized_input
         env['CONTENT_LENGTH'] &&= sanitized_input.size.to_s
@@ -106,9 +169,14 @@ module Rack
       end
     end
 
-    def sanitize_io(io, uri_encoded = false)
-      input = io.read
-      sanitized_input = sanitize_string(input)
+    def sanitize_io(io, uri_encoded = false, content_length = nil)
+      input = if content_length && content_length >= 0
+        io.read(content_length)
+      else
+        io.read
+      end
+      raise InvalidStream if input.nil?
+      sanitized_input = sanitize_string(strip_byte_order_mark(input))
       if uri_encoded
         sanitized_input = sanitize_uri_encoded_string(sanitized_input).
           force_encoding(Encoding::UTF_8)
@@ -117,6 +185,20 @@ module Rack
       SanitizedRackInput.new(io, StringIO.new(sanitized_input))
     end
 
+    # Cookies need to be split and then sanitized as url encoded strings
+    # since the cookie string itself is not url encoded (separated by `;`),
+    # and the normal method of `sanitize_uri_encoded_string` would break
+    # later cookie parsing in the case that a cookie value contained an
+    # encoded `;`.
+    def sanitize_cookies(env)
+      return unless env['HTTP_COOKIE']
+
+      env['HTTP_COOKIE'] = env['HTTP_COOKIE']
+        .split(/[;,] */n)
+        .map { |cookie| sanitize_uri_encoded_string(cookie) }
+        .join('; ')
+    end
+
     # URI.encode/decode expect the input to be in ASCII-8BIT.
     # However, there could be invalid UTF-8 characters both in
     # raw and percent-encoded form.
@@ -126,6 +208,7 @@ module Rack
     #
     # The result is guaranteed to be UTF-8-safe.
     def sanitize_uri_encoded_string(input)
+      return input if input.nil?
       decoded_value = decode_string(input)
       reencode_string(decoded_value)
     end
@@ -143,7 +226,8 @@ module Rack
 
     # This regexp matches all 'unreserved' characters from RFC3986 (2.3),
     # plus all multibyte UTF-8 characters.
-    UNRESERVED_OR_UTF8 = /[A-Za-z0-9\-._~\x80-\xFF]/
+    UNRESERVED_OR_UTF8 = /[A-Za-z0-9\-._~\x80-\xFF]/.freeze
+    UNRESERVED_OR_UTF8_OR_NULL = /[A-Za-z0-9\-._~\x00\x80-\xFF]/.freeze
 
     # RFC3986, 2.2 states that the characters from 'reserved' group must be
     # protected during normalization (which is what UTF8Sanitizer does).
@@ -152,9 +236,10 @@ module Rack
     # enough for our task.
     def unescape_unreserved(input)
       input.gsub(/%([a-f\d]{2})/i) do |encoded|
-        decoded = [$1.hex].pack('C')
+        decoded = $1.hex.chr
 
-        if decoded =~ UNRESERVED_OR_UTF8
+        decodable_regex = @sanitize_null_bytes ? UNRESERVED_OR_UTF8_OR_NULL : UNRESERVED_OR_UTF8
+        if decoded =~ decodable_regex
           decoded
         else
           encoded
@@ -173,21 +258,21 @@ module Rack
     # Performs the reverse function of `unescape_unreserved`. Unlike
     # the previous function, we can reuse the logic in URI#encode
     def escape_unreserved(input)
-      URI.encode(input, UNSAFE)
+      if Object.const_defined?("URI::RFC2396_PARSER")
+        URI::RFC2396_PARSER.escape(input, UNSAFE)
+      else
+        URI::DEFAULT_PARSER.escape(input, UNSAFE)
+      end
     end
 
     def sanitize_string(input)
       if input.is_a? String
         input = input.dup.force_encoding(Encoding::UTF_8)
 
-        if input.valid_encoding?
+        if input.valid_encoding? && !(@sanitize_null_bytes && input =~ NULL_BYTE_REGEX)
           input
         else
-          input.
-            force_encoding(Encoding::ASCII_8BIT).
-            encode!(Encoding::UTF_8,
-                    invalid: :replace,
-                    undef:   :replace)
+          @strategy.call(input, sanitize_null_bytes: @sanitize_null_bytes)
         end
       else
         input
@@ -201,5 +286,13 @@ module Rack
         to
       end
     end
+
+    UTF8_BOM = "\xef\xbb\xbf".dup.force_encoding(Encoding::BINARY).freeze
+    UTF8_BOM_SIZE = UTF8_BOM.bytesize
+
+    def strip_byte_order_mark(input)
+      return input unless input.start_with?(UTF8_BOM)
+      input.byteslice(UTF8_BOM_SIZE..-1)
+    end
   end
 end

data/rack-utf8_sanitizer.gemspec

@@ -1,23 +1,24 @@
 # -*- encoding: utf-8 -*-
+# frozen_string_literal: true
 
 Gem::Specification.new do |gem|
   gem.name          = "rack-utf8_sanitizer"
-  gem.version       = '1.3.1'
-  gem.authors       = ["Peter Zotov"]
+  gem.version       = '1.11.1'
+  gem.authors       = ["Catherine"]
+  gem.license       = "MIT"
   gem.email         = ["whitequark@whitequark.org"]
-  gem.description   = %{Rack::UTF8Sanitizer is a Rack middleware which cleans up } <<
-                      %{invalid UTF8 characters in request URI and headers.}
+  gem.description   = "Rack::UTF8Sanitizer is a Rack middleware which cleans up " \
+                      "invalid UTF8 characters in request URI and headers."
   gem.summary       = gem.description
-  gem.homepage      = "http://github.com/whitequark/rack-utf8_sanitizer"
+  gem.homepage      = "https://github.com/whitequark/rack-utf8_sanitizer"
 
   gem.files         = `git ls-files`.split($/)
-  gem.executables   = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
   gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
   gem.require_paths = ["lib"]
 
-  gem.required_ruby_version = '>= 1.9'
+  gem.required_ruby_version = '>= 2.3'
 
-  gem.add_dependency             "rack", '~> 1.0'
+  gem.add_dependency             "rack", '>= 1.0', '< 4.0'
 
   gem.add_development_dependency "bacon"
   gem.add_development_dependency "bacon-colored_output"

data/test/test_utf8_sanitizer.rb

@@ -1,6 +1,8 @@
 # encoding:ascii-8bit
+# frozen_string_literal: true
 
 require 'bacon/colored_output'
+require 'cgi'
 require 'rack/utf8_sanitizer'
 
 describe Rack::UTF8Sanitizer do
@@ -28,10 +30,21 @@ describe Rack::UTF8Sanitizer do
     end
   end
 
+  describe "with invalid host input" do
+    it "sanitizes host entity (SERVER_NAME)" do
+      host   = "host\xD0".dup.force_encoding('UTF-8')
+      env    = @app.({ "SERVER_NAME" => host })
+      result = env["SERVER_NAME"]
+
+      result.encoding.should == Encoding::US_ASCII
+      result.should.be.valid_encoding
+    end
+  end
+
   describe "with invalid UTF-8 input" do
     before do
-      @plain_input = "foo\xe0".force_encoding('UTF-8')
-      @uri_input   = "http://bar/foo%E0".force_encoding('UTF-8')
+      @plain_input = "foo\xe0".dup.force_encoding('UTF-8')
+      @uri_input   = "http://bar/foo%E0".dup.force_encoding('UTF-8')
     end
 
     behaves_like :does_sanitize_plain
@@ -40,7 +53,7 @@ describe Rack::UTF8Sanitizer do
 
   describe "with invalid, incorrectly percent-encoded UTF-8 URI input" do
     before do
-      @uri_input   = "http://bar/foo%E0\xe0".force_encoding('UTF-8')
+      @uri_input   = "http://bar/foo%E0\xe0".dup.force_encoding('UTF-8')
     end
 
     behaves_like :does_sanitize_uri
@@ -88,8 +101,8 @@ describe Rack::UTF8Sanitizer do
 
   describe "with valid UTF-8 input" do
     before do
-      @plain_input = "foo bar лол".force_encoding('UTF-8')
-      @uri_input   = "http://bar/foo+bar+%D0%BB%D0%BE%D0%BB".force_encoding('UTF-8')
+      @plain_input = "foo bar лол".dup.force_encoding('UTF-8')
+      @uri_input   = "http://bar/foo+bar+%D0%BB%D0%BE%D0%BB".dup.force_encoding('UTF-8')
     end
 
     behaves_like :identity_plain
@@ -97,7 +110,7 @@ describe Rack::UTF8Sanitizer do
 
     describe "with URI characters from reserved range" do
       before do
-        @uri_input   = "http://bar/foo+%2F%3A+bar+%D0%BB%D0%BE%D0%BB".force_encoding('UTF-8')
+        @uri_input   = "http://bar/foo+%2F%3A+bar+%D0%BB%D0%BE%D0%BB".dup.force_encoding('UTF-8')
       end
 
       behaves_like :identity_uri
@@ -106,7 +119,8 @@ describe Rack::UTF8Sanitizer do
 
   describe "with valid, not percent-encoded UTF-8 URI input" do
     before do
-      @uri_input   = "http://bar/foo+bar+лол".force_encoding('UTF-8')
+      @uri_input   = "http://bar/foo+bar+лол".dup.force_encoding('UTF-8')
+      @encoded     = "http://bar/foo+bar+#{CGI.escape("лол")}"
     end
 
     it "does not change URI-like entity (REQUEST_PATH)" do
@@ -115,7 +129,7 @@ describe Rack::UTF8Sanitizer do
 
       result.encoding.should == Encoding::US_ASCII
       result.should.be.valid_encoding
-      result.should == URI.encode(@uri_input)
+      result.should == @encoded
     end
   end
 
@@ -139,8 +153,8 @@ describe Rack::UTF8Sanitizer do
 
   describe "with frozen strings" do
     before do
-      @plain_input = "bar baz".freeze
-      @uri_input   = "http://bar/bar+baz".freeze
+      @plain_input = "bar baz"
+      @uri_input   = "http://bar/bar+baz"
     end
 
     it "preserves the frozen? status of input" do
@@ -152,9 +166,24 @@ describe Rack::UTF8Sanitizer do
     end
   end
 
+  describe "with mutable strings" do
+    before do
+      @plain_input = "bar baz".dup
+      @uri_input   = "http://bar/bar+baz".dup
+    end
+
+    it "preserves the frozen? status of input" do
+      env  = @app.({ "HTTP_USER_AGENT" => @plain_input,
+                     "REQUEST_PATH" => @uri_input })
+
+      env["HTTP_USER_AGENT"].should.not.be.frozen
+      env["REQUEST_PATH"].should.not.be.frozen
+    end
+  end
+
   describe "with symbols in the env" do
     before do
-      @uri_input = "http://bar/foo%E0\xe0".force_encoding('UTF-8')
+      @uri_input = "http://bar/foo%E0\xe0".dup.force_encoding('UTF-8')
     end
 
     it "sanitizes REQUEST_PATH with invalid UTF-8 URI input" do
@@ -170,7 +199,7 @@ describe Rack::UTF8Sanitizer do
 
   describe "with form data" do
     def request_env
-      @plain_input = "foo bar лол".force_encoding('UTF-8')
+      @plain_input = "foo bar лол".dup.force_encoding('UTF-8')
       {
          "REQUEST_METHOD" => "POST",
          "CONTENT_TYPE" => "application/x-www-form-urlencoded;foo=bar",
@@ -179,8 +208,8 @@ describe Rack::UTF8Sanitizer do
       }
     end
 
-    def sanitize_form_data(request_env = request_env)
-      @uri_input = "http://bar/foo+%2F%3A+bar+%D0%BB%D0%BE%D0%BB".force_encoding('UTF-8')
+    def sanitize_form_data(request_env = request_env())
+      @uri_input = "http://bar/foo+%2F%3A+bar+%D0%BB%D0%BE%D0%BB".dup.force_encoding('UTF-8')
       @response_env = @app.(request_env)
       sanitized_input = @response_env['rack.input'].read
 
@@ -194,6 +223,34 @@ describe Rack::UTF8Sanitizer do
       @response_env['rack.input'].close
     end
 
+    class BrokenIO < StringIO
+      def read
+        raise EOFError
+      end
+    end
+
+    it "returns HTTP 400 on EOF" do
+      @rack_input = BrokenIO.new
+      @response_env = @app.(request_env)
+      @response_env.should == [400, {"Content-Type"=>"text/plain"}, ["Bad Request"]]
+    end
+
+    it "Bad Request response can safety be mutated" do
+      @rack_input = BrokenIO.new
+      response_env = @app.(request_env)
+      response_env.should == [400, {"Content-Type"=>"text/plain"}, ["Bad Request"]]
+      response_env[1]["Set-Cookie"] = "you_are_admin"
+
+      response_env = @app.(request_env)
+      response_env[1]["Set-Cookie"].should == nil
+    end
+
+    it "returns HTTP 400 if CONTENT_LENGTH is larger than actual length of rack.input" do
+      @rack_input = StringIO.new("")
+      response_env = @app.(request_env.merge("CONTENT_LENGTH" => (@rack_input.length + 1).to_s))
+      response_env.should == [400, {"Content-Type"=>"text/plain"}, ["Bad Request"]]
+    end
+
     it "sanitizes StringIO rack.input" do
       input = "foo=bla&quux=bar"
       @rack_input = StringIO.new input
@@ -227,6 +284,29 @@ describe Rack::UTF8Sanitizer do
       end
     end
 
+    it "sanitizes the rack body if the charset is present and utf-8" do
+      input =  "name=#{CGI.escape("まつもと")}"
+      @rack_input = StringIO.new input
+
+      env = request_env.update('CONTENT_TYPE' => "application/x-www-form-urlencoded; charset=utf-8")
+      sanitize_form_data(env) do |sanitized_input|
+        sanitized_input.encoding.should == Encoding::UTF_8
+        sanitized_input.should.be.valid_encoding
+        sanitized_input.should == input
+      end
+    end
+
+    it "strip UTF-8 BOM from StringIO rack.input" do
+      input = %(\xef\xbb\xbf{"Hello": "World"})
+      @rack_input = StringIO.new input
+
+      sanitize_form_data(request_env.merge("CONTENT_TYPE" => "application/json")) do |sanitized_input|
+        sanitized_input.encoding.should == Encoding::UTF_8
+        sanitized_input.should.be.valid_encoding
+        sanitized_input.should == '{"Hello": "World"}'
+      end
+    end
+
     it "sanitizes StringIO rack.input with form encoded bad encoding" do
       input = "foo=bla&foo=baz&quux%ED=bar%ED"
       @rack_input = StringIO.new input
@@ -291,6 +371,18 @@ describe Rack::UTF8Sanitizer do
       end
     end
 
+    it "does not sanitize the rack body if the charset is present and not utf-8" do
+      input = "name=".encode("Shift_JIS") + CGI.escape("まつもと".encode("Shift_JIS", "UTF-8"))
+      @rack_input = StringIO.new input
+
+      env = request_env.update('CONTENT_TYPE' => "application/x-www-form-urlencoded; charset=Shift_JIS")
+      sanitize_form_data(env) do |sanitized_input|
+        sanitized_input.encoding.should == Encoding::SHIFT_JIS
+        sanitized_input.should.be.valid_encoding
+        sanitized_input.should == input
+      end
+    end
+
     it "adjusts content-length when replacing input" do
       input =  "foo=bla&quux=bar\xED"
       @rack_input = StringIO.new input
@@ -301,5 +393,321 @@ describe Rack::UTF8Sanitizer do
         @response_env["CONTENT_LENGTH"].should == sanitized_input.bytesize.to_s
       end
     end
+
+    it "does not sanitize null bytes by default" do
+      input =  "foo=bla&quux=bar%00"
+      @rack_input = StringIO.new input
+
+      sanitize_form_data do |sanitized_input|
+        sanitized_input.encoding.should == Encoding::UTF_8
+        sanitized_input.should.be.valid_encoding
+        sanitized_input.should == input
+      end
+    end
+
+    it "optionally sanitizes null bytes with the replace strategy" do
+      @app = Rack::UTF8Sanitizer.new(-> env { env }, sanitize_null_bytes: true)
+      input = "foo=bla\xED&quux=bar\x00"
+      @rack_input = StringIO.new input
+
+      sanitize_form_data do |sanitized_input|
+        sanitized_input.encoding.should == Encoding::UTF_8
+        sanitized_input.should.be.valid_encoding
+        sanitized_input.should == "foo=bla%EF%BF%BD&quux=bar"
+      end
+    end
+
+    it "optionally sanitizes encoded null bytes with the replace strategy" do
+      @app = Rack::UTF8Sanitizer.new(-> env { env }, sanitize_null_bytes: true)
+      input = "foo=bla%ED&quux=bar%00"
+      @rack_input = StringIO.new input
+
+      sanitize_form_data do |sanitized_input|
+        sanitized_input.encoding.should == Encoding::UTF_8
+        sanitized_input.should.be.valid_encoding
+        sanitized_input.should == "foo=bla%EF%BF%BD&quux=bar"
+      end
+    end
+
+    it "optionally raises on null bytes with the exception strategy" do
+      @app = Rack::UTF8Sanitizer.new(-> env { env }, sanitize_null_bytes: true, strategy: :exception)
+      input =  "foo=bla&quux=bar\x00"
+      @rack_input = StringIO.new input
+
+      should.raise(Rack::UTF8Sanitizer::NullByteInString) do
+        sanitize_form_data
+      end
+    end
+
+    it "optionally raises on encoded null bytes with the exception strategy" do
+      @app = Rack::UTF8Sanitizer.new(-> env { env }, sanitize_null_bytes: true, strategy: :exception)
+      input =  "foo=bla&quux=bar%00"
+      @rack_input = StringIO.new input
+
+      should.raise(Rack::UTF8Sanitizer::NullByteInString) do
+        sanitize_form_data
+      end
+    end
+
+    it "gives precedence to encoding errors with the exception strategy and null byte sanitisation" do
+      @app = Rack::UTF8Sanitizer.new(-> env { env }, sanitize_null_bytes: true, strategy: :exception)
+      input = "foo=bla\x00&quux=bar\xED"
+      @rack_input = StringIO.new input
+
+      should.raise(EncodingError) do
+        sanitize_form_data
+      end
+    end
+  end
+
+  describe "with custom content-type" do
+    def request_env
+      {
+          "REQUEST_METHOD" => "GET",
+          "CONTENT_TYPE" => "application/json",
+          "HTTP_COOKIE" => @cookie,
+          "rack.input" => StringIO.new,
+      }
+    end
+
+    it "sanitizes bad http cookie" do
+      @cookie = "foo=bla; quux=bar\xED"
+      response_env = @app.(request_env)
+      response_env['HTTP_COOKIE'].should != @cookie
+      response_env['HTTP_COOKIE'].should == 'foo=bla; quux=bar%EF%BF%BD'
+    end
+
+    it "does not change ok http cookie" do
+      @cookie = "foo=bla; quux=bar"
+      response_env = @app.(request_env)
+      response_env['HTTP_COOKIE'].should == @cookie
+
+      @cookie = "foo=b%3bla; quux=b%20a%20r"
+      response_env = @app.(request_env)
+      response_env['HTTP_COOKIE'].should == @cookie
+    end
+  end
+
+  describe "with custom content-type" do
+    def request_env
+      @plain_input = "foo bar лол".dup.force_encoding('UTF-8')
+      {
+          "REQUEST_METHOD" => "POST",
+          "CONTENT_TYPE" => "application/vnd.api+json",
+          "HTTP_USER_AGENT" => @plain_input,
+          "rack.input" => @rack_input,
+      }
+    end
+
+    def sanitize_data(request_env = request_env())
+      @uri_input = "http://bar/foo+%2F%3A+bar+%D0%BB%D0%BE%D0%BB".dup.force_encoding('UTF-8')
+      @response_env = @app.(request_env)
+      sanitized_input = @response_env['rack.input'].read
+
+      yield sanitized_input if block_given?
+    end
+
+    it "does not sanitize custom content-type by default" do
+      input =  "foo=bla&quux=bar\xED"
+      @rack_input = StringIO.new input
+
+      env = request_env
+      sanitize_data(env) do |sanitized_input|
+        sanitized_input.encoding.should == Encoding::ASCII_8BIT
+        sanitized_input.should.be.valid_encoding
+        sanitized_input.should == input
+      end
+    end
+
+    it "sanitizes custom content-type if additional_content_types given" do
+      @app = Rack::UTF8Sanitizer.new(-> env { env }, additional_content_types: ["application/vnd.api+json"])
+      input =  "foo=bla&quux=bar\xED"
+      @rack_input = StringIO.new input
+
+      env = request_env
+      sanitize_data(env) do |sanitized_input|
+        sanitized_input.encoding.should == Encoding::UTF_8
+        sanitized_input.should.be.valid_encoding
+        sanitized_input.should != input
+      end
+    end
+
+    it "sanitizes default content-type if additional_content_types given" do
+      @app = Rack::UTF8Sanitizer.new(-> env { env }, additional_content_types: ["application/vnd.api+json"])
+      input =  "foo=bla&quux=bar\xED"
+      @rack_input = StringIO.new input
+
+      env = request_env.update('CONTENT_TYPE' => 'application/json')
+      sanitize_data(env) do |sanitized_input|
+        sanitized_input.encoding.should == Encoding::UTF_8
+        sanitized_input.should.be.valid_encoding
+        sanitized_input.should != input
+      end
+    end
+
+    it "sanitizes custom content-type if sanitizable_content_types given" do
+      @app = Rack::UTF8Sanitizer.new(-> env { env }, sanitizable_content_types: ["application/vnd.api+json"])
+      input =  "foo=bla&quux=bar\xED"
+      @rack_input = StringIO.new input
+
+      env = request_env
+      sanitize_data(env) do |sanitized_input|
+        sanitized_input.encoding.should == Encoding::UTF_8
+        sanitized_input.should.be.valid_encoding
+        sanitized_input.should != input
+      end
+    end
+
+    it "does not sanitize default content-type if sanitizable_content_types does not include it" do
+      @app = Rack::UTF8Sanitizer.new(-> env { env }, sanitizable_content_types: ["application/vnd.api+json"])
+      input =  "foo=bla&quux=bar\xED"
+      @rack_input = StringIO.new input
+
+      env = request_env.update('CONTENT_TYPE' => 'application/json')
+      sanitize_data(env) do |sanitized_input|
+        sanitized_input.encoding.should == Encoding::ASCII_8BIT
+        sanitized_input.should.be.valid_encoding
+        sanitized_input.should == input
+      end
+    end
+  end
+
+  describe "with only and/or except options" do
+    before do
+      @plain_input = "foo\xe0".dup.force_encoding('UTF-8')
+    end
+
+    def request_env
+      {
+          "REQUEST_METHOD" => "POST",
+          "CONTENT_TYPE" => "application/json",
+          "HTTP_USER_AGENT" => @plain_input,
+          "HTTP_CUSTOM_HEADER" => @plain_input,
+          "rack.input" => @rack_input,
+      }
+    end
+
+    def sanitize_data(request_env = request_env())
+      @response_env = @app.(request_env)
+    end
+
+    it 'skips unless in only' do
+      @app = Rack::UTF8Sanitizer.new(
+        -> env { env },
+        only: ['HTTP_CUSTOM_HEADER']
+      )
+      @rack_input = StringIO.new('{}')
+
+      sanitize_data
+      @response_env['HTTP_CUSTOM_HEADER'].should != @plain_input
+      @response_env['HTTP_USER_AGENT'].should == @plain_input
+    end
+
+    it 'skips if in except' do
+      @app = Rack::UTF8Sanitizer.new(
+        -> env { env },
+        except: ['HTTP_CUSTOM_HEADER']
+      )
+      @rack_input = StringIO.new('{}')
+
+      sanitize_data
+      @response_env['HTTP_CUSTOM_HEADER'].should == @plain_input
+      @response_env['HTTP_USER_AGENT'].should != @plain_input
+    end
+
+    it 'works with regular expressions' do
+      @app = Rack::UTF8Sanitizer.new(
+        -> env { env },
+        only: ['HTTP_CUSTOM_HEADER', /(agent|input)/i]
+      )
+      @rack_input = StringIO.new(@plain_input.force_encoding(Encoding::ASCII_8BIT))
+
+      sanitize_data
+      @response_env['HTTP_CUSTOM_HEADER'].should != @plain_input
+      @response_env['HTTP_USER_AGENT'].should != @plain_input
+      @response_env['rack.input'].read.should != @plain_input
+    end
+  end
+
+  describe "with custom strategy" do
+    def request_env
+      @plain_input = "foo bar лол".dup.force_encoding('UTF-8')
+      {
+          "REQUEST_METHOD" => "POST",
+          "CONTENT_TYPE" => "application/json",
+          "HTTP_USER_AGENT" => @plain_input,
+          "rack.input" => @rack_input,
+      }
+    end
+
+    def sanitize_data(request_env = request_env())
+      @uri_input = "http://bar/foo+%2F%3A+bar+%D0%BB%D0%BE%D0%BB".dup.force_encoding('UTF-8')
+      @response_env = @app.(request_env)
+      sanitized_input = @response_env['rack.input'].read
+
+      yield sanitized_input if block_given?
+    end
+
+    it "calls a default strategy (replace)" do
+      @app = Rack::UTF8Sanitizer.new(-> env { env })
+
+      input = "foo=bla&quux=bar\xED"
+      @rack_input = StringIO.new input
+
+      env = request_env
+      sanitize_data(env) do |sanitized_input|
+        sanitized_input.encoding.should == Encoding::UTF_8
+        sanitized_input.should.be.valid_encoding
+        sanitized_input.should != input
+      end
+    end
+
+    it "calls the exception strategy" do
+      @app = Rack::UTF8Sanitizer.new(-> env { env }, strategy: :exception)
+
+      input = "foo=bla&quux=bar\xED"
+      @rack_input = StringIO.new input
+
+      env = request_env
+      should.raise(EncodingError) { sanitize_data(env) }
+    end
+
+    it "accepts a proc as a strategy" do
+      truncate = -> (input, sanitize_null_bytes:) do
+        sanitize_null_bytes.should == false
+        "replace".dup.force_encoding(Encoding::UTF_8)
+      end
+
+      @app = Rack::UTF8Sanitizer.new(-> env { env }, strategy: truncate)
+
+      input = "foo=bla&quux=bar\xED"
+      @rack_input = StringIO.new input
+
+      env = request_env
+      sanitize_data(env) do |sanitized_input|
+        sanitized_input.encoding.should == Encoding::UTF_8
+        sanitized_input.should.be.valid_encoding
+        sanitized_input.should == 'replace'
+      end
+    end
+
+    it "accepts a proc as a strategy and passes along sanitize_null_bytes" do
+      truncate = -> (input, sanitize_null_bytes:) do
+        sanitize_null_bytes.should == true
+        "replace".dup.force_encoding(Encoding::UTF_8)
+      end
+
+      @app = Rack::UTF8Sanitizer.new(-> env { env }, sanitize_null_bytes: true, strategy: truncate)
+      input = "foo=bla&quux=bar\x00"
+
+      @rack_input = StringIO.new input
+
+      env = request_env
+      sanitize_data(env) do |sanitized_input|
+        sanitized_input.encoding.should == Encoding::UTF_8
+        sanitized_input.should.be.valid_encoding
+        sanitized_input.should == 'replace'
+      end
+    end
   end
 end

metadata

@@ -1,29 +1,34 @@
 --- !ruby/object:Gem::Specification
 name: rack-utf8_sanitizer
 version: !ruby/object:Gem::Version
-  version: 1.3.1
+  version: 1.11.1
 platform: ruby
 authors:
-- Peter Zotov
-autorequire: 
+- Catherine
 bindir: bin
 cert_chain: []
-date: 2015-07-10 00:00:00.000000000 Z
+date: 1980-01-02 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rack
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
         version: '1.0'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '4.0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
         version: '1.0'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '4.0'
 - !ruby/object:Gem::Dependency
   name: bacon
   requirement: !ruby/object:Gem::Requirement
@@ -74,8 +79,10 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
+- ".editorconfig"
+- ".github/dependabot.yml"
+- ".github/workflows/ci.yml"
 - ".gitignore"
-- ".travis.yml"
 - CHANGELOG.md
 - Gemfile
 - LICENSE.txt
@@ -84,10 +91,10 @@ files:
 - lib/rack/utf8_sanitizer.rb
 - rack-utf8_sanitizer.gemspec
 - test/test_utf8_sanitizer.rb
-homepage: http://github.com/whitequark/rack-utf8_sanitizer
-licenses: []
+homepage: https://github.com/whitequark/rack-utf8_sanitizer
+licenses:
+- MIT
 metadata: {}
-post_install_message: 
 rdoc_options: []
 require_paths:
 - lib
@@ -95,16 +102,14 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: '1.9'
+      version: '2.3'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.4.6
-signing_key: 
+rubygems_version: 3.6.7
 specification_version: 4
 summary: Rack::UTF8Sanitizer is a Rack middleware which cleans up invalid UTF8 characters
   in request URI and headers.

data/.travis.yml

@@ -1,13 +0,0 @@
-language: ruby
-
-rvm:
-  - 1.9.2
-  - 1.9.3
-  - 2.0.0
-  # 2.1, not 2.1.0 until fixed https://github.com/travis-ci/travis-ci/issues/2220
-  - 2.1
-  - jruby
-  - rbx-2
-
-script:
-  - rake spec