rack-utf8_sanitizer 1.7.0 → 1.9.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 4156ca74bbd8c43750cdb733ca500a1cb974492ceb823ffa50e9adaa5733d7d9
-  data.tar.gz: 2acc566fb2020de35fa94822f3fcf018988e9166682ccf2f72f7bb9ca7c209d7
+  metadata.gz: 7825c2fec2176e38043c4d7a3c1fcbe1cf112bcc7a17a7ef42b249fab30118c4
+  data.tar.gz: 5090e3c92af9a74377d559be48685d343b29315e5a9ce0f76faf36a8b96437ee
 SHA512:
-  metadata.gz: 5332f698e7d2a06427fe009a1e2f368ca56c4ab04d5b4551f79689dabbce3351733d657e9df14266c6416f5627305e45e511aee701f5a2d783f1b28d7a7d4435
-  data.tar.gz: 7df6257e5945eec1c928ab2ab9e446fe9d853c74b572732b5318e655b1b34d89557d949ee1300004ac67111fd170fddba7ab89b842dbbf65c9ca79717bdf1aa2
+  metadata.gz: e20607b2c412ecfb3d2ba719a7d0aeb381cc4f685e08c6a7801fb2b60a0993c71cc5c219b9ff17cedb497f7d5d0ee907da94ab91476960143f25c704058f1ebc
+  data.tar.gz: 7df7e1d357a6d3b12f089c1d7fea0a55eeb31d2d8f7e3d2b2e2e8729c1ae21c6260a9eb370e8c54e6886344986b1dd436d55b404f7321263d6bbf120115d1788

data/.editorconfig

@@ -0,0 +1,17 @@
+root = true
+
+[*]
+indent_style = space
+indent_size = 2
+end_of_line = lf
+charset = utf-8
+trim_trailing_whitespace = true
+insert_final_newline = true
+
+[*.md]
+indent_style = space
+indent_size = 2
+
+[*.y{a,}ml]
+indent_style = space
+indent_size = 2

data/.github/dependabot.yml

@@ -0,0 +1,6 @@
+version: 2
+updates:
+  - package-ecosystem: "github-actions"
+    directory: "/"
+    schedule:
+      interval: "weekly"

data/.github/workflows/ci.yml

@@ -0,0 +1,23 @@
+name: CI
+
+on: [push, pull_request]
+
+jobs:
+  test:
+
+    runs-on: ubuntu-latest
+
+    strategy:
+      fail-fast: false
+      matrix:
+        ruby: ["2.5", "2.6", "2.7", "3.0", "3.1", "3.2", ruby-head, jruby-9.2, jruby-9.3, jruby-head]
+
+    steps:
+    - uses: actions/checkout@v3
+    - name: Set up Ruby
+      uses: ruby/setup-ruby@v1
+      with:
+        bundler-cache: true # 'bundle install' and cache gems
+        ruby-version: ${{ matrix.ruby }}
+    - name: Run tests
+      run: bundle exec rake

data/.travis.yml

@@ -1,17 +1,14 @@
 language: ruby
 
 rvm:
-  - 1.9.3
-  - 2.0.0
-  - 2.1
-  - 2.2
   - 2.3
   - 2.4
   - 2.5
+  - 2.6
+  - 2.7
+  - 3.0
+  - 3.1
   - jruby
 
 before_install:
   - gem install bundler
-
-script:
-  - rake spec

data/README.md

@@ -113,7 +113,7 @@ config.middleware.insert 0, Rack::UTF8Sanitizer, strategy: :exception
 ```
 
 ```ruby
-replace_string = lambda do |_invalid|
+replace_string = lambda do |_invalid, sanitize_null_bytes: false|
   Rails.logger.warn('Replacing invalid string')
 
   '<Bad Encoding>'.freeze
@@ -122,6 +122,10 @@ end
 config.middleware.insert 0, Rack::UTF8Sanitizer, strategy: replace_string
 ```
 
+### Sanitizing Null Bytes
+
+While null bytes are valid UTF-8, it can be useful to further restrict the valid character set to exclude null bytes. For example, PostgreSQL text columns do not allow storing null bytes. Passing `sanitize_null_bytes: true` in the configuration hash enables sanitizing null bytes, and the two built-in strategies both support this feature. Custom strategies should accept a keyword argument `sanitize_null_bytes` containing this configuration value.
+
 ## Contributing
 
 1. Fork it

data/lib/rack/utf8_sanitizer.rb

@@ -6,6 +6,10 @@ require 'stringio'
 module Rack
   class UTF8Sanitizer
     StringIO = ::StringIO
+    BAD_REQUEST = [400, { "Content-Type" => "text/plain" }, ["Bad Request"]]
+    NULL_BYTE_REGEX = /\x00/.freeze
+
+    class NullByteInString < StandardError; end
 
     # options[:sanitizable_content_types] Array
     # options[:additional_content_types] Array
@@ -16,28 +20,42 @@ module Rack
       @sanitizable_content_types ||= SANITIZABLE_CONTENT_TYPES + (options[:additional_content_types] || [])
       @only = Array(options[:only]).flatten
       @except = Array(options[:except]).flatten
+      @sanitize_null_bytes = options.fetch(:sanitize_null_bytes, false)
     end
 
     def call(env)
-      @app.call(sanitize(env))
+      begin
+        env = sanitize(env)
+      rescue EOFError
+        return BAD_REQUEST
+      end
+      @app.call(env)
     end
 
     DEFAULT_STRATEGIES = {
-      replace: lambda do |input|
+      replace: lambda do |input, sanitize_null_bytes: false|
         input.
           force_encoding(Encoding::ASCII_8BIT).
           encode!(Encoding::UTF_8,
                   invalid: :replace,
                   undef:   :replace)
+        if sanitize_null_bytes
+          input = input.gsub(NULL_BYTE_REGEX, "")
+        end
+        input
       end,
-      exception: lambda do |input|
+      exception: lambda do |input, sanitize_null_bytes: false|
         input.
           force_encoding(Encoding::ASCII_8BIT).
           encode!(Encoding::UTF_8)
+        if sanitize_null_bytes && input =~ NULL_BYTE_REGEX
+          raise NullByteInString
+        end
+        input
       end
     }.freeze
 
-    # http://rack.rubyforge.org/doc/SPEC.html
+    # https://github.com/rack/rack/blob/main/SPEC.rdoc
     URI_FIELDS  = %w(
         SCRIPT_NAME
         REQUEST_PATH REQUEST_URI PATH_INFO
@@ -201,7 +219,8 @@ module Rack
 
     # This regexp matches all 'unreserved' characters from RFC3986 (2.3),
     # plus all multibyte UTF-8 characters.
-    UNRESERVED_OR_UTF8 = /[A-Za-z0-9\-._~\x80-\xFF]/
+    UNRESERVED_OR_UTF8 = /[A-Za-z0-9\-._~\x80-\xFF]/.freeze
+    UNRESERVED_OR_UTF8_OR_NULL = /[A-Za-z0-9\-._~\x00\x80-\xFF]/.freeze
 
     # RFC3986, 2.2 states that the characters from 'reserved' group must be
     # protected during normalization (which is what UTF8Sanitizer does).
@@ -212,7 +231,8 @@ module Rack
       input.gsub(/%([a-f\d]{2})/i) do |encoded|
         decoded = $1.hex.chr
 
-        if decoded =~ UNRESERVED_OR_UTF8
+        decodable_regex = @sanitize_null_bytes ? UNRESERVED_OR_UTF8_OR_NULL : UNRESERVED_OR_UTF8
+        if decoded =~ decodable_regex
           decoded
         else
           encoded
@@ -238,10 +258,10 @@ module Rack
       if input.is_a? String
         input = input.dup.force_encoding(Encoding::UTF_8)
 
-        if input.valid_encoding?
+        if input.valid_encoding? && !(@sanitize_null_bytes && input =~ NULL_BYTE_REGEX)
           input
         else
-          @strategy.call(input)
+          @strategy.call(input, sanitize_null_bytes: @sanitize_null_bytes)
         end
       else
         input

data/rack-utf8_sanitizer.gemspec

@@ -2,7 +2,7 @@
 
 Gem::Specification.new do |gem|
   gem.name          = "rack-utf8_sanitizer"
-  gem.version       = '1.7.0'
+  gem.version       = '1.9.1'
   gem.authors       = ["whitequark"]
   gem.license       = "MIT"
   gem.email         = ["whitequark@whitequark.org"]
@@ -12,13 +12,12 @@ Gem::Specification.new do |gem|
   gem.homepage      = "http://github.com/whitequark/rack-utf8_sanitizer"
 
   gem.files         = `git ls-files`.split($/)
-  gem.executables   = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
   gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
   gem.require_paths = ["lib"]
 
   gem.required_ruby_version = '>= 1.9.3'
 
-  gem.add_dependency             "rack", '>= 1.0', '< 3.0'
+  gem.add_dependency             "rack", '>= 1.0', '< 4.0'
 
   gem.add_development_dependency "bacon"
   gem.add_development_dependency "bacon-colored_output"

data/test/test_utf8_sanitizer.rb

@@ -1,6 +1,7 @@
 # encoding:ascii-8bit
 
 require 'bacon/colored_output'
+require 'cgi'
 require 'rack/utf8_sanitizer'
 
 describe Rack::UTF8Sanitizer do
@@ -118,6 +119,7 @@ describe Rack::UTF8Sanitizer do
   describe "with valid, not percent-encoded UTF-8 URI input" do
     before do
       @uri_input   = "http://bar/foo+bar+лол".force_encoding('UTF-8')
+      @encoded     = "http://bar/foo+bar+#{CGI.escape("лол")}"
     end
 
     it "does not change URI-like entity (REQUEST_PATH)" do
@@ -126,7 +128,7 @@ describe Rack::UTF8Sanitizer do
 
       result.encoding.should == Encoding::US_ASCII
       result.should.be.valid_encoding
-      result.should == URI.encode(@uri_input)
+      result.should == @encoded
     end
   end
 
@@ -205,6 +207,18 @@ describe Rack::UTF8Sanitizer do
       @response_env['rack.input'].close
     end
 
+    class BrokenIO < StringIO
+      def read
+        raise EOFError
+      end
+    end
+
+    it "returns HTTP 400 on EOF" do
+      @rack_input = BrokenIO.new
+      @response_env = @app.(request_env)
+      @response_env.should == [400, {"Content-Type"=>"text/plain"}, ["Bad Request"]]
+    end
+
     it "sanitizes StringIO rack.input" do
       input = "foo=bla&quux=bar"
       @rack_input = StringIO.new input
@@ -323,6 +337,71 @@ describe Rack::UTF8Sanitizer do
         @response_env["CONTENT_LENGTH"].should == sanitized_input.bytesize.to_s
       end
     end
+
+    it "does not sanitize null bytes by default" do
+      input =  "foo=bla&quux=bar%00"
+      @rack_input = StringIO.new input
+
+      sanitize_form_data do |sanitized_input|
+        sanitized_input.encoding.should == Encoding::UTF_8
+        sanitized_input.should.be.valid_encoding
+        sanitized_input.should == input
+      end
+    end
+
+    it "optionally sanitizes null bytes with the replace strategy" do
+      @app = Rack::UTF8Sanitizer.new(-> env { env }, sanitize_null_bytes: true)
+      input = "foo=bla\xED&quux=bar\x00"
+      @rack_input = StringIO.new input
+
+      sanitize_form_data do |sanitized_input|
+        sanitized_input.encoding.should == Encoding::UTF_8
+        sanitized_input.should.be.valid_encoding
+        sanitized_input.should == "foo=bla%EF%BF%BD&quux=bar"
+      end
+    end
+
+    it "optionally sanitizes encoded null bytes with the replace strategy" do
+      @app = Rack::UTF8Sanitizer.new(-> env { env }, sanitize_null_bytes: true)
+      input = "foo=bla%ED&quux=bar%00"
+      @rack_input = StringIO.new input
+
+      sanitize_form_data do |sanitized_input|
+        sanitized_input.encoding.should == Encoding::UTF_8
+        sanitized_input.should.be.valid_encoding
+        sanitized_input.should == "foo=bla%EF%BF%BD&quux=bar"
+      end
+    end
+
+    it "optionally raises on null bytes with the exception strategy" do
+      @app = Rack::UTF8Sanitizer.new(-> env { env }, sanitize_null_bytes: true, strategy: :exception)
+      input =  "foo=bla&quux=bar\x00"
+      @rack_input = StringIO.new input
+
+      should.raise(Rack::UTF8Sanitizer::NullByteInString) do
+        sanitize_form_data
+      end
+    end
+
+    it "optionally raises on encoded null bytes with the exception strategy" do
+      @app = Rack::UTF8Sanitizer.new(-> env { env }, sanitize_null_bytes: true, strategy: :exception)
+      input =  "foo=bla&quux=bar%00"
+      @rack_input = StringIO.new input
+
+      should.raise(Rack::UTF8Sanitizer::NullByteInString) do
+        sanitize_form_data
+      end
+    end
+
+    it "gives precedence to encoding errors with the exception strategy and null byte sanitisation" do
+      @app = Rack::UTF8Sanitizer.new(-> env { env }, sanitize_null_bytes: true, strategy: :exception)
+      input = "foo=bla\x00&quux=bar\xED"
+      @rack_input = StringIO.new input
+
+      should.raise(EncodingError) do
+        sanitize_form_data
+      end
+    end
   end
 
   describe "with custom content-type" do
@@ -538,7 +617,10 @@ describe Rack::UTF8Sanitizer do
     end
 
     it "accepts a proc as a strategy" do
-      truncate = -> input { 'replace'.force_encoding(Encoding::UTF_8) }
+      truncate = -> (input, sanitize_null_bytes:) do
+        sanitize_null_bytes.should == false
+        'replace'.force_encoding(Encoding::UTF_8)
+      end
 
       @app = Rack::UTF8Sanitizer.new(-> env { env }, strategy: truncate)
 
@@ -549,7 +631,26 @@ describe Rack::UTF8Sanitizer do
       sanitize_data(env) do |sanitized_input|
         sanitized_input.encoding.should == Encoding::UTF_8
         sanitized_input.should.be.valid_encoding
-        sanitized_input.should == 'replace' 
+        sanitized_input.should == 'replace'
+      end
+    end
+
+    it "accepts a proc as a strategy and passes along sanitize_null_bytes" do
+      truncate = -> (input, sanitize_null_bytes:) do
+        sanitize_null_bytes.should == true
+        'replace'.force_encoding(Encoding::UTF_8)
+      end
+
+      @app = Rack::UTF8Sanitizer.new(-> env { env }, sanitize_null_bytes: true, strategy: truncate)
+      input = "foo=bla&quux=bar\x00"
+
+      @rack_input = StringIO.new input
+
+      env = request_env
+      sanitize_data(env) do |sanitized_input|
+        sanitized_input.encoding.should == Encoding::UTF_8
+        sanitized_input.should.be.valid_encoding
+        sanitized_input.should == 'replace'
       end
     end
   end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rack-utf8_sanitizer
 version: !ruby/object:Gem::Version
-  version: 1.7.0
+  version: 1.9.1
 platform: ruby
 authors:
 - whitequark
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2020-03-05 00:00:00.000000000 Z
+date: 2023-08-30 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rack
@@ -19,7 +19,7 @@ dependencies:
         version: '1.0'
     - - "<"
       - !ruby/object:Gem::Version
-        version: '3.0'
+        version: '4.0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -29,7 +29,7 @@ dependencies:
         version: '1.0'
     - - "<"
       - !ruby/object:Gem::Version
-        version: '3.0'
+        version: '4.0'
 - !ruby/object:Gem::Dependency
   name: bacon
   requirement: !ruby/object:Gem::Requirement
@@ -80,6 +80,9 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
+- ".editorconfig"
+- ".github/dependabot.yml"
+- ".github/workflows/ci.yml"
 - ".gitignore"
 - ".travis.yml"
 - CHANGELOG.md
@@ -94,7 +97,7 @@ homepage: http://github.com/whitequark/rack-utf8_sanitizer
 licenses:
 - MIT
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -109,9 +112,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.7.6.2
-signing_key: 
+rubygems_version: 3.3.15
+signing_key:
 specification_version: 4
 summary: Rack::UTF8Sanitizer is a Rack middleware which cleans up invalid UTF8 characters
   in request URI and headers.