rack-utf8_sanitizer 1.7.0 → 1.9.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA256:
-  metadata.gz: 4156ca74bbd8c43750cdb733ca500a1cb974492ceb823ffa50e9adaa5733d7d9
-  data.tar.gz: 2acc566fb2020de35fa94822f3fcf018988e9166682ccf2f72f7bb9ca7c209d7
+  metadata.gz: 9893353fe731d6c942263ff57af911b10d1c9f88ba2c2c2b56bb7a76586935e5
+  data.tar.gz: eeeaca39f55d680abe2707fa1aaadfe34eed06de2dd848506d184ae4ff326a8d
 SHA512:
-  metadata.gz: 5332f698e7d2a06427fe009a1e2f368ca56c4ab04d5b4551f79689dabbce3351733d657e9df14266c6416f5627305e45e511aee701f5a2d783f1b28d7a7d4435
-  data.tar.gz: 7df6257e5945eec1c928ab2ab9e446fe9d853c74b572732b5318e655b1b34d89557d949ee1300004ac67111fd170fddba7ab89b842dbbf65c9ca79717bdf1aa2
+  metadata.gz: 28cfbadcc1fbbf0678db3b51259618a9c9897c0cb78bcd8b8f5d87c91eddb44873a7a77fb21b9db47bcc704cb7df17d942c50436475858d0d9fcc17c2a8b59ce
+  data.tar.gz: 7e781931df60e405d533f5eb832416510a431a1c6fa74baeabffb9499533d9eba2f97ffa24f89ed79a193de5f6715c0461fff4fb0403c863a3643e4aaabbcf0a

data/.editorconfig

@@ -0,0 +1,17 @@
+root = true
+
+[*]
+indent_style = space
+indent_size = 2
+end_of_line = lf
+charset = utf-8
+trim_trailing_whitespace = true
+insert_final_newline = true
+
+[*.md]
+indent_style = space
+indent_size = 2
+
+[*.y{a,}ml]
+indent_style = space
+indent_size = 2

data/.github/dependabot.yml

@@ -0,0 +1,6 @@
+version: 2
+updates:
+  - package-ecosystem: "github-actions"
+    directory: "/"
+    schedule:
+      interval: "weekly"

data/.github/workflows/ci.yml

@@ -0,0 +1,23 @@
+name: CI
+
+on: [push, pull_request]
+
+jobs:
+  test:
+
+    runs-on: ubuntu-latest
+
+    strategy:
+      fail-fast: false
+      matrix:
+        ruby: ["2.5", "2.6", "2.7", "3.0", "3.1", "3.2", ruby-head, jruby-9.2, jruby-9.3, jruby-head]
+
+    steps:
+    - uses: actions/checkout@v3
+    - name: Set up Ruby
+      uses: ruby/setup-ruby@v1
+      with:
+        bundler-cache: true # 'bundle install' and cache gems
+        ruby-version: ${{ matrix.ruby }}
+    - name: Run tests
+      run: bundle exec rake

data/.travis.yml

@@ -1,17 +1,14 @@
 language: ruby
 
 rvm:
-  - 1.9.3
-  - 2.0.0
-  - 2.1
-  - 2.2
   - 2.3
   - 2.4
   - 2.5
+  - 2.6
+  - 2.7
+  - 3.0
+  - 3.1
   - jruby
 
 before_install:
   - gem install bundler
-
-script:
-  - rake spec

data/lib/rack/utf8_sanitizer.rb

@@ -6,6 +6,10 @@ require 'stringio'
 module Rack
   class UTF8Sanitizer
     StringIO = ::StringIO
+    BAD_REQUEST = [400, { "Content-Type" => "text/plain" }, ["Bad Request"]]
+    NULL_BYTE_REGEX = /\x00/.freeze
+
+    class NullByteInString < StandardError; end
 
     # options[:sanitizable_content_types] Array
     # options[:additional_content_types] Array
@@ -16,28 +20,40 @@ module Rack
       @sanitizable_content_types ||= SANITIZABLE_CONTENT_TYPES + (options[:additional_content_types] || [])
       @only = Array(options[:only]).flatten
       @except = Array(options[:except]).flatten
+      @sanitize_null_bytes = options.fetch(:sanitize_null_bytes, false)
     end
 
     def call(env)
-      @app.call(sanitize(env))
+      begin
+        env = sanitize(env)
+      rescue EOFError
+        return BAD_REQUEST
+      end
+      @app.call(env)
     end
 
     DEFAULT_STRATEGIES = {
-      replace: lambda do |input|
+      replace: lambda do |input, sanitize_null_bytes: false|
+        if sanitize_null_bytes
+          input = input.gsub(NULL_BYTE_REGEX, "")
+        end
         input.
           force_encoding(Encoding::ASCII_8BIT).
           encode!(Encoding::UTF_8,
                   invalid: :replace,
                   undef:   :replace)
       end,
-      exception: lambda do |input|
+      exception: lambda do |input, sanitize_null_bytes: false|
+        if sanitize_null_bytes && input =~ NULL_BYTE_REGEX
+          raise NullByteInString
+        end
         input.
           force_encoding(Encoding::ASCII_8BIT).
           encode!(Encoding::UTF_8)
       end
     }.freeze
 
-    # http://rack.rubyforge.org/doc/SPEC.html
+    # https://github.com/rack/rack/blob/main/SPEC.rdoc
     URI_FIELDS  = %w(
         SCRIPT_NAME
         REQUEST_PATH REQUEST_URI PATH_INFO
@@ -201,7 +217,8 @@ module Rack
 
     # This regexp matches all 'unreserved' characters from RFC3986 (2.3),
     # plus all multibyte UTF-8 characters.
-    UNRESERVED_OR_UTF8 = /[A-Za-z0-9\-._~\x80-\xFF]/
+    UNRESERVED_OR_UTF8 = /[A-Za-z0-9\-._~\x80-\xFF]/.freeze
+    UNRESERVED_OR_UTF8_OR_NULL = /[A-Za-z0-9\-._~\x00\x80-\xFF]/.freeze
 
     # RFC3986, 2.2 states that the characters from 'reserved' group must be
     # protected during normalization (which is what UTF8Sanitizer does).
@@ -212,7 +229,8 @@ module Rack
       input.gsub(/%([a-f\d]{2})/i) do |encoded|
         decoded = $1.hex.chr
 
-        if decoded =~ UNRESERVED_OR_UTF8
+        decodable_regex = @sanitize_null_bytes ? UNRESERVED_OR_UTF8_OR_NULL : UNRESERVED_OR_UTF8
+        if decoded =~ decodable_regex
           decoded
         else
           encoded
@@ -238,10 +256,10 @@ module Rack
       if input.is_a? String
         input = input.dup.force_encoding(Encoding::UTF_8)
 
-        if input.valid_encoding?
+        if input.valid_encoding? && !(@sanitize_null_bytes && input =~ NULL_BYTE_REGEX)
           input
         else
-          @strategy.call(input)
+          @strategy.call(input, sanitize_null_bytes: @sanitize_null_bytes)
         end
       else
         input

data/rack-utf8_sanitizer.gemspec

@@ -2,7 +2,7 @@
 
 Gem::Specification.new do |gem|
   gem.name          = "rack-utf8_sanitizer"
-  gem.version       = '1.7.0'
+  gem.version       = '1.9.0'
   gem.authors       = ["whitequark"]
   gem.license       = "MIT"
   gem.email         = ["whitequark@whitequark.org"]
@@ -12,13 +12,12 @@ Gem::Specification.new do |gem|
   gem.homepage      = "http://github.com/whitequark/rack-utf8_sanitizer"
 
   gem.files         = `git ls-files`.split($/)
-  gem.executables   = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
   gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
   gem.require_paths = ["lib"]
 
   gem.required_ruby_version = '>= 1.9.3'
 
-  gem.add_dependency             "rack", '>= 1.0', '< 3.0'
+  gem.add_dependency             "rack", '>= 1.0', '< 4.0'
 
   gem.add_development_dependency "bacon"
   gem.add_development_dependency "bacon-colored_output"

data/test/test_utf8_sanitizer.rb

@@ -1,6 +1,7 @@
 # encoding:ascii-8bit
 
 require 'bacon/colored_output'
+require 'cgi'
 require 'rack/utf8_sanitizer'
 
 describe Rack::UTF8Sanitizer do
@@ -118,6 +119,7 @@ describe Rack::UTF8Sanitizer do
   describe "with valid, not percent-encoded UTF-8 URI input" do
     before do
       @uri_input   = "http://bar/foo+bar+лол".force_encoding('UTF-8')
+      @encoded     = "http://bar/foo+bar+#{CGI.escape("лол")}"
     end
 
     it "does not change URI-like entity (REQUEST_PATH)" do
@@ -126,7 +128,7 @@ describe Rack::UTF8Sanitizer do
 
       result.encoding.should == Encoding::US_ASCII
       result.should.be.valid_encoding
-      result.should == URI.encode(@uri_input)
+      result.should == @encoded
     end
   end
 
@@ -205,6 +207,18 @@ describe Rack::UTF8Sanitizer do
       @response_env['rack.input'].close
     end
 
+    class BrokenIO < StringIO
+      def read
+        raise EOFError
+      end
+    end
+
+    it "returns HTTP 400 on EOF" do
+      @rack_input = BrokenIO.new
+      @response_env = @app.(request_env)
+      @response_env.should == [400, {"Content-Type"=>"text/plain"}, ["Bad Request"]]
+    end
+
     it "sanitizes StringIO rack.input" do
       input = "foo=bla&quux=bar"
       @rack_input = StringIO.new input
@@ -323,6 +337,61 @@ describe Rack::UTF8Sanitizer do
         @response_env["CONTENT_LENGTH"].should == sanitized_input.bytesize.to_s
       end
     end
+
+    it "does not sanitize null bytes by default" do
+      input =  "foo=bla&quux=bar%00"
+      @rack_input = StringIO.new input
+
+      sanitize_form_data do |sanitized_input|
+        sanitized_input.encoding.should == Encoding::UTF_8
+        sanitized_input.should.be.valid_encoding
+        sanitized_input.should == input
+      end
+    end
+
+    it "optionally sanitizes null bytes with the replace strategy" do
+      @app = Rack::UTF8Sanitizer.new(-> env { env }, sanitize_null_bytes: true)
+      input =  "foo=bla&quux=bar\x00"
+      @rack_input = StringIO.new input
+
+      sanitize_form_data do |sanitized_input|
+        sanitized_input.encoding.should == Encoding::UTF_8
+        sanitized_input.should.be.valid_encoding
+        sanitized_input.should == "foo=bla&quux=bar"
+      end
+    end
+
+    it "optionally sanitizes encoded null bytes with the replace strategy" do
+      @app = Rack::UTF8Sanitizer.new(-> env { env }, sanitize_null_bytes: true)
+      input =  "foo=bla&quux=bar%00"
+      @rack_input = StringIO.new input
+
+      sanitize_form_data do |sanitized_input|
+        sanitized_input.encoding.should == Encoding::UTF_8
+        sanitized_input.should.be.valid_encoding
+        sanitized_input.should == "foo=bla&quux=bar"
+      end
+    end
+
+    it "optionally raises on null bytes with the exception strategy" do
+      @app = Rack::UTF8Sanitizer.new(-> env { env }, sanitize_null_bytes: true, strategy: :exception)
+      input =  "foo=bla&quux=bar\x00"
+      @rack_input = StringIO.new input
+
+      should.raise(Rack::UTF8Sanitizer::NullByteInString) do
+        sanitize_form_data
+      end
+    end
+
+    it "optionally raises on encoded null bytes with the exception strategy" do
+      @app = Rack::UTF8Sanitizer.new(-> env { env }, sanitize_null_bytes: true, strategy: :exception)
+      input =  "foo=bla&quux=bar%00"
+      @rack_input = StringIO.new input
+
+      should.raise(Rack::UTF8Sanitizer::NullByteInString) do
+        sanitize_form_data
+      end
+    end
   end
 
   describe "with custom content-type" do
@@ -538,7 +607,10 @@ describe Rack::UTF8Sanitizer do
     end
 
     it "accepts a proc as a strategy" do
-      truncate = -> input { 'replace'.force_encoding(Encoding::UTF_8) }
+      truncate = -> (input, sanitize_null_bytes:) do
+        sanitize_null_bytes.should == false
+        'replace'.force_encoding(Encoding::UTF_8)
+      end
 
       @app = Rack::UTF8Sanitizer.new(-> env { env }, strategy: truncate)
 
@@ -549,7 +621,26 @@ describe Rack::UTF8Sanitizer do
       sanitize_data(env) do |sanitized_input|
         sanitized_input.encoding.should == Encoding::UTF_8
         sanitized_input.should.be.valid_encoding
-        sanitized_input.should == 'replace' 
+        sanitized_input.should == 'replace'
+      end
+    end
+
+    it "accepts a proc as a strategy and passes along sanitize_null_bytes" do
+      truncate = -> (input, sanitize_null_bytes:) do
+        sanitize_null_bytes.should == true
+        'replace'.force_encoding(Encoding::UTF_8)
+      end
+
+      @app = Rack::UTF8Sanitizer.new(-> env { env }, sanitize_null_bytes: true, strategy: truncate)
+      input = "foo=bla&quux=bar\x00"
+
+      @rack_input = StringIO.new input
+
+      env = request_env
+      sanitize_data(env) do |sanitized_input|
+        sanitized_input.encoding.should == Encoding::UTF_8
+        sanitized_input.should.be.valid_encoding
+        sanitized_input.should == 'replace'
       end
     end
   end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rack-utf8_sanitizer
 version: !ruby/object:Gem::Version
-  version: 1.7.0
+  version: 1.9.0
 platform: ruby
 authors:
 - whitequark
-autorequire: 
+autorequire:
 bindir: bin
 cert_chain: []
-date: 2020-03-05 00:00:00.000000000 Z
+date: 2023-07-05 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rack
@@ -19,7 +19,7 @@ dependencies:
         version: '1.0'
     - - "<"
       - !ruby/object:Gem::Version
-        version: '3.0'
+        version: '4.0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
@@ -29,7 +29,7 @@ dependencies:
         version: '1.0'
     - - "<"
       - !ruby/object:Gem::Version
-        version: '3.0'
+        version: '4.0'
 - !ruby/object:Gem::Dependency
   name: bacon
   requirement: !ruby/object:Gem::Requirement
@@ -80,6 +80,9 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
+- ".editorconfig"
+- ".github/dependabot.yml"
+- ".github/workflows/ci.yml"
 - ".gitignore"
 - ".travis.yml"
 - CHANGELOG.md
@@ -94,7 +97,7 @@ homepage: http://github.com/whitequark/rack-utf8_sanitizer
 licenses:
 - MIT
 metadata: {}
-post_install_message: 
+post_install_message:
 rdoc_options: []
 require_paths:
 - lib
@@ -109,9 +112,8 @@ required_rubygems_version: !ruby/object:Gem::Requirement
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
-rubyforge_project: 
-rubygems_version: 2.7.6.2
-signing_key: 
+rubygems_version: 3.2.5
+signing_key:
 specification_version: 4
 summary: Rack::UTF8Sanitizer is a Rack middleware which cleans up invalid UTF8 characters
   in request URI and headers.