rack-utf8_sanitizer 1.3.1 → 1.3.2

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: a21608edab6e71fde026d8c9b5fdf6afaeb831ab
-  data.tar.gz: d3648a488dd74a3ea6357d39cc87b1d486af2c1c
+  metadata.gz: ef9e02d0857dfe41f06d2422727019029dc9d50a
+  data.tar.gz: 6946827dd89a0a13cf0d8b4f0325c321d3c4db71
 SHA512:
-  metadata.gz: 8c8c7e485b0ca9584951b2689b090feaf4441eb96c1fb5ee3fd8f7c79bddcb4bf4d005b69e56a1fd0675b1e96de07dca21906ccca4b255e3893007278e819afe
-  data.tar.gz: b8dcdd43af94a1277b978184ba8e5b26c782f269f646805839d102cf216f7e8d1e375a4104ace50357c23c3761afd00acaef441ac13bcc05ae1555e719e71079
+  metadata.gz: 2b3f4ad2011e98e573803d751419474ac4bb1cd402dc8dd1dec8063d34a043fbd1ff7866ccc0aa84d9e8456c685c7b2b8f4addae4108d41825d816e521aa129f
+  data.tar.gz: 2aed120d37855fbacb7400d82fdd5da9273c8ea5eaefccc16f297006e3e8cd70f6cc88c7b4b2db296ef61f5424b14a111a36ba0da60047206adc47bf83399d7c

data/.travis.yml

@@ -1,11 +1,11 @@
 language: ruby
 
 rvm:
-  - 1.9.2
   - 1.9.3
   - 2.0.0
   # 2.1, not 2.1.0 until fixed https://github.com/travis-ci/travis-ci/issues/2220
   - 2.1
+  - 2.2
   - jruby
   - rbx-2
 

data/README.md

@@ -43,6 +43,18 @@ For fields with "raw data", the algorithm is applied once and the (UTF-8 encoded
 
 For fields with "percent-encoded data", the algorithm is applied twice to catch both invalid characters appearing as-is and invalid characters appearing in the percent encoding. The percent encoded, ASCII-8BIT encoded result is left in the environment.
 
+### Sanitizable content types
+
+The default content types to be sanitized are 'text/plain', 'application/x-www-form-urlencoded', 'application/json', 'text/javascript'. You may wish to modify this, for example if your app accepts specific or custom media types in the CONTENT_TYPE header. If you want to change the sanitizable content types, you can pass options when using Rack::UTF8Sanitizer. 
+
+To add sanitizable content types to the list of defaults, pass the `additional_content_types` options when using Rack::UTF8Sanitizer, e.g.
+
+    config.middleware.insert 0, Rack::UTF8Sanitizer, additional_content_types: ['application/vnd.api+json']
+
+To explicitly set sanitizable content types and override the defaults, use the `sanitizable_content_types` option:
+
+    config.middleware.insert 0, Rack::UTF8Sanitizer, sanitizable_content_types: ['application/vnd.api+json']
+
 ## Contributing
 
 1. Fork it
@@ -50,3 +62,5 @@ For fields with "percent-encoded data", the algorithm is applied twice to catch
 3. Commit your changes (`git commit -am 'Add some feature'`)
 4. Push to the branch (`git push origin my-new-feature`)
 5. Create new Pull Request
+
+To run the tests, run `rake spec` in the project directory.

data/lib/rack/utf8_sanitizer.rb

@@ -7,8 +7,12 @@ module Rack
   class UTF8Sanitizer
     StringIO = ::StringIO
 
-    def initialize(app)
+    # options[:sanitizable_content_types] Array
+    # options[:additional_content_types] Array
+    def initialize(app, options={})
       @app = app
+      @sanitizable_content_types = options[:sanitizable_content_types]
+      @sanitizable_content_types ||= SANITIZABLE_CONTENT_TYPES + (options[:additional_content_types] || [])
     end
 
     def call(env)
@@ -23,18 +27,18 @@ module Rack
         HTTP_REFERER
         ORIGINAL_FULLPATH
         ORIGINAL_SCRIPT_NAME
-    )
+    ).map(&:freeze).freeze
 
     SANITIZABLE_CONTENT_TYPES = %w(
       text/plain
       application/x-www-form-urlencoded
       application/json
       text/javascript
-    )
+    ).map(&:freeze).freeze
 
     URI_ENCODED_CONTENT_TYPES = %w(
       application/x-www-form-urlencoded
-    )
+    ).map(&:freeze).freeze
 
     def sanitize(env)
       sanitize_rack_input(env)
@@ -60,7 +64,7 @@ module Rack
       content_type   = env['CONTENT_TYPE']
       content_type &&= content_type.split(/\s*[;,]\s*/, 2).first
       content_type &&= content_type.downcase
-      return unless SANITIZABLE_CONTENT_TYPES.any? {|type| content_type == type }
+      return unless @sanitizable_content_types.any? {|type| content_type == type }
       uri_encoded = URI_ENCODED_CONTENT_TYPES.any? {|type| content_type == type}
 
       if env["rack.input"]
@@ -108,7 +112,7 @@ module Rack
 
     def sanitize_io(io, uri_encoded = false)
       input = io.read
-      sanitized_input = sanitize_string(input)
+      sanitized_input = sanitize_string(strip_byte_order_mark(input))
       if uri_encoded
         sanitized_input = sanitize_uri_encoded_string(sanitized_input).
           force_encoding(Encoding::UTF_8)
@@ -201,5 +205,13 @@ module Rack
         to
       end
     end
+
+    UTF8_BOM = "\xef\xbb\xbf".force_encoding(Encoding::BINARY).freeze
+    UTF8_BOM_SIZE = UTF8_BOM.bytesize
+
+    def strip_byte_order_mark(input)
+      return input unless input.start_with?(UTF8_BOM)
+      input.byteslice(UTF8_BOM_SIZE..-1)
+    end
   end
 end

data/rack-utf8_sanitizer.gemspec

@@ -2,8 +2,8 @@
 
 Gem::Specification.new do |gem|
   gem.name          = "rack-utf8_sanitizer"
-  gem.version       = '1.3.1'
-  gem.authors       = ["Peter Zotov"]
+  gem.version       = '1.3.2'
+  gem.authors       = ["whitequark"]
   gem.email         = ["whitequark@whitequark.org"]
   gem.description   = %{Rack::UTF8Sanitizer is a Rack middleware which cleans up } <<
                       %{invalid UTF8 characters in request URI and headers.}
@@ -15,9 +15,9 @@ Gem::Specification.new do |gem|
   gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
   gem.require_paths = ["lib"]
 
-  gem.required_ruby_version = '>= 1.9'
+  gem.required_ruby_version = '>= 1.9.3'
 
-  gem.add_dependency             "rack", '~> 1.0'
+  gem.add_dependency             "rack", '>= 1.0', '< 3.0'
 
   gem.add_development_dependency "bacon"
   gem.add_development_dependency "bacon-colored_output"

data/test/test_utf8_sanitizer.rb

@@ -179,7 +179,7 @@ describe Rack::UTF8Sanitizer do
       }
     end
 
-    def sanitize_form_data(request_env = request_env)
+    def sanitize_form_data(request_env = request_env())
       @uri_input = "http://bar/foo+%2F%3A+bar+%D0%BB%D0%BE%D0%BB".force_encoding('UTF-8')
       @response_env = @app.(request_env)
       sanitized_input = @response_env['rack.input'].read
@@ -227,6 +227,17 @@ describe Rack::UTF8Sanitizer do
       end
     end
 
+    it "strip UTF-8 BOM from StringIO rack.input" do
+      input = %(\xef\xbb\xbf{"Hello": "World"})
+      @rack_input = StringIO.new input
+
+      sanitize_form_data(request_env.merge("CONTENT_TYPE" => "application/json")) do |sanitized_input|
+        sanitized_input.encoding.should == Encoding::UTF_8
+        sanitized_input.should.be.valid_encoding
+        sanitized_input.should == '{"Hello": "World"}'
+      end
+    end
+
     it "sanitizes StringIO rack.input with form encoded bad encoding" do
       input = "foo=bla&foo=baz&quux%ED=bar%ED"
       @rack_input = StringIO.new input
@@ -302,4 +313,88 @@ describe Rack::UTF8Sanitizer do
       end
     end
   end
+
+  describe "with custom content-type" do
+    def request_env
+      @plain_input = "foo bar лол".force_encoding('UTF-8')
+      {
+          "REQUEST_METHOD" => "POST",
+          "CONTENT_TYPE" => "application/vnd.api+json",
+          "HTTP_USER_AGENT" => @plain_input,
+          "rack.input" => @rack_input,
+      }
+    end
+
+    def sanitize_data(request_env = request_env())
+      @uri_input = "http://bar/foo+%2F%3A+bar+%D0%BB%D0%BE%D0%BB".force_encoding('UTF-8')
+      @response_env = @app.(request_env)
+      sanitized_input = @response_env['rack.input'].read
+
+      yield sanitized_input if block_given?
+    end
+
+    it "does not sanitize custom content-type by default" do
+      input =  "foo=bla&quux=bar\xED"
+      @rack_input = StringIO.new input
+
+      env = request_env
+      sanitize_data(env) do |sanitized_input|
+        sanitized_input.encoding.should == Encoding::ASCII_8BIT
+        sanitized_input.should.be.valid_encoding
+        sanitized_input.should == input
+      end
+    end
+
+    it "sanitizes custom content-type if additional_content_types given" do
+      @app = Rack::UTF8Sanitizer.new(-> env { env }, additional_content_types: ["application/vnd.api+json"])
+      input =  "foo=bla&quux=bar\xED"
+      @rack_input = StringIO.new input
+
+      env = request_env
+      sanitize_data(env) do |sanitized_input|
+        sanitized_input.encoding.should == Encoding::UTF_8
+        sanitized_input.should.be.valid_encoding
+        sanitized_input.should != input
+      end
+    end
+
+    it "sanitizes default content-type if additional_content_types given" do
+      @app = Rack::UTF8Sanitizer.new(-> env { env }, additional_content_types: ["application/vnd.api+json"])
+      input =  "foo=bla&quux=bar\xED"
+      @rack_input = StringIO.new input
+
+      env = request_env.update('CONTENT_TYPE' => 'application/json')
+      sanitize_data(env) do |sanitized_input|
+        sanitized_input.encoding.should == Encoding::UTF_8
+        sanitized_input.should.be.valid_encoding
+        sanitized_input.should != input
+      end
+    end
+
+    it "sanitizes custom content-type if sanitizable_content_types given" do
+      @app = Rack::UTF8Sanitizer.new(-> env { env }, sanitizable_content_types: ["application/vnd.api+json"])
+      input =  "foo=bla&quux=bar\xED"
+      @rack_input = StringIO.new input
+
+      env = request_env
+      sanitize_data(env) do |sanitized_input|
+        sanitized_input.encoding.should == Encoding::UTF_8
+        sanitized_input.should.be.valid_encoding
+        sanitized_input.should != input
+      end
+    end
+
+    it "does not sanitize default content-type if sanitizable_content_types does not include it" do
+      @app = Rack::UTF8Sanitizer.new(-> env { env }, sanitizable_content_types: ["application/vnd.api+json"])
+      input =  "foo=bla&quux=bar\xED"
+      @rack_input = StringIO.new input
+
+      env = request_env.update('CONTENT_TYPE' => 'application/json')
+      sanitize_data(env) do |sanitized_input|
+        sanitized_input.encoding.should == Encoding::ASCII_8BIT
+        sanitized_input.should.be.valid_encoding
+        sanitized_input.should == input
+      end
+    end
+  end
 end

metadata

@@ -1,29 +1,35 @@
 --- !ruby/object:Gem::Specification
 name: rack-utf8_sanitizer
 version: !ruby/object:Gem::Version
-  version: 1.3.1
+  version: 1.3.2
 platform: ruby
 authors:
-- Peter Zotov
+- whitequark
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-07-10 00:00:00.000000000 Z
+date: 2015-12-23 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rack
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
         version: '1.0'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '3.0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - "~>"
+    - - ">="
       - !ruby/object:Gem::Version
         version: '1.0'
+    - - "<"
+      - !ruby/object:Gem::Version
+        version: '3.0'
 - !ruby/object:Gem::Dependency
   name: bacon
   requirement: !ruby/object:Gem::Requirement
@@ -95,7 +101,7 @@ required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
     - !ruby/object:Gem::Version
-      version: '1.9'
+      version: 1.9.3
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
   - - ">="
@@ -103,7 +109,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.4.6
+rubygems_version: 2.4.5.1
 signing_key: 
 specification_version: 4
 summary: Rack::UTF8Sanitizer is a Rack middleware which cleans up invalid UTF8 characters