rack-utf8_sanitizer 1.3.2 → 1.4.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: ef9e02d0857dfe41f06d2422727019029dc9d50a
-  data.tar.gz: 6946827dd89a0a13cf0d8b4f0325c321d3c4db71
+  metadata.gz: 69f0fb52847e155e4c7afd57565531b8bf506149
+  data.tar.gz: 0502a652822279ce9fd6b22e2db54126b9247b49
 SHA512:
-  metadata.gz: 2b3f4ad2011e98e573803d751419474ac4bb1cd402dc8dd1dec8063d34a043fbd1ff7866ccc0aa84d9e8456c685c7b2b8f4addae4108d41825d816e521aa129f
-  data.tar.gz: 2aed120d37855fbacb7400d82fdd5da9273c8ea5eaefccc16f297006e3e8cd70f6cc88c7b4b2db296ef61f5424b14a111a36ba0da60047206adc47bf83399d7c
+  metadata.gz: '01215648794d9d0b47ef805f40116644b0afab096a00d88ebe0d84df4aed839d025641ba1a40c032b62d525ff9467ec89498e396b6c3e69483535cd71b43e997'
+  data.tar.gz: 95d5d7cfe3a6ec0564a566fe95aaed96f48605265a101483e98dd82a46eaf5a7aa1ff3a29106105fba5aa03f1ef6f108eb29c80ccd11a053ad6b91642e8a3003

data/.travis.yml

@@ -3,11 +3,13 @@ language: ruby
 rvm:
   - 1.9.3
   - 2.0.0
-  # 2.1, not 2.1.0 until fixed https://github.com/travis-ci/travis-ci/issues/2220
   - 2.1
   - 2.2
   - jruby
   - rbx-2
 
+before_install:
+  - gem install bundler
+
 script:
   - rake spec

data/README.md

@@ -55,6 +55,26 @@ To explicitly set sanitizable content types and override the defaults, use the `
 
     config.middleware.insert 0, Rack::UTF8Sanitizer, sanitizable_content_types: ['application/vnd.api+json']
 
+### Strategies
+
+There are two built in strategies for handling invalid characters. The default strategy is `:replace`, which will cause any invalid characters to be replaces with the unicode replacement character (�). The second built in strategy is `:exception` which will cause an `EncodingError` exception to be raised if invalid characters are found (the exception can then be handled by another Rack middleware).
+
+An object that responds to `#call` and accepts the offending string with invalid characters as an argumant can also be passed as a `:strategy`. This is how you can define custom strategies.
+
+```ruby
+config.middleware.insert 0, Rack::UTF8Sanitizer, strategy: :exception
+```
+
+```ruby
+replace_string = lambda do |_invalid|
+  Rails.logger.warn('Replacing invalid string')
+
+  '<Bad Encoding>'.freeze
+end
+
+config.middleware.insert 0, Rack::UTF8Sanitizer, strategy: replace_string
+```
+
 ## Contributing
 
 1. Fork it
@@ -63,4 +83,4 @@ To explicitly set sanitizable content types and override the defaults, use the `
 4. Push to the branch (`git push origin my-new-feature`)
 5. Create new Pull Request
 
-To run the tests, run `rake spec` in the project directory.
+To run the tests, run `rake spec` in the project directory.

data/lib/rack/utf8_sanitizer.rb

@@ -11,6 +11,7 @@ module Rack
     # options[:additional_content_types] Array
     def initialize(app, options={})
       @app = app
+      @strategy = build_strategy(options)
       @sanitizable_content_types = options[:sanitizable_content_types]
       @sanitizable_content_types ||= SANITIZABLE_CONTENT_TYPES + (options[:additional_content_types] || [])
     end
@@ -19,6 +20,21 @@ module Rack
       @app.call(sanitize(env))
     end
 
+    DEFAULT_STRATEGIES = {
+      replace: lambda do |input|
+        input.
+          force_encoding(Encoding::ASCII_8BIT).
+          encode!(Encoding::UTF_8,
+                  invalid: :replace,
+                  undef:   :replace)
+      end,
+      exception: lambda do |input|
+        input.
+          force_encoding(Encoding::ASCII_8BIT).
+          encode!(Encoding::UTF_8)
+      end
+    }.freeze
+
     # http://rack.rubyforge.org/doc/SPEC.html
     URI_FIELDS  = %w(
         SCRIPT_NAME
@@ -27,6 +43,7 @@ module Rack
         HTTP_REFERER
         ORIGINAL_FULLPATH
         ORIGINAL_SCRIPT_NAME
+        SERVER_NAME
     ).map(&:freeze).freeze
 
     SANITIZABLE_CONTENT_TYPES = %w(
@@ -40,13 +57,15 @@ module Rack
       application/x-www-form-urlencoded
     ).map(&:freeze).freeze
 
+    HTTP_ = 'HTTP_'.freeze
+
     def sanitize(env)
       sanitize_rack_input(env)
       env.each do |key, value|
         if URI_FIELDS.include?(key)
           env[key] = transfer_frozen(value,
               sanitize_uri_encoded_string(value))
-        elsif key.to_s.start_with?("HTTP_")
+        elsif key.to_s.start_with?(HTTP_)
           # Just sanitize the headers and leave them in UTF-8. There is
           # no reason to have UTF-8 in headers, but if it's valid, let it be.
           env[key] = transfer_frozen(value,
@@ -57,6 +76,14 @@ module Rack
 
     protected
 
+    def build_strategy(options)
+      strategy = options.fetch(:strategy) { :replace }
+
+      return strategy unless DEFAULT_STRATEGIES.key?(strategy)
+
+      DEFAULT_STRATEGIES[strategy]
+    end
+
     def sanitize_rack_input(env)
       # https://github.com/rack/rack/blob/master/lib/rack/request.rb#L42
       # Logic borrowed from Rack::Request#media_type,#media_type_params,#content_charset
@@ -156,7 +183,7 @@ module Rack
     # enough for our task.
     def unescape_unreserved(input)
       input.gsub(/%([a-f\d]{2})/i) do |encoded|
-        decoded = [$1.hex].pack('C')
+        decoded = $1.hex.chr
 
         if decoded =~ UNRESERVED_OR_UTF8
           decoded
@@ -187,11 +214,7 @@ module Rack
         if input.valid_encoding?
           input
         else
-          input.
-            force_encoding(Encoding::ASCII_8BIT).
-            encode!(Encoding::UTF_8,
-                    invalid: :replace,
-                    undef:   :replace)
+          @strategy.call(input)
         end
       else
         input

data/rack-utf8_sanitizer.gemspec

@@ -2,8 +2,9 @@
 
 Gem::Specification.new do |gem|
   gem.name          = "rack-utf8_sanitizer"
-  gem.version       = '1.3.2'
+  gem.version       = '1.4.0'
   gem.authors       = ["whitequark"]
+  gem.license       = "MIT"
   gem.email         = ["whitequark@whitequark.org"]
   gem.description   = %{Rack::UTF8Sanitizer is a Rack middleware which cleans up } <<
                       %{invalid UTF8 characters in request URI and headers.}

data/test/test_utf8_sanitizer.rb

@@ -28,6 +28,17 @@ describe Rack::UTF8Sanitizer do
     end
   end
 
+  describe "with invalid host input" do
+    it "sanitizes host entity (SERVER_NAME)" do
+      host   = "host\xD0".force_encoding('UTF-8')
+      env    = @app.({ "SERVER_NAME" => host })
+      result = env["SERVER_NAME"]
+
+      result.encoding.should == Encoding::US_ASCII
+      result.should.be.valid_encoding
+    end
+  end
+
   describe "with invalid UTF-8 input" do
     before do
       @plain_input = "foo\xe0".force_encoding('UTF-8')
@@ -397,4 +408,64 @@ describe Rack::UTF8Sanitizer do
       end
     end
   end
+
+  describe "with custom strategy" do
+    def request_env
+      @plain_input = "foo bar лол".force_encoding('UTF-8')
+      {
+          "REQUEST_METHOD" => "POST",
+          "CONTENT_TYPE" => "application/json",
+          "HTTP_USER_AGENT" => @plain_input,
+          "rack.input" => @rack_input,
+      }
+    end
+
+    def sanitize_data(request_env = request_env())
+      @uri_input = "http://bar/foo+%2F%3A+bar+%D0%BB%D0%BE%D0%BB".force_encoding('UTF-8')
+      @response_env = @app.(request_env)
+      sanitized_input = @response_env['rack.input'].read
+
+      yield sanitized_input if block_given?
+    end
+
+    it "calls a default strategy (replace)" do
+      @app = Rack::UTF8Sanitizer.new(-> env { env })
+
+      input = "foo=bla&quux=bar\xED"
+      @rack_input = StringIO.new input
+
+      env = request_env
+      sanitize_data(env) do |sanitized_input|
+        sanitized_input.encoding.should == Encoding::UTF_8
+        sanitized_input.should.be.valid_encoding
+        sanitized_input.should != input
+      end
+    end
+
+    it "calls the exception strategy" do
+      @app = Rack::UTF8Sanitizer.new(-> env { env }, strategy: :exception)
+
+      input = "foo=bla&quux=bar\xED"
+      @rack_input = StringIO.new input
+
+      env = request_env
+      should.raise(EncodingError) { sanitize_data(env) }
+    end
+
+    it "accepts a proc as a strategy" do
+      truncate = -> input { 'replace'.force_encoding(Encoding::UTF_8) }
+
+      @app = Rack::UTF8Sanitizer.new(-> env { env }, strategy: truncate)
+
+      input = "foo=bla&quux=bar\xED"
+      @rack_input = StringIO.new input
+
+      env = request_env
+      sanitize_data(env) do |sanitized_input|
+        sanitized_input.encoding.should == Encoding::UTF_8
+        sanitized_input.should.be.valid_encoding
+        sanitized_input.should == 'replace' 
+      end
+    end
+  end
 end

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: rack-utf8_sanitizer
 version: !ruby/object:Gem::Version
-  version: 1.3.2
+  version: 1.4.0
 platform: ruby
 authors:
 - whitequark
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2015-12-23 00:00:00.000000000 Z
+date: 2017-12-22 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rack
@@ -91,7 +91,8 @@ files:
 - rack-utf8_sanitizer.gemspec
 - test/test_utf8_sanitizer.rb
 homepage: http://github.com/whitequark/rack-utf8_sanitizer
-licenses: []
+licenses:
+- MIT
 metadata: {}
 post_install_message: 
 rdoc_options: []
@@ -109,7 +110,7 @@ required_rubygems_version: !ruby/object:Gem::Requirement
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.4.5.1
+rubygems_version: 2.5.2.1
 signing_key: 
 specification_version: 4
 summary: Rack::UTF8Sanitizer is a Rack middleware which cleans up invalid UTF8 characters