rack-utf8_sanitizer 1.1.0 → 1.2.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: be92627f205eac2e80ab554ad5add8ca2a084026
+  data.tar.gz: 5325281ed65e1e17d74cabbb75df877e0387e2e8
+SHA512:
+  metadata.gz: a3cc3842501b3c7e7bd4da92ae81432a6b6d014ab4af8a828a4269d781554fdb774e6c7fe3dd19ea582d959ea8d3093abc1984882587aede7084bf6e3381c10c
+  data.tar.gz: 108c557d20ce9c4716ae5d89fcd2878b8eb6ef3cd8a359d3e240a2cef575ef9a69a9cd129f8ab00bbde5628badb5f4d7f3f374badcaa04693162fa2f40531ac0

data/.travis.yml

@@ -4,12 +4,10 @@ rvm:
   - 1.9.2
   - 1.9.3
   - 2.0.0
-  - jruby-19mode
-  - rbx-19mode
+  # 2.1, not 2.1.0 until fixed https://github.com/travis-ci/travis-ci/issues/2220
+  - 2.1
+  - jruby
+  - rbx-2
 
 script:
   - rake spec
-
-matrix:
-  allow_failures:
-    - rvm: jruby-19mode

data/README.md

@@ -19,7 +19,7 @@ Or install it yourself as:
 For Rails, add this to your `application.rb`:
 
 ``` ruby
-config.middleware.insert_before "Rack::Lock", Rack::UTF8Sanitizer
+config.middleware.insert_before "Rack::Runtime", Rack::UTF8Sanitizer
 ```
 
 For Rack apps, add this to `config.ru`:
@@ -30,7 +30,7 @@ use Rack::UTF8Sanitizer
 
 ## Usage
 
-Rack::UTF8Sanitizer divides all keys in the [Rack environment](rack.rubyforge.org/doc/SPEC.html) in two distinct groups: keys which contain raw data and the ones with percent-encoded data. The fields which are treated as percent-encoded are: `SCRIPT_NAME`, `REQUEST_PATH`, `REQUEST_URI`, `PATH_INFO`, `QUERY_STRING`, `HTTP_REFERER`.
+Rack::UTF8Sanitizer divides all keys in the [Rack environment](http://rack.rubyforge.org/doc/SPEC.html) in two distinct groups: keys which contain raw data and the ones with percent-encoded data. The fields which are treated as percent-encoded are: `SCRIPT_NAME`, `REQUEST_PATH`, `REQUEST_URI`, `PATH_INFO`, `QUERY_STRING`, `HTTP_REFERER`.
 
 The generic sanitization algorithm is as follows:
 

data/lib/rack/utf8_sanitizer.rb

@@ -1,9 +1,12 @@
 # encoding: ascii-8bit
 
 require 'uri'
+require 'stringio'
 
 module Rack
   class UTF8Sanitizer
+    StringIO = ::StringIO
+
     def initialize(app)
       @app = app
     end
@@ -20,30 +23,27 @@ module Rack
         HTTP_REFERER
     )
 
+    SANITIZABLE_CONTENT_TYPES = %w(
+      text/plain
+      application/x-www-form-urlencoded
+    )
+
+    # MRI-optimization
+    POST = 'POST'
+    PUT  = 'PUT'
+
     def sanitize(env)
+      request_method = env['REQUEST_METHOD']
+      if request_method == POST || request_method == PUT
+        sanitize_rack_input(env)
+      end
       env.each do |key, value|
         if URI_FIELDS.include?(key)
-          # URI.encode/decode expect the input to be in ASCII-8BIT.
-          # However, there could be invalid UTF-8 characters both in
-          # raw and percent-encoded form.
-          #
-          # So, first sanitize the value, then percent-decode it while
-          # treating as UTF-8, then sanitize the result and encode it back.
-          #
-          # The result is guaranteed to be UTF-8-safe.
-
-          decoded_value = unescape_unreserved(
-              sanitize_string(value).
-              force_encoding('ASCII-8BIT'))
-
           env[key] = transfer_frozen(value,
-              escape_unreserved(
-                sanitize_string(decoded_value)))
-
-        elsif key =~ /^HTTP_/
+              sanitize_uri_encoded_string(value))
+        elsif key.start_with?("HTTP_")
           # Just sanitize the headers and leave them in UTF-8. There is
           # no reason to have UTF-8 in headers, but if it's valid, let it be.
-
           env[key] = transfer_frozen(value,
               sanitize_string(value))
         end
@@ -52,6 +52,73 @@ module Rack
 
     protected
 
+    def sanitize_rack_input(env)
+      # https://github.com/rack/rack/blob/master/lib/rack/request.rb#L42
+      # Logic borrowed from Rack::Request#media_type,#media_type_params,#content_charset
+      # Ignoring charset in content type.
+      content_type   = env['CONTENT_TYPE']
+      content_type &&= content_type.split(/\s*[;,]\s*/, 2).first
+      content_type &&= content_type.downcase
+      return unless SANITIZABLE_CONTENT_TYPES.any? {|type| content_type == type }
+      env['rack.input'] &&= sanitize_io(env['rack.input'])
+    end
+
+    # Modeled after Rack::RewindableInput
+    # TODO: Should this delegate any methods to the original io?
+    class SanitizedRackInput
+      def initialize(original_io, sanitized_io)
+        @original_io = original_io
+        @sanitized_io = sanitized_io
+      end
+      def gets
+        @sanitized_io.gets
+      end
+      def read(*args)
+        @sanitized_io.read(*args)
+      end
+      def each(&block)
+        @sanitized_io.each(&block)
+      end
+      def rewind
+        @sanitized_io.rewind
+      end
+      def close
+        @sanitized_io.close
+        @original_io.close if @original_io.respond_to?(:close)
+      end
+    end
+
+    def sanitize_io(io)
+      input = io.read
+      sanitized_io = transfer_frozen(input,
+                      sanitize_string(input))
+      SanitizedRackInput.new(io, StringIO.new(sanitized_io))
+    end
+
+    # URI.encode/decode expect the input to be in ASCII-8BIT.
+    # However, there could be invalid UTF-8 characters both in
+    # raw and percent-encoded form.
+    #
+    # So, first sanitize the value, then percent-decode it while
+    # treating as UTF-8, then sanitize the result and encode it back.
+    #
+    # The result is guaranteed to be UTF-8-safe.
+    def sanitize_uri_encoded_string(input)
+      decoded_value = decode_string(input)
+      reencode_string(decoded_value)
+    end
+
+    def reencode_string(decoded_value)
+      escape_unreserved(
+        sanitize_string(decoded_value))
+    end
+
+    def decode_string(input)
+      unescape_unreserved(
+        sanitize_string(input).
+          force_encoding(Encoding::ASCII_8BIT))
+    end
+
     # This regexp matches all 'unreserved' characters from RFC3986 (2.3),
     # plus all multibyte UTF-8 characters.
     UNRESERVED_OR_UTF8 = /[A-Za-z0-9\-._~\x80-\xFF]/
@@ -82,21 +149,21 @@ module Rack
     UNSAFE           = /[^\-_.!~*'()a-zA-Z\d;\/?:@&=+$,\[\]%]/
 
     # Performs the reverse function of `unescape_unreserved`. Unlike
-    # the previous function, we can reuse the logic in URI#escape.
+    # the previous function, we can reuse the logic in URI#encode
     def escape_unreserved(input)
-      URI.escape(input, UNSAFE)
+      URI.encode(input, UNSAFE)
     end
 
     def sanitize_string(input)
       if input.is_a? String
-        input = input.dup.force_encoding('UTF-8')
+        input = input.dup.force_encoding(Encoding::UTF_8)
 
         if input.valid_encoding?
           input
         else
           input.
-            force_encoding('ASCII-8BIT').
-            encode!('UTF-8',
+            force_encoding(Encoding::ASCII_8BIT).
+            encode!(Encoding::UTF_8,
                     invalid: :replace,
                     undef:   :replace)
         end

data/rack-utf8_sanitizer.gemspec

@@ -2,7 +2,7 @@
 
 Gem::Specification.new do |gem|
   gem.name          = "rack-utf8_sanitizer"
-  gem.version       = '1.1.0'
+  gem.version       = '1.2.1'
   gem.authors       = ["Peter Zotov"]
   gem.email         = ["whitequark@whitequark.org"]
   gem.description   = %{Rack::UTF8Sanitizer is a Rack middleware which cleans up } <<

data/test/test_utf8_sanitizer.rb

@@ -151,4 +151,99 @@ describe Rack::UTF8Sanitizer do
       env["REQUEST_PATH"].should.be.frozen
     end
   end
+
+  describe "with form data" do
+    def request_env
+      @plain_input = "foo bar лол".force_encoding('UTF-8')
+      {
+         "REQUEST_METHOD" => "POST",
+         "CONTENT_TYPE" => "application/x-www-form-urlencoded;foo=bar",
+         "HTTP_USER_AGENT" => @plain_input,
+         "rack.input" => @rack_input,
+      }
+    end
+    def sanitize_form_data(request_env = request_env)
+      @uri_input   = "http://bar/foo+%2F%3A+bar+%D0%BB%D0%BE%D0%BB".force_encoding('UTF-8')
+      env = @app.(request_env)
+      sanitized_input = env['rack.input'].read
+      yield sanitized_input if block_given?
+      env['rack.input'].rewind
+      behaves_like :does_sanitize_plain
+      behaves_like :does_sanitize_uri
+      behaves_like :identity_plain
+      behaves_like :identity_uri
+      env['rack.input'].close
+    end
+
+    it "sanitizes StringIO rack.input" do
+      input = "foo=bla&quux=bar"
+      @rack_input = StringIO.new input
+
+      sanitize_form_data do |sanitized_input|
+        sanitized_input.encoding.should == Encoding::UTF_8
+        sanitized_input.should.be.valid_encoding
+        sanitized_input.should == input
+      end
+    end
+
+    it "sanitizes StringIO rack.input with bad encoding" do
+      input =  "foo=bla&quux=bar\xED"
+      @rack_input = StringIO.new input
+
+      sanitize_form_data do |sanitized_input|
+        sanitized_input.encoding.should == Encoding::UTF_8
+        sanitized_input.should.be.valid_encoding
+        sanitized_input.should != input
+      end
+    end
+
+    it "sanitizes non-StringIO rack.input" do
+      require 'rack/rewindable_input'
+      input = "foo=bla&quux=bar"
+      @rack_input = Rack::RewindableInput.new(StringIO.new(input))
+
+      sanitize_form_data do |sanitized_input|
+        sanitized_input.encoding.should == Encoding::UTF_8
+        sanitized_input.should.be.valid_encoding
+        sanitized_input.should == input
+      end
+    end
+
+    it "sanitizes non-StringIO rack.input with bad encoding" do
+      require 'rack/rewindable_input'
+      input =  "foo=bla&quux=bar\xED"
+      @rack_input = Rack::RewindableInput.new(StringIO.new(input))
+
+      sanitize_form_data do |sanitized_input|
+        sanitized_input.encoding.should == Encoding::UTF_8
+        sanitized_input.should.be.valid_encoding
+        sanitized_input.should != input
+      end
+    end
+
+    it "does not sanitize the rack body if there is no CONTENT_TYPE" do
+      input =  "foo=bla&quux=bar\xED"
+      @rack_input = StringIO.new input
+
+      env = request_env.update('CONTENT_TYPE' => nil)
+      sanitize_form_data(env) do |sanitized_input|
+        sanitized_input.encoding.should == Encoding::ASCII_8BIT
+        sanitized_input.should.be.valid_encoding
+        sanitized_input.should == input
+      end
+    end
+
+    it "does not sanitize the rack body if there is empty CONTENT_TYPE" do
+      input =  "foo=bla&quux=bar\xED"
+      @rack_input = StringIO.new input
+
+      env = request_env.update('CONTENT_TYPE' => '')
+      sanitize_form_data(env) do |sanitized_input|
+        sanitized_input.encoding.should == Encoding::ASCII_8BIT
+        sanitized_input.should.be.valid_encoding
+        sanitized_input.should == input
+      end
+    end
+
+  end
 end

metadata

@@ -1,62 +1,55 @@
 --- !ruby/object:Gem::Specification
 name: rack-utf8_sanitizer
 version: !ruby/object:Gem::Version
-  version: 1.1.0
-  prerelease: 
+  version: 1.2.1
 platform: ruby
 authors:
 - Peter Zotov
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2013-03-15 00:00:00.000000000 Z
+date: 2014-05-27 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: rack
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '1.0'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '1.0'
 - !ruby/object:Gem::Dependency
   name: bacon
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 - !ruby/object:Gem::Dependency
   name: bacon-colored_output
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - ">="
       - !ruby/object:Gem::Version
         version: '0'
 description: Rack::UTF8Sanitizer is a Rack middleware which cleans up invalid UTF8
@@ -67,8 +60,8 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
-- .gitignore
-- .travis.yml
+- ".gitignore"
+- ".travis.yml"
 - Gemfile
 - LICENSE.txt
 - README.md
@@ -78,27 +71,26 @@ files:
 - test/test_utf8_sanitizer.rb
 homepage: http://github.com/whitequark/rack-utf8_sanitizer
 licenses: []
+metadata: {}
 post_install_message: 
 rdoc_options: []
 require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
-  none: false
   requirements:
-  - - ! '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '1.9'
 required_rubygems_version: !ruby/object:Gem::Requirement
-  none: false
   requirements:
-  - - ! '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 1.8.23
+rubygems_version: 2.2.2
 signing_key: 
-specification_version: 3
+specification_version: 4
 summary: Rack::UTF8Sanitizer is a Rack middleware which cleans up invalid UTF8 characters
   in request URI and headers.
 test_files: