rack-utf8_sanitizer 1.0.0

data/.gitignore

@@ -0,0 +1,17 @@
+*.gem
+*.rbc
+.bundle
+.config
+.yardoc
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp

data/.travis.yml

@@ -0,0 +1,11 @@
+language: ruby
+
+rvm:
+  - 1.9.2
+  - 1.9.3
+  - 2.0.0
+  - jruby-19mode
+  - rbx-19mode
+
+script:
+  - rake spec

data/Gemfile

@@ -0,0 +1,4 @@
+source 'https://rubygems.org'
+
+# Specify your gem's dependencies in rack-utf8_sanitizer.gemspec
+gemspec

data/LICENSE.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2013 Peter Zotov
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,48 @@
+# Rack::UTF8Sanitizer
+
+Rack::UTF8Sanitizer is a Rack middleware which cleans up invalid UTF8 characters in request URI and headers.
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+    gem 'rack-utf8_sanitizer'
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install rack-utf8_sanitizer
+
+For Rails, add this to your `application.rb`:
+
+    config.middleware.insert_before "Rack::Lock", Rack::UTF8Sanitizer
+
+For Rack apps, add this to `config.ru`:
+
+    use Rack::UTF8Sanitizer
+
+## Usage
+
+Rack::UTF8Sanitizer divides all keys in the [Rack environment](rack.rubyforge.org/doc/SPEC.html) in two distinct groups: keys which contain raw data and the ones with percent-encoded data. The fields which are treated as percent-encoded are: `SCRIPT_NAME`, `REQUEST_PATH`, `REQUEST_URI`, `PATH_INFO`, `QUERY_STRING`, `HTTP_REFERER`.
+
+The generic sanitization algorithm is as follows:
+
+  1. Force the encoding to UTF-8.
+  2. If the result contains invalid characters:
+      1. Force the encoding to ASCII8-BIT.
+      2. Re-encode it as UTF-8, replacing invalid and undefined characters as U+FFFD.
+
+For fields with "raw data", the algorithm is applied once and the (UTF-8 encoded) result is left in the environment.
+
+For fields with "percent-encoded data", the algorithm is applied twice to catch both invalid characters appearing as-is and invalid characters appearing in the percent encoding. The percent encoded, ASCII-8BIT encoded result is left in the environment.
+
+## Contributing
+
+1. Fork it
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create new Pull Request

data/Rakefile

@@ -0,0 +1,8 @@
+require "bundler/gem_tasks"
+
+task :default => :spec
+
+desc "Run tests"
+task :spec do
+  sh 'bacon -a'
+end

data/lib/rack/utf8_sanitizer.rb

@@ -0,0 +1,78 @@
+require 'uri'
+
+module Rack
+  class UTF8Sanitizer
+    def initialize(app)
+      @app = app
+    end
+
+    def call(env)
+      @app.call(sanitize(env))
+    end
+
+    # http://rack.rubyforge.org/doc/SPEC.html
+    URI_FIELDS  = %w(
+        SCRIPT_NAME
+        REQUEST_PATH REQUEST_URI PATH_INFO
+        QUERY_STRING
+        HTTP_REFERER
+    )
+
+    def sanitize(env)
+      env.each do |key, value|
+        if URI_FIELDS.include?(key)
+          # URI.encode/decode expect the input to be in ASCII-8BIT.
+          # However, there could be invalid UTF-8 characters both in
+          # raw and percent-encoded form.
+          #
+          # So, first sanitize the value, then percent-decode it while
+          # treating as UTF-8, then sanitize the result and encode it back.
+          #
+          # The result is guaranteed to be UTF-8-safe.
+
+          decoded_value = URI.decode(
+              sanitize_string(value).
+              force_encoding('ASCII-8BIT'))
+
+          env[key] = transfer_frozen(value,
+              URI.encode(sanitize_string(decoded_value)))
+
+        elsif key =~ /^HTTP_/
+          # Just sanitize the headers and leave them in UTF-8. There is
+          # no reason to have UTF-8 in headers, but if it's valid, let it be.
+
+          env[key] = transfer_frozen(value,
+              sanitize_string(value))
+        end
+      end
+    end
+
+    protected
+
+    def sanitize_string(input)
+      if input.is_a? String
+        input = input.dup.force_encoding('UTF-8')
+
+        if input.valid_encoding?
+          input
+        else
+          input.
+            force_encoding('ASCII-8BIT').
+            encode!('UTF-8',
+                    invalid: :replace,
+                    undef:   :replace)
+        end
+      else
+        input
+      end
+    end
+
+    def transfer_frozen(from, to)
+      if from.frozen?
+        to.freeze
+      else
+        to
+      end
+    end
+  end
+end

data/rack-utf8_sanitizer.gemspec

@@ -0,0 +1,24 @@
+# -*- encoding: utf-8 -*-
+
+Gem::Specification.new do |gem|
+  gem.name          = "rack-utf8_sanitizer"
+  gem.version       = '1.0.0'
+  gem.authors       = ["Peter Zotov"]
+  gem.email         = ["whitequark@whitequark.org"]
+  gem.description   = %{Rack::UTF8Sanitizer is a Rack middleware which cleans up } <<
+                      %{invalid UTF8 characters in request URI and headers.}
+  gem.summary       = gem.description
+  gem.homepage      = "http://github.com/whitequark/rack-utf8_sanitizer"
+
+  gem.files         = `git ls-files`.split($/)
+  gem.executables   = gem.files.grep(%r{^bin/}).map{ |f| File.basename(f) }
+  gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
+  gem.require_paths = ["lib"]
+
+  gem.required_ruby_version = '>= 1.9'
+
+  gem.add_dependency             "rack", '~> 1.0'
+
+  gem.add_development_dependency "bacon"
+  gem.add_development_dependency "bacon-colored_output"
+end

data/test/test_utf8_sanitizer.rb

@@ -0,0 +1,138 @@
+# encoding:ascii-8bit
+
+require 'bacon/colored_output'
+require 'rack/utf8_sanitizer'
+
+describe Rack::UTF8Sanitizer do
+  before do
+    @app = Rack::UTF8Sanitizer.new(-> env { env })
+  end
+
+  shared :does_sanitize_plain do
+    it "sanitizes plaintext entity (HTTP_USER_AGENT)" do
+      env    = @app.({ "HTTP_USER_AGENT" => @plain_input })
+      result = env["HTTP_USER_AGENT"]
+
+      result.encoding.should == Encoding::UTF_8
+      result.should.be.valid_encoding
+    end
+  end
+
+  shared :does_sanitize_uri do
+    it "sanitizes URI-like entity (REQUEST_PATH)" do
+      env    = @app.({ "REQUEST_PATH" => @uri_input })
+      result = env["REQUEST_PATH"]
+
+      result.encoding.should == Encoding::US_ASCII
+      result.should.be.valid_encoding
+    end
+  end
+
+  describe "with invalid UTF-8 input" do
+    before do
+      @plain_input = "foo\xe0".force_encoding('UTF-8')
+      @uri_input   = "foo%E0".force_encoding('UTF-8')
+    end
+
+    behaves_like :does_sanitize_plain
+    behaves_like :does_sanitize_uri
+  end
+
+  describe "with invalid, incorrectly percent-encoded UTF-8 URI input" do
+    before do
+      @uri_input   = "foo%E0\xe0".force_encoding('UTF-8')
+    end
+
+    behaves_like :does_sanitize_uri
+  end
+
+  describe "with invalid ASCII-8BIT input" do
+    before do
+      @plain_input = "foo\xe0"
+      @uri_input   = "foo%E0"
+    end
+
+    behaves_like :does_sanitize_plain
+    behaves_like :does_sanitize_uri
+  end
+
+  describe "with invalid, incorrectly percent-encoded ASCII-8BIT URI input" do
+    before do
+      @uri_input   = "foo%E0\xe0"
+    end
+
+    behaves_like :does_sanitize_uri
+  end
+
+  shared :identity_plain do
+    it "does not change plaintext entity (HTTP_USER_AGENT)" do
+      env    = @app.({ "HTTP_USER_AGENT" => @plain_input })
+      result = env["HTTP_USER_AGENT"]
+
+      result.encoding.should == Encoding::UTF_8
+      result.should.be.valid_encoding
+      result.should == @plain_input
+    end
+  end
+
+  shared :identity_uri do
+    it "does not change URI-like entity (REQUEST_PATH)" do
+      env    = @app.({ "REQUEST_PATH" => @uri_input })
+      result = env["REQUEST_PATH"]
+
+      result.encoding.should == Encoding::US_ASCII
+      result.should.be.valid_encoding
+      result.should == @uri_input
+    end
+  end
+
+  describe "with valid UTF-8 input" do
+    before do
+      @plain_input = "foo bar лол".force_encoding('UTF-8')
+      @uri_input   = "foo+bar+%D0%BB%D0%BE%D0%BB".force_encoding('UTF-8')
+    end
+
+    behaves_like :identity_plain
+    behaves_like :identity_uri
+  end
+
+  describe "with valid, not percent-encoded UTF-8 URI input" do
+    before do
+      @uri_input   = "foo+bar+лол".force_encoding('UTF-8')
+    end
+
+    it "does not change URI-like entity (REQUEST_PATH)" do
+      env    = @app.({ "REQUEST_PATH" => @uri_input })
+      result = env["REQUEST_PATH"]
+
+      result.encoding.should == Encoding::US_ASCII
+      result.should.be.valid_encoding
+      result.should == URI.encode(@uri_input)
+    end
+  end
+
+  describe "with valid ASCII-8BIT input" do
+    before do
+      @plain_input = "bar baz"
+      @uri_input   = "bar+baz"
+    end
+
+    behaves_like :identity_plain
+    behaves_like :identity_uri
+  end
+
+  describe "with frozen strings" do
+    before do
+      @plain_input = "bar baz".freeze
+      @uri_input   = "bar+baz".freeze
+    end
+
+    it "preserves the frozen? status of input" do
+      env  = @app.({ "HTTP_USER_AGENT" => @plain_input,
+                     "REQUEST_PATH" => @uri_input })
+
+      env["HTTP_USER_AGENT"].should.be.frozen
+      env["REQUEST_PATH"].should.be.frozen
+    end
+  end
+end

metadata

@@ -0,0 +1,106 @@
+--- !ruby/object:Gem::Specification
+name: rack-utf8_sanitizer
+version: !ruby/object:Gem::Version
+  version: 1.0.0
+  prerelease: 
+platform: ruby
+authors:
+- Peter Zotov
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2013-03-05 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: rack
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.0'
+- !ruby/object:Gem::Dependency
+  name: bacon
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: bacon-colored_output
+  requirement: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ! '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Rack::UTF8Sanitizer is a Rack middleware which cleans up invalid UTF8
+  characters in request URI and headers.
+email:
+- whitequark@whitequark.org
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- .travis.yml
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- lib/rack/utf8_sanitizer.rb
+- rack-utf8_sanitizer.gemspec
+- test/test_utf8_sanitizer.rb
+homepage: http://github.com/whitequark/rack-utf8_sanitizer
+licenses: []
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '1.9'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  none: false
+  requirements:
+  - - ! '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 1.8.23
+signing_key: 
+specification_version: 3
+summary: Rack::UTF8Sanitizer is a Rack middleware which cleans up invalid UTF8 characters
+  in request URI and headers.
+test_files:
+- test/test_utf8_sanitizer.rb
+has_rdoc: