utf8-cleaner 0.0.9 → 0.1.1

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 34470c6433b9f2fc055514464d9c4794881ed579
-  data.tar.gz: 5135b3b7e6ea404b3dcc57d07ea9aee798a528d4
+  metadata.gz: d4f4b8427b2de1cca2aad8cf672164873552352c
+  data.tar.gz: f4c6e987bd3dcdcdd459b896ca6e8768fe103ad8
 SHA512:
-  metadata.gz: 35b02095acfd32a1a5c1a380ed017e130c25dae095fc2f1bee135a1451f1394f6018eba74f7d4567b3158fbf8978d79ad968fd07a947fde054bf6d0e38766e68
-  data.tar.gz: 9fef626eb986add0193d716339e6df9e7faf22cc8767c7257b2514265ccd3f2294360df45f6c04afd4977f88ced34b4e51f9160231c12e041e0e586ec35bf64d
+  metadata.gz: 5984ac142b4497cc4dfb3626f09b79a5c74c0e27ac449239876a68cf1d73adc49433b163573d037ed5c91963b7c020e9cb8653a68a75c5eb09b13cf006f8533f
+  data.tar.gz: b9640a1a528a8873338dd08c1bf278994fec361a91c4dfc6562d01c21871f8029b48b120bc35453ff8fc7c948fe4c02df4d3da59dac68504393990d882f7c29e

data/.travis.yml

@@ -2,4 +2,5 @@ language: ruby
 rvm:
   - 1.9.3
   - 2.0.0
-  - 2.1.1
+  - 2.1.1
+  - 2.2.1

data/CHANGELOG.md

@@ -0,0 +1,10 @@
+# CHANGELOG
+
+## v0.1.1
+
+* Now cleans HTTP_USER_AGENT
+* Replaces some Windows (ISO-8859-1 and CP1252) characters with UTF8 equivalents
+
+## v0.1.0
+
+Broken.

data/Guardfile

@@ -1,7 +1,7 @@
 # A sample Guardfile
 # More info at https://github.com/guard/guard#readme
 
-guard :rspec do
+guard :rspec, cmd: 'bundle exec rspec -b' do
   watch(%r{^spec/.+_spec\.rb$})
   watch(%r{^lib/utf8-cleaner/(.+)\.rb$}) { |m| "spec/#{m[1]}_spec.rb" }
   watch('spec/spec_helper.rb') { "spec" }

data/lib/utf8-cleaner/middleware.rb

@@ -1,8 +1,11 @@
+require 'active_support/multibyte/unicode'
+
 module UTF8Cleaner
   class Middleware
 
     SANITIZE_ENV_KEYS = [
      "HTTP_REFERER",
+     "HTTP_USER_AGENT",
      "PATH_INFO",
      "QUERY_STRING",
      "REQUEST_PATH",
@@ -20,6 +23,8 @@ module UTF8Cleaner
 
     private
 
+    include ActiveSupport::Multibyte::Unicode
+
     def sanitize_env(env)
       sanitize_env_keys(env)
       sanitize_env_rack_input(env)
@@ -29,8 +34,7 @@ module UTF8Cleaner
     def sanitize_env_keys(env)
       SANITIZE_ENV_KEYS.each do |key|
         next unless value = env[key]
-        cleaned_value = cleaned_uri_string(value)
-        env[key] = cleaned_value if cleaned_value
+        env[key] = cleaned_uri_string(value)
       end
     end
 
@@ -48,9 +52,9 @@ module UTF8Cleaner
     end
 
     def cleaned_uri_string(value)
-      if value.include?('%')
-        URIString.new(value).cleaned
-      end
+      value = tidy_bytes(value) if value && !value.ascii_only?
+      value = URIString.new(value).cleaned if value.include?('%')
+      value
     end
   end
 end

data/lib/utf8-cleaner/uri_string.rb

@@ -16,6 +16,11 @@ module UTF8Cleaner
 
     def valid?
       valid_uri_encoded_utf8(data)
+    rescue ArgumentError => e
+      if e.message =~ /invalid byte sequence/
+        return false
+      end
+      raise e
     end
 
     private

data/lib/utf8-cleaner/version.rb

@@ -1,3 +1,3 @@
 module UTF8Cleaner
-  VERSION = "0.0.9"
+  VERSION = "0.1.1"
 end

data/spec/middleware_spec.rb

@@ -1,51 +1,78 @@
 require 'spec_helper'
 require 'rack/lint'
 
-describe UTF8Cleaner::Middleware do
-  let :env do
-    {
-      'PATH_INFO' => 'foo/%FFbar%2e%2fbaz%26%3B',
-      'QUERY_STRING' => 'foo=bar%FF',
-      'HTTP_REFERER' => 'http://example.com/blog+Result:+%ED%E5+%ED%E0%F8%EB%EE%F1%FC+%F4%EE%F0%EC%FB+%E4%EB%FF+%EE%F2%EF%F0%E0%E2%EA%E8',
-      'REQUEST_URI' => '%C3%89%E2%9C%93',
-      'rack.input' => StringIO.new("foo=%FFbar%F8"),
-      'CONTENT_TYPE' => 'application/x-www-form-urlencoded'
-    }
-  end
+module UTF8Cleaner
+  describe Middleware do
+    let :new_env do
+      Middleware.new(nil).send(:sanitize_env, env)
+    end
 
-  let :new_env do
-    UTF8Cleaner::Middleware.new(nil).send(:sanitize_env, env)
-  end
+    describe "with a big nasty env" do
+      let :env do
+        {
+          'PATH_INFO' => 'foo/%FFbar%2e%2fbaz%26%3B',
+          'QUERY_STRING' => 'foo=bar%FF',
+          'HTTP_REFERER' => 'http://example.com/blog+Result:+%ED%E5+%ED%E0%F8%EB%EE%F1%FC+%F4%EE%F0%EC%FB+%E4%EB%FF+%EE%F2%EF%F0%E0%E2%EA%E8',
+          'HTTP_USER_AGENT' => "Android Versi\xF3n/4.0",
+          'REQUEST_URI' => '%C3%89%E2%9C%93',
+          'rack.input' => StringIO.new("foo=%FFbar%F8"),
+          'CONTENT_TYPE' => 'application/x-www-form-urlencoded'
+        }
+      end
 
-  describe "removes invalid UTF-8 sequences" do
-    it { new_env['QUERY_STRING'].should == 'foo=bar' }
-    it { new_env['HTTP_REFERER'].should == 'http://example.com/blog+Result:+++++' }
-    it { new_env['rack.input'].read.should == 'foo=bar' }
-  end
+      describe "removes invalid %-encoded UTF-8 sequences" do
+        it { expect(new_env['QUERY_STRING']).to eq('foo=bar') }
+        it { expect(new_env['HTTP_REFERER']).to eq('http://example.com/blog+Result:+++++') }
+        it { expect(new_env['rack.input'].read).to eq('foo=bar') }
+      end
 
-  describe "leaves all valid characters untouched" do
-    it { new_env['PATH_INFO'].should == 'foo/bar%2e%2fbaz%26%3B' }
-    it { new_env['REQUEST_URI'].should == '%C3%89%E2%9C%93' }
-  end
+      describe 'replaces \x-encoded characters from the ISO-8859-1 and CP1252 code pages with their UTF-8 equivalents' do
+        it { expect(new_env['HTTP_USER_AGENT']).to eq('Android Versión/4.0') }
+      end
 
-  describe "when rack.input is wrapped" do
-    # rack.input responds only to methods gets, each, rewind, read and close
-    # Rack::Lint::InputWrapper is the class which servers wrappers are based on
-    it "removes invalid UTF-8 sequences" do
-      wrapped_rack_input = Rack::Lint::InputWrapper.new(StringIO.new("foo=%FFbar%F8"))
-      env.merge!('rack.input' => wrapped_rack_input)
-      new_env = UTF8Cleaner::Middleware.new(nil).send(:sanitize_env, env)
-      new_env['rack.input'].read.should == 'foo=bar'
-    end
-  end
+      describe "leaves all valid characters untouched" do
+        it { expect(new_env['PATH_INFO']).to eq('foo/bar%2e%2fbaz%26%3B') }
+        it { expect(new_env['REQUEST_URI']).to eq('%C3%89%E2%9C%93') }
+      end
+
+      describe "when rack.input is wrapped" do
+        # rack.input responds only to methods gets, each, rewind, read and close
+        # Rack::Lint::InputWrapper is the class which servers wrappers are based on
+        it "removes invalid UTF-8 sequences" do
+          wrapped_rack_input = Rack::Lint::InputWrapper.new(StringIO.new("foo=%FFbar%F8"))
+          env.merge!('rack.input' => wrapped_rack_input)
+          new_env = Middleware.new(nil).send(:sanitize_env, env)
+          expect(new_env['rack.input'].read).to eq('foo=bar')
+        end
+      end
 
-  describe "when binary data is POSTed" do
-    before do
-      env['CONTENT_TYPE'] = 'multipart/form-data'
+      describe "when binary data is POSTed" do
+        before do
+          env['CONTENT_TYPE'] = 'multipart/form-data'
+        end
+        it "leaves the body alone" do
+          env['rack.input'].rewind
+          expect(new_env['rack.input'].read).to eq "foo=%FFbar%F8"
+        end
+      end
     end
-    it "leaves the body alone" do
-      env['rack.input'].rewind
-      new_env['rack.input'].read.should == "foo=%FFbar%F8"
+
+    describe "with a minimal env" do
+      let(:env) do
+        {
+          'PATH_INFO' => '/this/is/safe',
+          'QUERY_STRING' => 'foo=bar%FF'
+        }
+      end
+
+      it "only runs URIString cleaning on potentially unclean strings" do
+        expect(URIString).to receive(:new).once.and_call_original
+        new_env
+      end
+
+      it "leaves clean values alone" do
+        expect(new_env['PATH_INFO']).to eq('/this/is/safe')
+      end
     end
   end
-end
+end

data/spec/spec_helper.rb

@@ -1,12 +1,8 @@
 require 'rubygems'
-require 'rspec/autorun'
-
 require 'utf8-cleaner'
 require 'uri'
 
 RSpec.configure do |config|
-  config.treat_symbols_as_metadata_keys_with_true_values = true
-
   # Run specs in random order to surface order dependencies. If you find an
   # order dependency and want to debug it, you can fix the order by providing
   # the seed, which is printed after each run.

data/spec/uri_string_spec.rb

@@ -11,24 +11,24 @@ module UTF8Cleaner
                                                 # foo/   bar.  /  baz&  ;  √              baz
 
     describe '#new' do
-      it { encoded_string.should be_a URIString }
+      it { expect(encoded_string).to be_a(URIString) }
     end
 
     describe '#cleaned' do
-      it { invalid_string.cleaned.should eq('') }
-      it { ascii_string.cleaned.should eq('foo') }
-      it { encoded_string.cleaned.should eq('%26') }
-      it { multibyte_string.cleaned.should eq('%E2%9C%93') }
-      it { complex_invalid_string.cleaned.should eq('foo/bar%2e%2fbaz%26%3B%E2%9C%93baz') }
+      it { expect(invalid_string.cleaned).to eq('') }
+      it { expect(ascii_string.cleaned).to eq('foo') }
+      it { expect(encoded_string.cleaned).to eq('%26') }
+      it { expect(multibyte_string.cleaned).to eq('%E2%9C%93') }
+      it { expect(complex_invalid_string.cleaned).to eq('foo/bar%2e%2fbaz%26%3B%E2%9C%93baz') }
     end
 
     describe '#valid?' do
-      it { ascii_string.should be_valid }
-      it { encoded_string.should be_valid }
-      it { multibyte_string.should be_valid }
+      it { expect(ascii_string).to be_valid }
+      it { expect(encoded_string).to be_valid }
+      it { expect(multibyte_string).to be_valid }
 
-      it { invalid_string.should_not be_valid }
-      it { complex_invalid_string.should_not be_valid }
+      it { expect(invalid_string).to_not be_valid }
+      it { expect(complex_invalid_string).to_not be_valid }
     end
 
   end

data/utf8-cleaner.gemspec

@@ -17,6 +17,8 @@ Gem::Specification.new do |gem|
   gem.test_files    = gem.files.grep(%r{^(test|spec|features)/})
   gem.require_paths = ["lib"]
 
+  gem.add_dependency 'activesupport'
+
   gem.add_development_dependency "rake"
   gem.add_development_dependency "guard"
   gem.add_development_dependency "guard-rspec"

metadata

@@ -1,15 +1,29 @@
 --- !ruby/object:Gem::Specification
 name: utf8-cleaner
 version: !ruby/object:Gem::Version
-  version: 0.0.9
+  version: 0.1.1
 platform: ruby
 authors:
 - Leon Miller-Out
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-04-16 00:00:00.000000000 Z
+date: 2015-09-19 00:00:00.000000000 Z
 dependencies:
+- !ruby/object:Gem::Dependency
+  name: activesupport
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ">="
+      - !ruby/object:Gem::Version
+        version: '0'
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
@@ -89,6 +103,7 @@ extra_rdoc_files: []
 files:
 - ".gitignore"
 - ".travis.yml"
+- CHANGELOG.md
 - Gemfile
 - Guardfile
 - LICENSE.txt