url_normalizer 0.0.1

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: f3d1f7c14baac847e69e2183e034ba1f264eb427
+  data.tar.gz: 3d9b4927ab8615bdd3f5b627eb506a7f318ffdf6
+SHA512:
+  metadata.gz: c3869cc402f2d7cb00328fdcfe232fd91f1c99844230f3ce119e7052d41b75c6855a620cd858373e857d73d61debe83648eb1a81bd7d0d77f6803c096f210aa3
+  data.tar.gz: 87c6f90f9f09c484c7db6a0b78a9f12a97f924d65b3839cd4eaa1310c41a3a036f71c0c64e7c00f34c57828d748acddecab1c457503c069cee0787ad2af132ad

data/.gitignore

@@ -0,0 +1,17 @@
+*.gem
+*.rbc
+.bundle
+.config
+.yardoc
+Gemfile.lock
+InstalledFiles
+_yardoc
+coverage
+doc/
+lib/bundler/man
+pkg
+rdoc
+spec/reports
+test/tmp
+test/version_tmp
+tmp

data/Gemfile

@@ -0,0 +1,3 @@
+source 'https://rubygems.org'
+
+gemspec

data/LICENSE.txt

@@ -0,0 +1,22 @@
+Copyright (c) 2013 Factlink, Inc
+
+MIT License
+
+Permission is hereby granted, free of charge, to any person obtaining
+a copy of this software and associated documentation files (the
+"Software"), to deal in the Software without restriction, including
+without limitation the rights to use, copy, modify, merge, publish,
+distribute, sublicense, and/or sell copies of the Software, and to
+permit persons to whom the Software is furnished to do so, subject to
+the following conditions:
+
+The above copyright notice and this permission notice shall be
+included in all copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
+EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
+MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND
+NONINFRINGEMENT. IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE
+LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION
+OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION
+WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.

data/README.md

@@ -0,0 +1,29 @@
+# UrlNormalizer
+
+TODO: Write a gem description
+
+## Installation
+
+Add this line to your application's Gemfile:
+
+    gem 'url_normalizer'
+
+And then execute:
+
+    $ bundle
+
+Or install it yourself as:
+
+    $ gem install url_normalizer
+
+## Usage
+
+TODO: Write usage instructions here
+
+## Contributing
+
+1. Fork it
+2. Create your feature branch (`git checkout -b my-new-feature`)
+3. Commit your changes (`git commit -am 'Add some feature'`)
+4. Push to the branch (`git push origin my-new-feature`)
+5. Create new Pull Request

data/Rakefile

@@ -0,0 +1 @@
+require "bundler/gem_tasks"

data/lib/url_normalizer.rb

@@ -0,0 +1,83 @@
+require "url_normalizer/version"
+require 'addressable/uri'
+require 'cgi'
+
+class UrlNormalizer
+  @@normalizer_for = Hash.new(UrlNormalizer)
+
+  def self.normalize_for domain
+    @@normalizer_for[domain] = self
+  end
+
+  def self.normalize url
+    url.sub!(/#(?!\!)[^#]*$/,'')
+
+    uri = Addressable::URI.parse(url)
+
+    @@normalizer_for[uri.host].new(uri).normalize
+  end
+
+  def initialize uri
+    @uri = uri
+  end
+
+  def normalize
+    uri = @uri
+
+    uri.query = clean_query(uri.query)
+    uri.normalize!
+
+    url = uri.to_s
+    url.sub(/\?$/,'')
+  end
+
+  def forbidden_uri_params
+    [:utm_source, :utm_content, :utm_medium, :utm_campaign]
+  end
+
+  def whitelisted_uri_params
+    nil
+  end
+
+  def clean_query query
+    return unless query
+
+    uri_params = CGI.parse(query)
+
+    forbidden_params = forbidden_uri_params.map(&:to_s)
+    if forbidden_params
+      uri_params.reject! {|k,v| forbidden_params.include? k}
+    end
+
+    allowed_params = whitelisted_uri_params.andand.map(&:to_s)
+    if allowed_params
+      uri_params.select! {|k,v| allowed_params.include? k}
+    end
+
+    build_query(uri_params)
+  end
+
+  def encode_component component
+    Addressable::URI.encode_component component
+  end
+
+  def build_query(params)
+    params.map do |name,values|
+      escaped_name = encode_component name
+      if values.length > 0
+        values.map do |value|
+          escaped_value = encode_component value
+          "#{escaped_name}=#{escaped_value}"
+        end
+      else
+        ["#{escaped_name}"]
+      end
+    end.flatten.join("&")
+  end
+end
+
+require_relative 'url_normalizer/proxy'
+require_relative 'url_normalizer/new_york_times'
+require_relative 'url_normalizer/think_progress'
+require_relative 'url_normalizer/linkedin'
+require_relative 'url_normalizer/newyorker'

data/lib/url_normalizer/linkedin.rb

@@ -0,0 +1,10 @@
+class UrlNormalizer
+  class LinkedIn < UrlNormalizer
+    normalize_for 'linkedin.com'
+    normalize_for 'www.linkedin.com'
+
+    def forbidden_uri_params
+      super + [:ref]
+    end
+  end
+end

data/lib/url_normalizer/new_york_times.rb

@@ -0,0 +1,15 @@
+class UrlNormalizer
+  class NewYorkTimes < UrlNormalizer
+    normalize_for 'nytimes.com'
+    normalize_for 'www.nytimes.com'
+
+    def forbidden_uri_params
+      super + [:_r]
+    end
+
+    def whitelisted_uri_params
+      [:pagewanted]
+    end
+
+  end
+end

data/lib/url_normalizer/newyorker.rb

@@ -0,0 +1,10 @@
+class UrlNormalizer
+  class Newyorker < UrlNormalizer
+    normalize_for 'newyorker.com'
+    normalize_for 'www.newyorker.com'
+
+    def forbidden_uri_params
+      super + [:mobify]
+    end
+  end
+end

data/lib/url_normalizer/proxy.rb

@@ -0,0 +1,16 @@
+require 'andand'
+
+class UrlNormalizer
+  class Proxy < UrlNormalizer
+    normalize_for 'fct.li'
+
+    def normalize
+      url = @uri.query && CGI.parse(@uri.query)['url'].andand[0]
+      if url
+        UrlNormalizer.normalize url
+      else
+        super
+      end
+    end
+  end
+end

data/lib/url_normalizer/think_progress.rb

@@ -0,0 +1,10 @@
+class UrlNormalizer
+  class ThinkProgress < UrlNormalizer
+    normalize_for 'thinkprogress.org'
+    normalize_for 'www.thinkprogress.org'
+
+    def forbidden_uri_params
+      super + [:mobile]
+    end
+  end
+end

data/lib/url_normalizer/version.rb

@@ -0,0 +1,3 @@
+class UrlNormalizer
+  VERSION = "0.0.1"
+end

data/spec/url_normalizer/linkedin_spec.rb

@@ -0,0 +1,15 @@
+require_relative '../../lib/url_normalizer.rb'
+require 'uri'
+require 'base64'
+
+describe UrlNormalizer do
+  describe ".normalize" do
+    def normalized url
+      UrlNormalizer.normalize(url)
+    end
+    it do
+      normalized('http://www.linkedin.com/today/post/article/20130131131416-174077701-lessons-from-my-bosses?ref=email').should ==
+                 'http://www.linkedin.com/today/post/article/20130131131416-174077701-lessons-from-my-bosses'
+    end
+  end
+end

data/spec/url_normalizer/new_york_times_spec.rb

@@ -0,0 +1,27 @@
+require File.expand_path('../../../lib/url_normalizer.rb', __FILE__)
+require 'uri'
+require 'base64'
+
+describe UrlNormalizer do
+  describe ".normalize" do
+    def normalized url
+      UrlNormalizer.normalize(url)
+    end
+
+    let(:base){'http://www.nytimes.com/2011/10/30/opinion/mona-simpsons-eulogy-for-steve-jobs.html'}
+
+    it { normalized(base + '?_r=1').should == base }
+    it { normalized(base + '?_r=1&utm_source=frank').should == base }
+
+    it do
+     normalized(base + '?pagewanted=all&_r=0').should ==
+                    base + '?pagewanted=all'
+    end
+
+    it do
+     normalized(base + '?pagewanted=2&_r=1utm_source=buffer&buffer_share=b0b26').should ==
+                    base + '?pagewanted=2'
+    end
+
+  end
+end

data/spec/url_normalizer/thinkprogress_spec.rb

@@ -0,0 +1,15 @@
+require File.expand_path('../../../lib/url_normalizer.rb', __FILE__)
+require 'uri'
+require 'base64'
+
+describe UrlNormalizer do
+  describe ".normalize" do
+    def normalized url
+      UrlNormalizer.normalize(url)
+    end
+    it do
+      normalized('http://thinkprogress.org/politics/2012/08/17/705401/how-paul-ryans-budget-would-devastate-social-programs-for-todays-lower-income-americans/?mobile=nc').should ==
+                   'http://thinkprogress.org/politics/2012/08/17/705401/how-paul-ryans-budget-would-devastate-social-programs-for-todays-lower-income-americans/'
+    end
+  end
+end

data/spec/url_normalizer_spec.rb

@@ -0,0 +1,96 @@
+require File.expand_path('../../lib/url_normalizer.rb', __FILE__)
+require 'uri'
+require 'base64'
+
+describe UrlNormalizer do
+  describe ".normalize" do
+    def normalized url
+      UrlNormalizer.normalize(url)
+    end
+
+    it { normalized('http://www.google.com/foo').should == 'http://www.google.com/foo' }
+    it { normalized('http://www.google.com/foo#bar').should == 'http://www.google.com/foo' }
+
+    ['utm_source', 'utm_medium', 'utm_source', 'utm_content', 'utm_campaign'].each do |strip_param|
+      it "should strip #{strip_param} if it is the only parameter" do
+        normalized("http://www.google.com/foo?#{strip_param}=bar").should == 'http://www.google.com/foo'
+      end
+      it "should strip #{strip_param} if there are other parameters" do
+        normalized("http://www.google.com/foo?#{strip_param}=bar&x=y").should == 'http://www.google.com/foo?x=y'
+      end
+    end
+
+    it { normalized( 'http://www.google.com/?x=y|z').should == 'http://www.google.com/?x=y%7Cz' }
+    it { normalized( 'http://www.google.com/?x=y|z').should == 'http://www.google.com/?x=y%7Cz' }
+
+    it { normalized( 'http://www.example.org/file\bier.png').should == 'http://www.example.org/file%5Cbier.png' }
+
+    it do
+     normalized( 'http://www.ikea.com/nl/nl/catalog/products/30101154/?icid=nl|ic|hp_main|smarteasyliving|ikea365').should ==
+                             'http://www.ikea.com/nl/nl/catalog/products/30101154/?icid=nl%7Cic%7Chp_main%7Csmarteasyliving%7Cikea365'
+    end
+
+    it do
+     normalized( 'http://www.amazon.de/s/url=search-alias%3Daps').should ==
+                            'http://www.amazon.de/s/url=search-alias=aps'
+    end
+
+    it do
+     normalized( 'http://www.amazon.de/s/?url=search-alias%3Daps').should ==
+                             'http://www.amazon.de/s/?url=search-alias=aps'
+    end
+
+    it do
+     normalized( 'http://www.newyorker.com/online/blogs/borowitzreport/2013/08/amazon-founder-says-he-clicked-on-washington-post-by-mistake.html?mobify=0&utm_source=buffer&utm_campaign=Buffer&utm_content=buffer1526d&utm_medium=twitter').should ==
+                             'http://www.newyorker.com/online/blogs/borowitzreport/2013/08/amazon-founder-says-he-clicked-on-washington-post-by-mistake.html'
+    end
+
+    describe "improvements" do
+
+      it { normalized( 'http://www.google.com/a[b]').should == 'http://www.google.com/a%5Bb%5D' } # [ and ] are not allowed according to RFC 2732 http://www.ietf.org/rfc/rfc2732.txt
+      it { normalized( 'http://www.google.com/foo?bar=bax|zuup').should == 'http://www.google.com/foo?bar=bax%7Czuup' }
+
+      describe "normalizing proxy urls" do
+        it { normalized( "http://fct.li/parse?url=http%3A%2F%2Fwww.google.com&factlinkModus=default").should == "http://www.google.com/" }
+      end
+      describe "it should work on normal proxy urls" do
+        it { normalized( "http://fct.li/").should == "http://fct.li/" }
+      end
+    end
+    it {normalized( 'http://www.example.com/ff/entry.asp?123').should == 'http://www.example.com/ff/entry.asp?123'}
+
+
+    it {normalized( 'http://example.com/foo?x=^y').should == 'http://example.com/foo?x=%5Ey'}
+    it {normalized( 'http://example.com/foo?x=%y').should == 'http://example.com/foo?x=%25y'}
+    it {normalized( 'http://example.com/foo?x={y}').should == 'http://example.com/foo?x=%7By%7D'}
+    it {normalized( 'http://example.org/search.php?a=%F6').should == 'http://example.org/search.php?a=%F6'}
+    def decode_utf8_b64(string)
+      URI.unescape(string)
+    end
+    it "should not bug with invalid encodings" do
+      normalized( decode_utf8_b64("http%3A%2F%2Fwww.mercuryserver.com%2Fforums%2Fshowthread.php%3F100800-Stephan-Bodzin-amp-Marc-Romboy-%2596-Live-Luna-Live-Tour-(Harry-Klein-M%25FCnchen)-%2596-23-04-2")).should ==
+               "http://www.mercuryserver.com/forums/showthread.php?100800-Stephan-Bodzin-amp-Marc-Romboy-%96-Live-Luna-Live-Tour-(Harry-Klein-M%FCnchen)-%96-23-04-2"
+    end
+    describe 'xss protection' do
+      it "url encodes < and >" do
+        url = 'http://hoi/<>'
+        expect(normalized(url)).to eq 'http://hoi/%3C%3E'
+      end
+
+      it "url encodes \"" do
+        url = 'http://hoi/"'
+        expect(normalized(url)).to eq 'http://hoi/%22'
+      end
+
+      it "encodes all explicit spacing to a space" do
+        url = "http://hoi/%20a%09b%0Ac%0Dd"
+        expect(normalized(url)).to eq  "http://hoi/%20a%09b%0Ac%0Dd"
+      end
+
+      it "leaves ' in the url (valid)" do
+        url = "http://hoi/'"
+        expect(normalized(url)).to eq "http://hoi/'"
+      end
+    end
+  end
+end

data/url_normalizer.gemspec

@@ -0,0 +1,27 @@
+# coding: utf-8
+lib = File.expand_path('../lib', __FILE__)
+$LOAD_PATH.unshift(lib) unless $LOAD_PATH.include?(lib)
+require 'url_normalizer/version'
+
+Gem::Specification.new do |spec|
+  spec.name          = "url_normalizer"
+  spec.version       = UrlNormalizer::VERSION
+  spec.authors       = ["Mark IJbema", "Martijn Russchen", "Tom de Vries"]
+  spec.email         = ["markijbema+url_normalizer@gmail.com"]
+  spec.description   = %q{Ruby gem to normalize urls}
+  spec.summary       = %q{Ruby gem to normalize urls}
+  spec.homepage      = ""
+  spec.license       = "MIT"
+
+  spec.files         = `git ls-files`.split($/)
+  spec.executables   = spec.files.grep(%r{^bin/}) { |f| File.basename(f) }
+  spec.test_files    = spec.files.grep(%r{^(test|spec|features)/})
+  spec.require_paths = ["lib"]
+
+  spec.add_dependency "andand"
+  spec.add_dependency "addressable"
+
+  spec.add_development_dependency "bundler", "~> 1.3"
+  spec.add_development_dependency "rake"
+  spec.add_development_dependency "rspec"
+end

metadata

@@ -0,0 +1,138 @@
+--- !ruby/object:Gem::Specification
+name: url_normalizer
+version: !ruby/object:Gem::Version
+  version: 0.0.1
+platform: ruby
+authors:
+- Mark IJbema
+- Martijn Russchen
+- Tom de Vries
+autorequire: 
+bindir: bin
+cert_chain: []
+date: 2013-09-29 00:00:00.000000000 Z
+dependencies:
+- !ruby/object:Gem::Dependency
+  name: andand
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: addressable
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :runtime
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: bundler
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.3'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        version: '1.3'
+- !ruby/object:Gem::Dependency
+  name: rake
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+- !ruby/object:Gem::Dependency
+  name: rspec
+  requirement: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+  type: :development
+  prerelease: false
+  version_requirements: !ruby/object:Gem::Requirement
+    requirements:
+    - - '>='
+      - !ruby/object:Gem::Version
+        version: '0'
+description: Ruby gem to normalize urls
+email:
+- markijbema+url_normalizer@gmail.com
+executables: []
+extensions: []
+extra_rdoc_files: []
+files:
+- .gitignore
+- Gemfile
+- LICENSE.txt
+- README.md
+- Rakefile
+- lib/url_normalizer.rb
+- lib/url_normalizer/linkedin.rb
+- lib/url_normalizer/new_york_times.rb
+- lib/url_normalizer/newyorker.rb
+- lib/url_normalizer/proxy.rb
+- lib/url_normalizer/think_progress.rb
+- lib/url_normalizer/version.rb
+- spec/url_normalizer/linkedin_spec.rb
+- spec/url_normalizer/new_york_times_spec.rb
+- spec/url_normalizer/thinkprogress_spec.rb
+- spec/url_normalizer_spec.rb
+- url_normalizer.gemspec
+homepage: ''
+licenses:
+- MIT
+metadata: {}
+post_install_message: 
+rdoc_options: []
+require_paths:
+- lib
+required_ruby_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+required_rubygems_version: !ruby/object:Gem::Requirement
+  requirements:
+  - - '>='
+    - !ruby/object:Gem::Version
+      version: '0'
+requirements: []
+rubyforge_project: 
+rubygems_version: 2.1.3
+signing_key: 
+specification_version: 4
+summary: Ruby gem to normalize urls
+test_files:
+- spec/url_normalizer/linkedin_spec.rb
+- spec/url_normalizer/new_york_times_spec.rb
+- spec/url_normalizer/thinkprogress_spec.rb
+- spec/url_normalizer_spec.rb
+has_rdoc: