postrank-uri 1.0.17 → 1.0.18

checksums.yaml

@@ -0,0 +1,7 @@
+---
+SHA1:
+  metadata.gz: 1f9a235d0c2287732278b672997027c7d4093d0f
+  data.tar.gz: 8c91e4b1c787c43e6ea2ccc51593be3809746fa7
+SHA512:
+  metadata.gz: c702922f05473c762d7223777eb48641db8c4b0e156539c729433f3aa9174b010bce707801ee8c7ec310e4181edb3402c77131d3a66b0bdc02e99f87c1d9982a
+  data.tar.gz: 06040dead1c6348febb64211e2787216c287c87baa09555947e03a4bea09b7293ed33faed08851e28aaa2ff08389d121a48746cc6a6f3af6c7e023f75db1917a

data/README.md

@@ -14,7 +14,7 @@ In a nutshell, we need to make sure that creative cases like the ones below all
 ## API
 
 - **PostRank::URI.extract(text)** - Detect URIs in text, discard bad TLD's
-- **PostRank::URI.clean(uri)** - Unescape, normalize, apply c18n filters - 95% use case.
+- **PostRank::URI.clean(uri)** - Unescape, normalize, apply c14n filters - 95% use case.
 
 - **PostRank::URI.normalize(uri)** - Apply RFC normalization rules, discard extra path characters, drop anchors
 - **PostRank::URI.unescape(uri)** - Unescape URI entities, handle +/%20's, etc
@@ -33,8 +33,8 @@ In a nutshell, we need to make sure that creative cases like the ones below all
         [0] "http://link.to/?a=b"
     ]
 
-## C18N
+## C14N
 
-As part of URI canonicalization the library will remove common tracking parameters from Google Analytics and several other providers. Beyond that, host-specific rules are also applied. For example, nytimes.com likes to add a 'partner' query parameter for tracking purposes, but which has no effect on the content - hence, it is removed from the URI. For full list, see the c18n.yml file.
+As part of URI canonicalization the library will remove common tracking parameters from Google Analytics and several other providers. Beyond that, host-specific rules are also applied. For example, nytimes.com likes to add a 'partner' query parameter for tracking purposes, but which has no effect on the content - hence, it is removed from the URI. For full list, see the c14n.yml file.
 
 Detecting "duplicate URLs" is a hard problem to solve (expensive in all senses), instead we are compiling a manually assembled database. If you find cases which are missing, please do report them, or send us a pull request!

data/lib/postrank-uri.rb

@@ -1,4 +1,3 @@
-# -*- encoding: utf-8 -*-
 
 require 'addressable/uri'
 require 'digest/md5'
@@ -34,11 +33,11 @@ end
 module PostRank
   module URI
 
-    c18ndb = YAML.load_file(File.dirname(__FILE__) + '/postrank-uri/c18n.yml')
+    c14ndb = YAML.load_file(File.dirname(__FILE__) + '/postrank-uri/c14n.yml')
 
-    C18N = {}
-    C18N[:global] = c18ndb[:all].freeze
-    C18N[:hosts]  = c18ndb[:hosts].inject({}) {|h,(k,v)| h[/#{Regexp.escape(k)}$/.freeze] = v; h}
+    C14N = {}
+    C14N[:global] = c14ndb[:all].freeze
+    C14N[:hosts]  = c14ndb[:hosts].inject({}) {|h,(k,v)| h[/#{Regexp.escape(k)}$/.freeze] = v; h}
 
     URIREGEX = {}
     URIREGEX[:protocol] = /https?:\/\//i
@@ -130,13 +129,15 @@ module PostRank
     end
 
     def unescape(uri)
-      uri.tr('+', ' ').gsub(URIREGEX[:unescape]) do
+      u = parse(uri)
+      u.query = u.query.tr('+', ' ') if u.query
+      u.to_s.gsub(URIREGEX[:unescape]) do
         [$1.delete('%')].pack('H*')
       end
     end
 
     def clean(uri, opts = {})
-      uri = normalize(c18n(unescape(uri), opts))
+      uri = normalize(c14n(unescape(uri), opts))
       opts[:raw] ? uri : uri.to_s
     end
 
@@ -153,13 +154,13 @@ module PostRank
       u
     end
 
-    def c18n(uri, opts = {})
+    def c14n(uri, opts = {})
       u = parse(uri, opts)
       u = embedded(u)
 
       if q = u.query_values(Array)
-        q.delete_if { |k,v| C18N[:global].include?(k) }
-        q.delete_if { |k,v| C18N[:hosts].find {|r,p| u.host =~ r && p.include?(k) } }
+        q.delete_if { |k,v| C14N[:global].include?(k) }
+        q.delete_if { |k,v| C14N[:hosts].find {|r,p| u.host =~ r && p.include?(k) } }
       end
       u.query_values = q
 
@@ -224,7 +225,7 @@ module PostRank
       cleaned_uri = clean(uri, :raw => true)
 
       if host = cleaned_uri.host
-        is_valid = PublicSuffix.valid?(host)
+        is_valid = PublicSuffix.valid?(Addressable::IDNA.to_unicode(host))
       end
 
       is_valid

data/lib/postrank-uri/{c18n.yml → c14n.yml}

@@ -7,6 +7,8 @@
 - utm_campaign  # Google Analytics: campaign name
 - sms_ss        # addthis.com tracker
 - awesm         # awe.sm tracker
+- xtor          # AT Internet tracker
+- PHPSESSID     # Legacy PHP session identifier
 
 :hosts:
   nytimes.com:

data/lib/postrank-uri/version.rb

@@ -1,5 +1,5 @@
 module PostRank
   module URI
-    VERSION = "1.0.17"
+    VERSION = "1.0.18"
   end
 end

data/postrank-uri.gemspec

@@ -9,14 +9,14 @@ Gem::Specification.new do |s|
   s.authors     = ["Ilya Grigorik"]
   s.email       = ["ilya@igvita.com"]
   s.homepage    = "http://github.com/postrank-labs/postrank-uri"
-  s.summary     = "URI normalization, c18n, escaping, and extraction"
+  s.summary     = "URI normalization, c14n, escaping, and extraction"
   s.description = s.summary
 
   s.rubyforge_project = "postrank-uri"
 
   s.add_dependency "addressable",   "~> 2.3.0"
   s.add_dependency "public_suffix", "~> 1.1.3"
-  s.add_dependency "nokogiri",      "~> 1.5.5"
+  s.add_dependency "nokogiri",      "~> 1.6.1"
 
   s.add_development_dependency "rspec"
 

data/spec/postrank-uri_spec.rb

@@ -29,15 +29,19 @@ describe PostRank::URI do
       # See http://tools.ietf.org/html/rfc3986#section-2.3
 
       it "should unescape PostRank::URI with spaces encoded as '+'" do
-        PostRank::URI.unescape('id=+1').should == 'id= 1'
+        PostRank::URI.unescape('?id=+1').should == '?id= 1'
       end
 
       it "should unescape PostRank::URI with spaces encoded as '+'" do
-        PostRank::URI.unescape('id%3D+1').should == 'id= 1'
+        PostRank::URI.unescape('?id%3D+1').should == '?id= 1'
       end
 
       it "should unescape PostRank::URI with spaces encoded as %20" do
-        PostRank::URI.unescape('id=%201').should == 'id= 1'
+        PostRank::URI.unescape('?id=%201').should == '?id= 1'
+      end
+
+      it "should not unescape '+' to spaces in paths" do
+        PostRank::URI.unescape('/foo+bar?id=foo+bar').should == '/foo+bar?id=foo bar'
       end
     end
 
@@ -100,7 +104,7 @@ describe PostRank::URI do
 
   context "canonicalization" do
     def c(uri)
-      PostRank::URI.c18n(uri).to_s
+      PostRank::URI.c14n(uri).to_s
     end
 
     context "query parameters" do
@@ -122,6 +126,12 @@ describe PostRank::URI do
         c('igvita.com/?id=a&utm_source=a&awesm=b').should == 'http://igvita.com/?id=a'
         c('igvita.com/?id=a&sms_ss=a').should == 'http://igvita.com/?id=a'
       end
+
+      it "should remove PHPSESSID parameter" do
+        c('http://www.nachi.org/forum?PHPSESSID=9ee2fb10b7274ef2b15d1d4006b8c8dd').should == 'http://www.nachi.org/forum?'
+        c('http://www.nachi.org/forum/?PHPSESSID=9ee2fb10b7274ef2b15d1d4006b8c8dd').should == 'http://www.nachi.org/forum/?'
+        c('http://www.nachi.org/forum?id=123&PHPSESSID=9ee2fb10b7274ef2b15d1d4006b8c8dd').should == 'http://www.nachi.org/forum?id=123'
+      end
     end
 
     context "hashbang" do
@@ -162,7 +172,7 @@ describe PostRank::URI do
       PostRank::URI.clean(uri)
     end
 
-    it "should unescape, c18n and normalize" do
+    it "should unescape, c14n and normalize" do
       c('http://igvita.com/?id=1').should == 'http://igvita.com/?id=1'
       c('igvita.com/?id=1').should == 'http://igvita.com/?id=1'
 
@@ -179,7 +189,7 @@ describe PostRank::URI do
     end
 
     it "should clean host specific parameters" do
-      YAML.load_file('spec/c18n_hosts.yml').each do |orig, clean|
+      YAML.load_file('spec/c14n_hosts.yml').each do |orig, clean|
         c(orig).should == clean
       end
     end
@@ -350,5 +360,13 @@ describe PostRank::URI do
     it 'marks www.test.com as valid' do
       PostRank::URI.valid?('http://www.test.com').should be_true
     end
+
+    it 'marks Unicode domain as valid (NOTE: works only with a scheme)' do
+      PostRank::URI.valid?('http://президент.рф').should be_true
+    end
+
+    it 'marks punycode domain domain as valid' do
+      PostRank::URI.valid?('xn--d1abbgf6aiiy.xn--p1ai').should be_true
+    end
   end
 end

metadata

@@ -1,20 +1,18 @@
 --- !ruby/object:Gem::Specification
 name: postrank-uri
 version: !ruby/object:Gem::Version
-  version: 1.0.17
-  prerelease: 
+  version: 1.0.18
 platform: ruby
 authors:
 - Ilya Grigorik
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2012-10-07 00:00:00.000000000 Z
+date: 2014-04-10 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: addressable
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
@@ -22,7 +20,6 @@ dependencies:
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
@@ -30,7 +27,6 @@ dependencies:
 - !ruby/object:Gem::Dependency
   name: public_suffix
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
@@ -38,7 +34,6 @@ dependencies:
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
@@ -46,36 +41,32 @@ dependencies:
 - !ruby/object:Gem::Dependency
   name: nokogiri
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: 1.5.5
+        version: 1.6.1
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
     - - ~>
       - !ruby/object:Gem::Version
-        version: 1.5.5
+        version: 1.6.1
 - !ruby/object:Gem::Dependency
   name: rspec
   requirement: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - '>='
       - !ruby/object:Gem::Version
         version: '0'
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
-    none: false
     requirements:
-    - - ! '>='
+    - - '>='
       - !ruby/object:Gem::Version
         version: '0'
-description: URI normalization, c18n, escaping, and extraction
+description: URI normalization, c14n, escaping, and extraction
 email:
 - ilya@igvita.com
 executables: []
@@ -88,38 +79,37 @@ files:
 - README.md
 - Rakefile
 - lib/postrank-uri.rb
-- lib/postrank-uri/c18n.yml
+- lib/postrank-uri/c14n.yml
 - lib/postrank-uri/version.rb
 - postrank-uri.gemspec
-- spec/c18n_hosts.yml
+- spec/c14n_hosts.yml
 - spec/helper.rb
 - spec/postrank-uri_spec.rb
 homepage: http://github.com/postrank-labs/postrank-uri
 licenses: []
+metadata: {}
 post_install_message: 
 rdoc_options: []
 require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
-  none: false
   requirements:
-  - - ! '>='
+  - - '>='
     - !ruby/object:Gem::Version
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
-  none: false
   requirements:
-  - - ! '>='
+  - - '>='
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
 rubyforge_project: postrank-uri
-rubygems_version: 1.8.24
+rubygems_version: 2.0.6
 signing_key: 
-specification_version: 3
-summary: URI normalization, c18n, escaping, and extraction
+specification_version: 4
+summary: URI normalization, c14n, escaping, and extraction
 test_files:
-- spec/c18n_hosts.yml
+- spec/c14n_hosts.yml
 - spec/helper.rb
 - spec/postrank-uri_spec.rb
 has_rdoc: