postrank-uri 1.0.8 → 1.0.9

data/lib/postrank-uri.rb

@@ -6,6 +6,21 @@ require 'digest/md5'
 require 'nokogiri'
 require 'yaml'
 
+module Addressable
+  class URI
+    def domain
+      begin
+        dp = Domainatrix.parse(self)
+      rescue
+        return nil
+      end
+
+      dom = dp.public_suffix
+      dom = dp.domain.downcase + "." + dom unless dp.domain.empty?
+    end
+  end
+end
+
 module PostRank
   module URI
 
@@ -87,7 +102,7 @@ module PostRank
       urls = []
       Nokogiri.HTML(text).search('a').each do |a|
         begin
-          url = clean(a.attr('href'), false)
+          url = clean(a.attr('href'), :raw => true)
           if url.host.empty?
             next if host.nil?
             url.host = host
@@ -113,13 +128,13 @@ module PostRank
       end
     end
 
-    def clean(uri, string = true)
+    def clean(uri, opts = {})
       uri = normalize(c18n(unescape(uri)))
-      string ? uri.to_s : uri
+      opts[:raw] ? uri : uri.to_s
     end
 
-    def hash(uri)
-      Digest::MD5.hexdigest(clean(uri))
+    def hash(uri, opts = {})
+      Digest::MD5.hexdigest(opts[:skip_clean] ? uri : clean(uri))
     end
 
     def normalize(uri)
@@ -158,7 +173,7 @@ module PostRank
         embedded = uri.query_values['u']
       end
 
-      uri = clean(embedded, false) if embedded
+      uri = clean(embedded, :raw => true) if embedded
       uri
     end
 

data/lib/postrank-uri/version.rb

@@ -1,5 +1,5 @@
 module PostRank
   module URI
-    VERSION = "1.0.8"
+    VERSION = "1.0.9"
   end
 end

data/spec/postrank-uri_spec.rb

@@ -177,8 +177,8 @@ describe PostRank::URI do
   end
 
   context "hash" do
-    def h(uri)
-      PostRank::URI.hash(uri)
+    def h(uri, opts = {})
+      PostRank::URI.hash(uri, opts)
     end
 
     it "should compute MD5 hash of the normalized URI" do
@@ -189,6 +189,13 @@ describe PostRank::URI do
       h('everburning.com/feed/post/1').should == hash
       h('everburning.com/feed/post/1/').should == hash
     end
+
+    it "should not clean the URI if requested" do
+      hash = '55fae8910d312b7878a3201ed653b881'
+
+      h('http://everburning.com/feed/post/1', :skip_clean => true).should == hash
+      h('everburning.com/feed/post/1', :skip_clean => true).should_not == hash
+    end
   end
 
   context "extract" do
@@ -265,6 +272,31 @@ describe PostRank::URI do
         i.last.should == 'link to stuff'
       end
     end
-  end
 
+    context 'domain extraction' do
+      url_list = {"http://alex.pages.example.com" => "example.com",
+                  "alex.pages.example.com" => "example.com",
+                  "http://example.com/2011/04/01/blah" => "example.com",
+                  "http://example.com" => "example.com",
+                  "example.com" => "example.com",
+                  "ExampLe.com" => "example.com",
+                  "ExampLe.com:3000" => "example.com",
+                  "http://alex.pages.example.COM" => "example.com",
+                  "http://www.example.ag.it/2011/04/01/blah" => "example.ag.it",
+                  "ftp://www.example.com/2011/04/01/blah" => nil,
+                  "http://com" => nil,
+                  "http://alex.pages.examplecom" => nil,
+                  "example" => nil,
+                  "http://127.0.0.1" => nil,
+                  "localhost" => nil
+                  }
+
+       url_list.each_pair do |url, expected_result|
+         it "should extract #{expected_result.inspect} from #{url}" do
+            u = PostRank::URI.clean(url, :raw => true)
+            u.domain.should == expected_result
+         end
+       end
+    end
+  end
 end

metadata

@@ -2,7 +2,7 @@
 name: postrank-uri
 version: !ruby/object:Gem::Version 
   prerelease: 
-  version: 1.0.8
+  version: 1.0.9
 platform: ruby
 authors: 
 - Ilya Grigorik
@@ -10,7 +10,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2011-03-31 00:00:00 -04:00
+date: 2011-04-08 00:00:00 -04:00
 default_executable: 
 dependencies: 
 - !ruby/object:Gem::Dependency