RubyGems - metainspector - Versions diffs - 1.15.0 → 1.15.1 - Mend

metainspector 1.15.0 → 1.15.1

Files changed (7) hide show

data/lib/meta_inspector/scraper.rb +10 -9
data/lib/meta_inspector/version.rb +1 -1
data/meta_inspector.gemspec +1 -0
data/spec/fixtures/international.response +1 -0
data/spec/metainspector_spec.rb +46 -13
data/spec/redirections_spec.rb +3 -3
metadata +28 -12

data/lib/meta_inspector/scraper.rb CHANGED Viewed

@@ -2,6 +2,7 @@
 require 'open-uri'
 require 'open_uri_redirections'
+require 'addressable/uri'
 require 'nokogiri'
 require 'hashie/rash'
 require 'timeout'
@@ -22,7 +23,7 @@ module MetaInspector
     def initialize(url, options = {})
       options   = defaults.merge(options)
-      @url      = with_default_scheme(encode_url(url))
+      @url      = with_default_scheme(normalize_url(url))
       @scheme   = URI.parse(@url).scheme
       @host     = URI.parse(@url).host
       @root_url = "#{@scheme}://#{@host}/"
@@ -49,7 +50,7 @@ module MetaInspector
     # Links found on the page, as absolute URLs
     def links
-      @links ||= parsed_links.map{ |l| absolutify_url(unrelativize_url(l)) }.compact
+      @links ||= parsed_links.map{ |l| absolutify_url(unrelativize_url(l)) }.compact.uniq
     end
     # Internal links found on the page, as absolute URLs
@@ -217,9 +218,9 @@ module MetaInspector
       @errors << error
     end
-    # Encode url to deal with international characters
-    def encode_url(url)
-      URI.encode(url).to_s.gsub("%23", "#")
+    # Normalize url to deal with characters that should be encodes, add trailing slash, convert to downcase...
+    def normalize_url(url)
+      Addressable::URI.parse(url).normalize.to_s
     end
     # Adds 'http' as default scheme, if there if none
@@ -231,11 +232,11 @@ module MetaInspector
     # Respecting already absolute URLs like the ones starting with http:, ftp:, telnet:, mailto:, javascript: ...
     def absolutify_url(url)
       if url =~ /^\w*\:/i
-        encode_url(url)
+        normalize_url(url)
       else
-        URI.parse(root_url).merge(encode_url(url)).to_s
+        URI.parse(root_url).merge(normalize_url(url)).to_s
       end
-    rescue URI::InvalidURIError => e
+    rescue URI::InvalidURIError, Addressable::URI::InvalidURIError => e
       add_fatal_error "Link parsing exception: #{e.message}" and nil
     end
@@ -247,7 +248,7 @@ module MetaInspector
     # Extracts the host from a given URL
     def host_from_url(url)
       URI.parse(url).host
-    rescue URI::InvalidURIError, URI::InvalidComponentError => e
+    rescue URI::InvalidURIError, URI::InvalidComponentError, Addressable::URI::InvalidURIError => e
       add_fatal_error "Link parsing exception: #{e.message}" and nil
     end

data/lib/meta_inspector/version.rb CHANGED Viewed

@@ -1,5 +1,5 @@
 # -*- encoding: utf-8 -*-
 module MetaInspector
-  VERSION = "1.15.0"
+  VERSION = "1.15.1"
 end

data/meta_inspector.gemspec CHANGED Viewed

@@ -17,6 +17,7 @@ Gem::Specification.new do |gem|
   gem.add_dependency 'nokogiri', '~> 1.5'
   gem.add_dependency 'rash', '0.3.2'
   gem.add_dependency 'open_uri_redirections', '~> 0.1.0'
+  gem.add_dependency 'addressable', '~> 2.3.2'
   gem.add_development_dependency 'rspec', '2.12.0'
   gem.add_development_dependency 'fakeweb', '1.3.0'

data/spec/fixtures/international.response CHANGED Viewed

@@ -23,6 +23,7 @@ Cache-control: private
   <a href="/faqs#camión">FAQs camión</a>
   <a href="/search?q=camión">Search camión</a>
   <a href="/search?q=españa#top">Search España at top</a>
+  <a href="/index.php?q=españa&url=aHR0zZQ%3D%3D&amp;cntnt01pageid=21">A link with an encoded param, %3D should not be double-escaped</a>
   <h1>External links:</h1>
   <a href="http://example.com/españa.asp">España</a>

data/spec/metainspector_spec.rb CHANGED Viewed

@@ -5,7 +5,7 @@ require File.join(File.dirname(__FILE__), "/spec_helper")
 describe MetaInspector do
   describe 'Initialization' do
     it 'should accept an URL with a scheme' do
-      MetaInspector.new('http://pagerankalert.com').url.should == 'http://pagerankalert.com'
+      MetaInspector.new('http://pagerankalert.com').url.should == 'http://pagerankalert.com/'
     end
     it "should use http:// as a default scheme" do
@@ -114,7 +114,7 @@ describe MetaInspector do
   end
   describe 'Doing a basic scrape from passed url html' do
     before(:each) do
       @m = MetaInspector.new("http://cnn.com", :document => "<html><head><title>Hello From Passed Html</title><a href='/hello'>Hello link</a></head><body></body></html>")
     end
@@ -147,7 +147,7 @@ describe MetaInspector do
                            "http://pagerankalert.com/users/sign_up",
                            "http://pagerankalert.com/users/sign_in",
                            "mailto:pagerankalert@gmail.com",
-                           "http://pagerankalert.posterous.com",
+                           "http://pagerankalert.posterous.com/",
                            "http://twitter.com/pagerankalert",
                            "http://twitter.com/share" ]
     end
@@ -161,7 +161,7 @@ describe MetaInspector do
     it "should get correct absolute links for external pages" do
       @m.external_links.should == [ "mailto:pagerankalert@gmail.com",
-                           "http://pagerankalert.posterous.com",
+                           "http://pagerankalert.posterous.com/",
                            "http://twitter.com/pagerankalert",
                            "http://twitter.com/share" ]
     end
@@ -178,13 +178,14 @@ describe MetaInspector do
     end
     describe "links with international characters" do
-      it "should get correct absolute links, encoding the URLs as needed but respecting # and ?" do
+      it "should get correct absolute links, encoding the URLs as needed" do
         m = MetaInspector.new('http://international.com')
         m.links.should == [ "http://international.com/espa%C3%B1a.asp",
                             "http://international.com/roman%C3%A9e",
                             "http://international.com/faqs#cami%C3%B3n",
                             "http://international.com/search?q=cami%C3%B3n",
                             "http://international.com/search?q=espa%C3%B1a#top",
+                            "http://international.com/index.php?q=espa%C3%B1a&url=aHR0zZQ==&cntnt01pageid=21",
                             "http://example.com/espa%C3%B1a.asp",
                             "http://example.com/roman%C3%A9e",
                             "http://example.com/faqs#cami%C3%B3n",
@@ -199,7 +200,8 @@ describe MetaInspector do
                                        "http://international.com/roman%C3%A9e",
                                        "http://international.com/faqs#cami%C3%B3n",
                                        "http://international.com/search?q=cami%C3%B3n",
-                                       "http://international.com/search?q=espa%C3%B1a#top"]
+                                       "http://international.com/search?q=espa%C3%B1a#top",
+                                       "http://international.com/index.php?q=espa%C3%B1a&url=aHR0zZQ==&cntnt01pageid=21"]
         end
         it "should not crash when processing malformed hrefs" do
@@ -225,8 +227,7 @@ describe MetaInspector do
           m = MetaInspector.new('http://example.com/malformed_href')
           expect {
             m.external_links.should == ["skype:joeuser?call", "telnet://telnet.cdrom.com",
-                                        "javascript:alert('ok');", "javascript://",
-                                        "mailto:email(at)example.com"]
+                                        "javascript:alert('ok');", "mailto:email(at)example.com"]
             m.should_not be_ok
           }.to_not raise_error
         end
@@ -257,8 +258,7 @@ describe MetaInspector do
     it "should get the links" do
       @m.links.sort.should == [
-                                "FTP://FTP.CDROM.COM",
-                                "ftp://ftp.cdrom.com",
+                                "ftp://ftp.cdrom.com/",
                                 "javascript:alert('hey');",
                                 "mailto:user@example.com",
                                 "skype:joeuser?call",
@@ -275,12 +275,12 @@ describe MetaInspector do
     it "should convert protocol-relative links to http" do
       @m_http.links.should include('http://protocol-relative.com/contact')
-      @m_http.links.should include('http://yahoo.com')
+      @m_http.links.should include('http://yahoo.com/')
     end
     it "should convert protocol-relative links to https" do
       @m_https.links.should include('https://protocol-relative.com/contact')
-      @m_https.links.should include('https://yahoo.com')
+      @m_https.links.should include('https://yahoo.com/')
     end
   end
@@ -355,7 +355,40 @@ describe MetaInspector do
   describe 'to_hash' do
     it "should return a hash with all the values set" do
       @m = MetaInspector.new('http://pagerankalert.com')
-      @m.to_hash.should == {"title"=>"PageRankAlert.com :: Track your PageRank changes", "url"=>"http://pagerankalert.com", "meta"=>{"name"=>{"robots"=>"all,follow", "csrf_param"=>"authenticity_token", "description"=>"Track your PageRank(TM) changes and receive alerts by email", "keywords"=>"pagerank, seo, optimization, google", "csrf_token"=>"iW1/w+R8zrtDkhOlivkLZ793BN04Kr3X/pS+ixObHsE="}, "property"=>{}}, "images"=>["http://pagerankalert.com/images/pagerank_alert.png?1305794559"], "charset"=>"utf-8", "feed"=>"http://feeds.feedburner.com/PageRankAlert", "links"=>["http://pagerankalert.com/", "http://pagerankalert.com/es?language=es", "http://pagerankalert.com/users/sign_up", "http://pagerankalert.com/users/sign_in", "mailto:pagerankalert@gmail.com", "http://pagerankalert.posterous.com", "http://twitter.com/pagerankalert", "http://twitter.com/share"], "internal_links"=>["http://pagerankalert.com/", "http://pagerankalert.com/es?language=es", "http://pagerankalert.com/users/sign_up", "http://pagerankalert.com/users/sign_in"], "external_links" => ["mailto:pagerankalert@gmail.com", "http://pagerankalert.posterous.com", "http://twitter.com/pagerankalert", "http://twitter.com/share"], "content_type" => "text/html"}
+      @m.to_hash.should == {
+                              "url"             =>"http://pagerankalert.com/",
+                              "title"           =>"PageRankAlert.com :: Track your PageRank changes",
+                              "links"           => ["http://pagerankalert.com/",
+                                                    "http://pagerankalert.com/es?language=es",
+                                                    "http://pagerankalert.com/users/sign_up",
+                                                    "http://pagerankalert.com/users/sign_in",
+                                                    "mailto:pagerankalert@gmail.com",
+                                                    "http://pagerankalert.posterous.com/",
+                                                    "http://twitter.com/pagerankalert",
+                                                    "http://twitter.com/share"],
+                              "internal_links"  => ["http://pagerankalert.com/",
+                                                    "http://pagerankalert.com/es?language=es",
+                                                    "http://pagerankalert.com/users/sign_up",
+                                                    "http://pagerankalert.com/users/sign_in"],
+                              "external_links"  => ["mailto:pagerankalert@gmail.com",
+                                                    "http://pagerankalert.posterous.com/",
+                                                    "http://twitter.com/pagerankalert",
+                                                    "http://twitter.com/share"],
+                              "images"          => ["http://pagerankalert.com/images/pagerank_alert.png?1305794559"],
+                              "charset"         => "utf-8",
+                              "feed"            => "http://feeds.feedburner.com/PageRankAlert",
+                              "content_type"    =>"text/html",
+                              "meta"            => {
+                                                      "name" => {
+                                                                  "description"=> "Track your PageRank(TM) changes and receive alerts by email",
+                                                                  "keywords" => "pagerank, seo, optimization, google",
+                                                                  "robots" => "all,follow",
+                                                                  "csrf_param" => "authenticity_token",
+                                                                  "csrf_token" => "iW1/w+R8zrtDkhOlivkLZ793BN04Kr3X/pS+ixObHsE="
+                                                                },
+                                                      "property"=>{}
+                                                   }
+                           }
     end
   end

data/spec/redirections_spec.rb CHANGED Viewed

@@ -9,7 +9,7 @@ describe MetaInspector do
         m = MetaInspector.new("http://facebook.com")
         m.title.should be_nil
         m.should_not be_ok
-        m.errors.first.should == "Scraping exception: redirection forbidden: http://facebook.com -> https://www.facebook.com/"
+        m.errors.first.should == "Scraping exception: redirection forbidden: http://facebook.com/ -> https://www.facebook.com/"
       end
       it "allows safe redirections when :allow_redirections => :safe" do
@@ -30,14 +30,14 @@ describe MetaInspector do
         m = MetaInspector.new("https://unsafe-facebook.com")
         m.title.should be_nil
         m.should_not be_ok
-        m.errors.first.should == "Scraping exception: redirection forbidden: https://unsafe-facebook.com -> http://unsafe-facebook.com/"
+        m.errors.first.should == "Scraping exception: redirection forbidden: https://unsafe-facebook.com/ -> http://unsafe-facebook.com/"
       end
       it "disallows unsafe redirections when :allow_redirections => :safe" do
         m = MetaInspector.new("https://unsafe-facebook.com", :allow_redirections => :safe)
         m.title.should be_nil
         m.should_not be_ok
-        m.errors.first.should == "Scraping exception: redirection forbidden: https://unsafe-facebook.com -> http://unsafe-facebook.com/"
+        m.errors.first.should == "Scraping exception: redirection forbidden: https://unsafe-facebook.com/ -> http://unsafe-facebook.com/"
       end
       it "allows unsafe redirections when :allow_redirections => :all" do

metadata CHANGED Viewed

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification
 name: metainspector
 version: !ruby/object:Gem::Version
-  hash: 43
+  hash: 41
   prerelease:
   segments:
   - 1
   - 15
-  - 0
-  version: 1.15.0
+  - 1
+  version: 1.15.1
 platform: ruby
 authors:
 - Jaime Iniesta
@@ -15,7 +15,7 @@ autorequire:
 bindir: bin
 cert_chain: []
-date: 2013-01-19 00:00:00 Z
+date: 2013-02-06 00:00:00 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: nokogiri
@@ -65,9 +65,25 @@ dependencies:
   type: :runtime
   version_requirements: *id003
 - !ruby/object:Gem::Dependency
-  name: rspec
+  name: addressable
   prerelease: false
   requirement: &id004 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        hash: 7
+        segments:
+        - 2
+        - 3
+        - 2
+        version: 2.3.2
+  type: :runtime
+  version_requirements: *id004
+- !ruby/object:Gem::Dependency
+  name: rspec
+  prerelease: false
+  requirement: &id005 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - "="
@@ -79,11 +95,11 @@ dependencies:
         - 0
         version: 2.12.0
   type: :development
-  version_requirements: *id004
+  version_requirements: *id005
 - !ruby/object:Gem::Dependency
   name: fakeweb
   prerelease: false
-  requirement: &id005 !ruby/object:Gem::Requirement
+  requirement: &id006 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - "="
@@ -95,11 +111,11 @@ dependencies:
         - 0
         version: 1.3.0
   type: :development
-  version_requirements: *id005
+  version_requirements: *id006
 - !ruby/object:Gem::Dependency
   name: awesome_print
   prerelease: false
-  requirement: &id006 !ruby/object:Gem::Requirement
+  requirement: &id007 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - "="
@@ -111,11 +127,11 @@ dependencies:
         - 0
         version: 1.1.0
   type: :development
-  version_requirements: *id006
+  version_requirements: *id007
 - !ruby/object:Gem::Dependency
   name: rake
   prerelease: false
-  requirement: &id007 !ruby/object:Gem::Requirement
+  requirement: &id008 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -127,7 +143,7 @@ dependencies:
         - 3
         version: 10.0.3
   type: :development
-  version_requirements: *id007
+  version_requirements: *id008
 description: MetaInspector lets you scrape a web page and get its title, charset, link and meta tags
 email:
 - jaimeiniesta@gmail.com