RubyGems - metainspector - Versions diffs - 1.15.0 → 1.15.1 - Mend

metainspector 1.15.0 → 1.15.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.

Files changed (7) hide show

data/lib/meta_inspector/scraper.rb +10 -9
data/lib/meta_inspector/version.rb +1 -1
data/meta_inspector.gemspec +1 -0
data/spec/fixtures/international.response +1 -0
data/spec/metainspector_spec.rb +46 -13
data/spec/redirections_spec.rb +3 -3
metadata +28 -12

data/lib/meta_inspector/scraper.rb CHANGED Viewed

@@ -2,6 +2,7 @@
 require 'open-uri'
 require 'open_uri_redirections'
+require 'addressable/uri'
 require 'nokogiri'
 require 'hashie/rash'
 require 'timeout'
@@ -22,7 +23,7 @@ module MetaInspector
     def initialize(url, options = {})
       options   = defaults.merge(options)
-      @url      = with_default_scheme(encode_url(url))
+      @url      = with_default_scheme(normalize_url(url))
       @scheme   = URI.parse(@url).scheme
       @host     = URI.parse(@url).host
       @root_url = "#{@scheme}://#{@host}/"
@@ -49,7 +50,7 @@ module MetaInspector
     # Links found on the page, as absolute URLs
     def links
-      @links ||= parsed_links.map{ |l| absolutify_url(unrelativize_url(l)) }.compact
+      @links ||= parsed_links.map{ |l| absolutify_url(unrelativize_url(l)) }.compact.uniq
     end
     # Internal links found on the page, as absolute URLs
@@ -217,9 +218,9 @@ module MetaInspector
       @errors << error
     end
-    # Encode url to deal with international characters
-    def encode_url(url)
-      URI.encode(url).to_s.gsub("%23", "#")
+    # Normalize url to deal with characters that should be encodes, add trailing slash, convert to downcase...
+    def normalize_url(url)
+      Addressable::URI.parse(url).normalize.to_s
     end
     # Adds 'http' as default scheme, if there if none
@@ -231,11 +232,11 @@ module MetaInspector
     # Respecting already absolute URLs like the ones starting with http:, ftp:, telnet:, mailto:, javascript: ...
     def absolutify_url(url)
       if url =~ /^\w*\:/i
-        encode_url(url)
+        normalize_url(url)
       else
-        URI.parse(root_url).merge(encode_url(url)).to_s
+        URI.parse(root_url).merge(normalize_url(url)).to_s
       end
-    rescue URI::InvalidURIError => e
+    rescue URI::InvalidURIError, Addressable::URI::InvalidURIError => e
       add_fatal_error "Link parsing exception: #{e.message}" and nil
     end
@@ -247,7 +248,7 @@ module MetaInspector
     # Extracts the host from a given URL
     def host_from_url(url)
       URI.parse(url).host
-    rescue URI::InvalidURIError, URI::InvalidComponentError => e
+    rescue URI::InvalidURIError, URI::InvalidComponentError, Addressable::URI::InvalidURIError => e
       add_fatal_error "Link parsing exception: #{e.message}" and nil
     end

data/lib/meta_inspector/version.rb CHANGED Viewed

@@ -1,5 +1,5 @@
 # -*- encoding: utf-8 -*-
 module MetaInspector
-  VERSION = "1.15.0"
+  VERSION = "1.15.1"
 end

data/meta_inspector.gemspec CHANGED Viewed

@@ -17,6 +17,7 @@ Gem::Specification.new do |gem|
   gem.add_dependency 'nokogiri', '~> 1.5'
   gem.add_dependency 'rash', '0.3.2'
   gem.add_dependency 'open_uri_redirections', '~> 0.1.0'
+  gem.add_dependency 'addressable', '~> 2.3.2'
   gem.add_development_dependency 'rspec', '2.12.0'
   gem.add_development_dependency 'fakeweb', '1.3.0'

data/spec/fixtures/international.response CHANGED Viewed

@@ -23,6 +23,7 @@ Cache-control: private
   <a href="/faqs#camión">FAQs camión</a>
   <a href="/search?q=camión">Search camión</a>
   <a href="/search?q=españa#top">Search España at top</a>
+  <a href="/index.php?q=españa&url=aHR0zZQ%3D%3D&amp;cntnt01pageid=21">A link with an encoded param, %3D should not be double-escaped</a>
   <h1>External links:</h1>
   <a href="http://example.com/españa.asp">España</a>

data/spec/metainspector_spec.rb CHANGED Viewed

@@ -5,7 +5,7 @@ require File.join(File.dirname(__FILE__), "/spec_helper")
 describe MetaInspector do
   describe 'Initialization' do
     it 'should accept an URL with a scheme' do
-      MetaInspector.new('http://pagerankalert.com').url.should == 'http://pagerankalert.com'
+      MetaInspector.new('http://pagerankalert.com').url.should == 'http://pagerankalert.com/'
     end
     it "should use http:// as a default scheme" do
@@ -114,7 +114,7 @@ describe MetaInspector do
   end
   describe 'Doing a basic scrape from passed url html' do
     before(:each) do
       @m = MetaInspector.new("http://cnn.com", :document => "<html><head><title>Hello From Passed Html</title><a href='/hello'>Hello link</a></head><body></body></html>")
     end
@@ -147,7 +147,7 @@ describe MetaInspector do
                            "http://pagerankalert.com/users/sign_up",
                            "http://pagerankalert.com/users/sign_in",
                            "mailto:pagerankalert@gmail.com",
-                           "http://pagerankalert.posterous.com",
+                           "http://pagerankalert.posterous.com/",
                            "http://twitter.com/pagerankalert",
                            "http://twitter.com/share" ]
     end
@@ -161,7 +161,7 @@ describe MetaInspector do
     it "should get correct absolute links for external pages" do
       @m.external_links.should == [ "mailto:pagerankalert@gmail.com",
-                           "http://pagerankalert.posterous.com",
+                           "http://pagerankalert.posterous.com/",
                            "http://twitter.com/pagerankalert",
                            "http://twitter.com/share" ]
     end
@@ -178,13 +178,14 @@ describe MetaInspector do
     end
     describe "links with international characters" do
-      it "should get correct absolute links, encoding the URLs as needed but respecting # and ?" do
+      it "should get correct absolute links, encoding the URLs as needed" do
         m = MetaInspector.new('http://international.com')
         m.links.should == [ "http://international.com/espa%C3%B1a.asp",
                             "http://international.com/roman%C3%A9e",
                             "http://international.com/faqs#cami%C3%B3n",
                             "http://international.com/search?q=cami%C3%B3n",
                             "http://international.com/search?q=espa%C3%B1a#top",
+                            "http://international.com/index.php?q=espa%C3%B1a&url=aHR0zZQ==&cntnt01pageid=21",
                             "http://example.com/espa%C3%B1a.asp",
                             "http://example.com/roman%C3%A9e",
                             "http://example.com/faqs#cami%C3%B3n",
@@ -199,7 +200,8 @@ describe MetaInspector do
                                        "http://international.com/roman%C3%A9e",
                                        "http://international.com/faqs#cami%C3%B3n",
                                        "http://international.com/search?q=cami%C3%B3n",
-                                       "http://international.com/search?q=espa%C3%B1a#top"]
+                                       "http://international.com/search?q=espa%C3%B1a#top",
+                                       "http://international.com/index.php?q=espa%C3%B1a&url=aHR0zZQ==&cntnt01pageid=21"]
         end
         it "should not crash when processing malformed hrefs" do
@@ -225,8 +227,7 @@ describe MetaInspector do
           m = MetaInspector.new('http://example.com/malformed_href')
           expect {
             m.external_links.should == ["skype:joeuser?call", "telnet://telnet.cdrom.com",
-                                        "javascript:alert('ok');", "javascript://",
-                                        "mailto:email(at)example.com"]
+                                        "javascript:alert('ok');", "mailto:email(at)example.com"]
             m.should_not be_ok
           }.to_not raise_error
         end
@@ -257,8 +258,7 @@ describe MetaInspector do
     it "should get the links" do
       @m.links.sort.should == [
-                                "FTP://FTP.CDROM.COM",
-                                "ftp://ftp.cdrom.com",
+                                "ftp://ftp.cdrom.com/",
                                 "javascript:alert('hey');",
                                 "mailto:user@example.com",
                                 "skype:joeuser?call",
@@ -275,12 +275,12 @@ describe MetaInspector do
     it "should convert protocol-relative links to http" do
       @m_http.links.should include('http://protocol-relative.com/contact')
-      @m_http.links.should include('http://yahoo.com')
+      @m_http.links.should include('http://yahoo.com/')
     end
     it "should convert protocol-relative links to https" do
       @m_https.links.should include('https://protocol-relative.com/contact')
-      @m_https.links.should include('https://yahoo.com')
+      @m_https.links.should include('https://yahoo.com/')
     end
   end
@@ -355,7 +355,40 @@ describe MetaInspector do
   describe 'to_hash' do
     it "should return a hash with all the values set" do
       @m = MetaInspector.new('http://pagerankalert.com')
-      @m.to_hash.should == {"title"=>"PageRankAlert.com :: Track your PageRank changes", "url"=>"http://pagerankalert.com", "meta"=>{"name"=>{"robots"=>"all,follow", "csrf_param"=>"authenticity_token", "description"=>"Track your PageRank(TM) changes and receive alerts by email", "keywords"=>"pagerank, seo, optimization, google", "csrf_token"=>"iW1/w+R8zrtDkhOlivkLZ793BN04Kr3X/pS+ixObHsE="}, "property"=>{}}, "images"=>["http://pagerankalert.com/images/pagerank_alert.png?1305794559"], "charset"=>"utf-8", "feed"=>"http://feeds.feedburner.com/PageRankAlert", "links"=>["http://pagerankalert.com/", "http://pagerankalert.com/es?language=es", "http://pagerankalert.com/users/sign_up", "http://pagerankalert.com/users/sign_in", "mailto:pagerankalert@gmail.com", "http://pagerankalert.posterous.com", "http://twitter.com/pagerankalert", "http://twitter.com/share"], "internal_links"=>["http://pagerankalert.com/", "http://pagerankalert.com/es?language=es", "http://pagerankalert.com/users/sign_up", "http://pagerankalert.com/users/sign_in"], "external_links" => ["mailto:pagerankalert@gmail.com", "http://pagerankalert.posterous.com", "http://twitter.com/pagerankalert", "http://twitter.com/share"], "content_type" => "text/html"}
+      @m.to_hash.should == {
+                              "url"             =>"http://pagerankalert.com/",
+                              "title"           =>"PageRankAlert.com :: Track your PageRank changes",
+                              "links"           => ["http://pagerankalert.com/",
+                                                    "http://pagerankalert.com/es?language=es",
+                                                    "http://pagerankalert.com/users/sign_up",
+                                                    "http://pagerankalert.com/users/sign_in",
+                                                    "mailto:pagerankalert@gmail.com",
+                                                    "http://pagerankalert.posterous.com/",
+                                                    "http://twitter.com/pagerankalert",
+                                                    "http://twitter.com/share"],
+                              "internal_links"  => ["http://pagerankalert.com/",
+                                                    "http://pagerankalert.com/es?language=es",
+                                                    "http://pagerankalert.com/users/sign_up",
+                                                    "http://pagerankalert.com/users/sign_in"],
+                              "external_links"  => ["mailto:pagerankalert@gmail.com",
+                                                    "http://pagerankalert.posterous.com/",
+                                                    "http://twitter.com/pagerankalert",
+                                                    "http://twitter.com/share"],
+                              "images"          => ["http://pagerankalert.com/images/pagerank_alert.png?1305794559"],
+                              "charset"         => "utf-8",
+                              "feed"            => "http://feeds.feedburner.com/PageRankAlert",
+                              "content_type"    =>"text/html",
+                              "meta"            => {
+                                                      "name" => {
+                                                                  "description"=> "Track your PageRank(TM) changes and receive alerts by email",
+                                                                  "keywords" => "pagerank, seo, optimization, google",
+                                                                  "robots" => "all,follow",
+                                                                  "csrf_param" => "authenticity_token",
+                                                                  "csrf_token" => "iW1/w+R8zrtDkhOlivkLZ793BN04Kr3X/pS+ixObHsE="
+                                                                },
+                                                      "property"=>{}
+                                                   }
+                           }
     end
   end

data/spec/redirections_spec.rb CHANGED Viewed

@@ -9,7 +9,7 @@ describe MetaInspector do
         m = MetaInspector.new("http://facebook.com")
         m.title.should be_nil
         m.should_not be_ok
-        m.errors.first.should == "Scraping exception: redirection forbidden: http://facebook.com -> https://www.facebook.com/"
+        m.errors.first.should == "Scraping exception: redirection forbidden: http://facebook.com/ -> https://www.facebook.com/"
       end
       it "allows safe redirections when :allow_redirections => :safe" do
@@ -30,14 +30,14 @@ describe MetaInspector do
         m = MetaInspector.new("https://unsafe-facebook.com")
         m.title.should be_nil
         m.should_not be_ok
-        m.errors.first.should == "Scraping exception: redirection forbidden: https://unsafe-facebook.com -> http://unsafe-facebook.com/"
+        m.errors.first.should == "Scraping exception: redirection forbidden: https://unsafe-facebook.com/ -> http://unsafe-facebook.com/"
       end
       it "disallows unsafe redirections when :allow_redirections => :safe" do
         m = MetaInspector.new("https://unsafe-facebook.com", :allow_redirections => :safe)
         m.title.should be_nil
         m.should_not be_ok
-        m.errors.first.should == "Scraping exception: redirection forbidden: https://unsafe-facebook.com -> http://unsafe-facebook.com/"
+        m.errors.first.should == "Scraping exception: redirection forbidden: https://unsafe-facebook.com/ -> http://unsafe-facebook.com/"
       end
       it "allows unsafe redirections when :allow_redirections => :all" do

metadata CHANGED Viewed

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification
 name: metainspector
 version: !ruby/object:Gem::Version
-  hash: 43
+  hash: 41
   prerelease:
   segments:
   - 1
   - 15
-  - 0
-  version: 1.15.0
+  - 1
+  version: 1.15.1
 platform: ruby
 authors:
 - Jaime Iniesta
@@ -15,7 +15,7 @@ autorequire:
 bindir: bin
 cert_chain: []
-date: 2013-01-19 00:00:00 Z
+date: 2013-02-06 00:00:00 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: nokogiri
@@ -65,9 +65,25 @@ dependencies:
   type: :runtime
   version_requirements: *id003
 - !ruby/object:Gem::Dependency
-  name: rspec
+  name: addressable
   prerelease: false
   requirement: &id004 !ruby/object:Gem::Requirement
+    none: false
+    requirements:
+    - - ~>
+      - !ruby/object:Gem::Version
+        hash: 7
+        segments:
+        - 2
+        - 3
+        - 2
+        version: 2.3.2
+  type: :runtime
+  version_requirements: *id004
+- !ruby/object:Gem::Dependency
+  name: rspec
+  prerelease: false
+  requirement: &id005 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - "="
@@ -79,11 +95,11 @@ dependencies:
         - 0
         version: 2.12.0
   type: :development
-  version_requirements: *id004
+  version_requirements: *id005
 - !ruby/object:Gem::Dependency
   name: fakeweb
   prerelease: false
-  requirement: &id005 !ruby/object:Gem::Requirement
+  requirement: &id006 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - "="
@@ -95,11 +111,11 @@ dependencies:
         - 0
         version: 1.3.0
   type: :development
-  version_requirements: *id005
+  version_requirements: *id006
 - !ruby/object:Gem::Dependency
   name: awesome_print
   prerelease: false
-  requirement: &id006 !ruby/object:Gem::Requirement
+  requirement: &id007 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - "="
@@ -111,11 +127,11 @@ dependencies:
         - 0
         version: 1.1.0
   type: :development
-  version_requirements: *id006
+  version_requirements: *id007
 - !ruby/object:Gem::Dependency
   name: rake
   prerelease: false
-  requirement: &id007 !ruby/object:Gem::Requirement
+  requirement: &id008 !ruby/object:Gem::Requirement
     none: false
     requirements:
     - - ~>
@@ -127,7 +143,7 @@ dependencies:
         - 3
         version: 10.0.3
   type: :development
-  version_requirements: *id007
+  version_requirements: *id008
 description: MetaInspector lets you scrape a web page and get its title, charset, link and meta tags
 email:
 - jaimeiniesta@gmail.com