metainspector 1.11.0 → 1.12.0

data/README.rdoc

@@ -38,6 +38,14 @@ MetaInspector will try to parse all URLs by default. If you want to parse only t
 
   page = MetaInspector.new('markupvalidator.com', :html_content_only => true)
 
+MetaInspector allows safe redirects from http to https sites by default. Passing allow_safe_redirections as false will throw exceptions on such redirects.
+
+  page = MetaInspector.new('facebook.com', :allow_safe_redirections => false)
+
+To enable unsafe redirects from https to http sites you can pass allow_unsafe_redirections as true. If this option is not specified or is false an exception is thrown on such redirects.
+
+  page = MetaInspector.new('facebook.com', :allow_unsafe_redirections => true)
+
 Then you can see the scraped data like this:
 
   page.url                # URL of the page
@@ -91,6 +99,10 @@ In case there have been any errors, you can check them with:
 
   page.errors                 # Will return an array with the error messages
 
+If you also want to see the errors on console, you can initialize MetaInspector with the verbose option like that:
+
+  page = MetaInspector.new('http://example.com', :verbose => true)
+
 = Examples
 
 You can find some sample scripts on the samples folder, including a basic scraping and a spider that will follow external links using a queue. What follows is an example of use from irb:

data/lib/meta_inspector.rb

@@ -1,6 +1,7 @@
 # -*- encoding: utf-8 -*-
 
 require File.expand_path(File.join(File.dirname(__FILE__), 'meta_inspector/scraper'))
+require File.expand_path(File.join(File.dirname(__FILE__), 'meta_inspector/open_uri'))
 
 module MetaInspector
   extend self

data/lib/meta_inspector/open_uri.rb

@@ -0,0 +1,50 @@
+# Patch to allow open-uri to follow safe (http to https) and unsafe redirects (https to http).
+# Original gist URL:
+# https://gist.github.com/1271420
+#
+# Relevant issue:
+# http://redmine.ruby-lang.org/issues/3719
+#
+# Source here:
+# https://github.com/ruby/ruby/blob/trunk/lib/open-uri.rb
+
+module OpenURI
+  class <<self
+    alias_method :open_uri_original, :open_uri
+    alias_method :redirectable_cautious?, :redirectable?
+
+    def redirectable_safe?(uri1, uri2)
+      uri1.scheme.downcase == uri2.scheme.downcase || (uri1.scheme.downcase == "http" && uri2.scheme.downcase == "https")
+    end
+
+    def redirectable_unsafe?(uri1, uri2)
+      !redirectable_safe?(uri1, uri2)
+    end
+  end
+
+  # The original open_uri takes *args but then doesn't do anything with them.
+  # Assume we can only handle a hash.
+  def self.open_uri(name, options = {})
+    redirectable_unsafe = options.delete :allow_unsafe_redirections
+    redirectable_safe   = options.delete :allow_safe_redirections
+
+    if redirectable_unsafe
+      class <<self
+        remove_method :redirectable?
+        alias_method :redirectable?, :redirectable_unsafe?
+      end
+    elsif redirectable_safe
+      class <<self
+        remove_method :redirectable?
+        alias_method :redirectable?, :redirectable_safe?
+      end
+    else
+      class <<self
+        remove_method :redirectable?
+        alias_method :redirectable?, :redirectable_cautious?
+      end
+    end
+
+    self.open_uri_original name, options
+  end
+end

data/lib/meta_inspector/scraper.rb

@@ -9,20 +9,28 @@ require 'timeout'
 module MetaInspector
   class Scraper
     attr_reader :url, :scheme, :host, :root_url, :errors, :content_type, :timeout, :html_content_only
+    attr_reader :allow_safe_redirections, :allow_unsafe_redirections, :verbose
 
     # Initializes a new instance of MetaInspector, setting the URL to the one given
     # Options:
     # => timeout: defaults to 20 seconds
     # => html_content_type_only: if an exception should be raised if request content-type is not text/html. Defaults to false
+    # => allow_safe_redirections:   if redirects from http to https sites on the same domain should be allowed or not
+    # => allow_unsafe_redirections: if redirects from https to http sites on the same domain should be allowed or not
     def initialize(url, options = {})
+      options   = defaults.merge(options)
+
       @url      = with_default_scheme(encode_url(url))
       @scheme   = URI.parse(@url).scheme
       @host     = URI.parse(@url).host
       @root_url = "#{@scheme}://#{@host}/"
-      @timeout  = options[:timeout] || 20
+      @timeout  = options[:timeout]
       @data     = Hashie::Rash.new
       @errors   = []
-      @html_content_only = options[:html_content_only] || false
+      @html_content_only          = options[:html_content_only]
+      @allow_safe_redirections    = options[:allow_safe_redirections]
+      @allow_unsafe_redirections  = options[:allow_unsafe_redirections]
+      @verbose                    = options[:verbose]
     end
 
     # Returns the parsed document title, from the content of the <title> tag.
@@ -134,6 +142,16 @@ module MetaInspector
 
     private
 
+    def defaults
+      {
+        :timeout                    => 20,
+        :html_content_only          => false,
+        :allow_safe_redirections    => true,
+        :allow_unsafe_redirections  => false,
+        :verbose                    => false
+      }
+    end
+
     # Scrapers for all meta_tags in the form of "meta_name" are automatically defined. This has been tested for
     # meta name: keywords, description, robots, generator
     # meta http-equiv: content-language, Content-Type
@@ -156,7 +174,7 @@ module MetaInspector
 
     # Makes the request to the server
     def request
-      Timeout::timeout(timeout) { @request ||= open(url) }
+      Timeout::timeout(timeout) { @request ||= open(url, {:allow_safe_redirections => allow_safe_redirections, :allow_unsafe_redirections => allow_unsafe_redirections}) }
 
       rescue TimeoutError
         add_fatal_error 'Timeout!!!'
@@ -199,7 +217,7 @@ module MetaInspector
 
     # Stores the error for later inspection
     def add_fatal_error(error)
-      warn error
+      warn error if verbose
       @errors << error
     end
 

data/lib/meta_inspector/version.rb

@@ -1,5 +1,5 @@
 # -*- encoding: utf-8 -*-
 
 module MetaInspector
-  VERSION = "1.11.0"
+  VERSION = "1.12.0"
 end

data/spec/fixtures/facebook.com.response

@@ -0,0 +1,6 @@
+HTTP/1.1 302 Found
+Location: https://www.facebook.com/
+Connection:keep-alive
+Content-Length:0
+Content-Type:text/html; charset=utf-8
+Date:Mon, 26 Nov 2012 23:02:17 GMT

data/spec/fixtures/https.facebook.com.response

@@ -0,0 +1,18 @@
+HTTP/1.1 200 OK
+Server: nginx/0.7.67
+Date: Fri, 18 Nov 2011 21:46:46 GMT
+Content-Type: text/html
+Connection: keep-alive
+Last-Modified: Mon, 14 Nov 2011 16:53:18 GMT
+Content-Length: 4987
+X-Varnish: 2000423390
+Age: 0
+Via: 1.1 varnish
+
+<html>
+  <head>
+    <title>Hello From Facebook</title>
+  </head>
+  <body>
+  </body>
+</html>

data/spec/fixtures/unsafe_facebook.com.response

@@ -0,0 +1,18 @@
+HTTP/1.1 200 OK
+Server: nginx/0.7.67
+Date: Fri, 18 Nov 2011 21:46:46 GMT
+Content-Type: text/html
+Connection: keep-alive
+Last-Modified: Mon, 14 Nov 2011 16:53:18 GMT
+Content-Length: 4987
+X-Varnish: 2000423390
+Age: 0
+Via: 1.1 varnish
+
+<html>
+  <head>
+    <title>Hello From Unsafe Facebook</title>
+  </head>
+  <body>
+  </body>
+</html>

data/spec/fixtures/unsafe_https.facebook.com.response

@@ -0,0 +1,6 @@
+HTTP/1.1 302 Found
+Location: http://unsafe-facebook.com/
+Connection:keep-alive
+Content-Length:0
+Content-Type:text/html; charset=utf-8
+Date:Mon, 26 Nov 2012 23:02:17 GMT

data/spec/metainspector_spec.rb

@@ -3,31 +3,6 @@
 require File.join(File.dirname(__FILE__), "/spec_helper")
 
 describe MetaInspector do
-  FakeWeb.register_uri(:get, "http://pagerankalert.com", :response => fixture_file("pagerankalert.com.response"))
-  FakeWeb.register_uri(:get, "pagerankalert.com", :response => fixture_file("pagerankalert.com.response"))
-  FakeWeb.register_uri(:get, "http://www.alazan.com", :response => fixture_file("alazan.com.response"))
-  FakeWeb.register_uri(:get, "http://alazan.com/websolution.asp", :response => fixture_file("alazan_websolution.response"))
-  FakeWeb.register_uri(:get, "http://www.theonion.com/articles/apple-claims-new-iphone-only-visible-to-most-loyal,2772/", :response => fixture_file("theonion.com.response"))
-  FakeWeb.register_uri(:get, "http://theonion-no-description.com", :response => fixture_file("theonion-no-description.com.response"))
-  FakeWeb.register_uri(:get, "http://www.iteh.at", :response => fixture_file("iteh.at.response"))
-  FakeWeb.register_uri(:get, "http://www.tea-tron.com/jbravo/blog/", :response => fixture_file("tea-tron.com.response"))
-  FakeWeb.register_uri(:get, "http://www.guardian.co.uk/media/pda/2011/sep/15/techcrunch-arrington-startups", :response => fixture_file("guardian.co.uk.response"))
-  FakeWeb.register_uri(:get, "http://protocol-relative.com", :response => fixture_file("protocol_relative.response"))
-  FakeWeb.register_uri(:get, "https://protocol-relative.com", :response => fixture_file("protocol_relative.response"))
-  FakeWeb.register_uri(:get, "http://example.com/nonhttp", :response => fixture_file("nonhttp.response"))
-  FakeWeb.register_uri(:get, "http://example.com/invalid_href", :response => fixture_file("invalid_href.response"))
-  FakeWeb.register_uri(:get, "http://www.youtube.com/watch?v=iaGSSrp49uc", :response => fixture_file("youtube.response"))
-  FakeWeb.register_uri(:get, "http://markupvalidator.com/faqs", :response => fixture_file("markupvalidator_faqs.response"))
-  FakeWeb.register_uri(:get, "https://twitter.com/markupvalidator", :response => fixture_file("twitter_markupvalidator.response"))
-  FakeWeb.register_uri(:get, "https://example.com/empty", :response => fixture_file("empty_page.response"))
-  FakeWeb.register_uri(:get, "http://international.com", :response => fixture_file("international.response"))
-  FakeWeb.register_uri(:get, "http://charset000.com", :response => fixture_file("charset_000.response"))
-  FakeWeb.register_uri(:get, "http://charset001.com", :response => fixture_file("charset_001.response"))
-  FakeWeb.register_uri(:get, "http://charset002.com", :response => fixture_file("charset_002.response"))
-  FakeWeb.register_uri(:get, "http://www.inkthemes.com/", :response => fixture_file("wordpress_site.response"))
-  FakeWeb.register_uri(:get, "http://pagerankalert.com/image.png", :body => "Image", :content_type => "image/png")
-  FakeWeb.register_uri(:get, "http://pagerankalert.com/file.tar.gz", :body => "Image", :content_type => "application/x-gzip")
-
   describe 'Initialization' do
     it 'should accept an URL with a scheme' do
       MetaInspector.new('http://pagerankalert.com').url.should == 'http://pagerankalert.com'

data/spec/redirections_spec.rb

@@ -0,0 +1,50 @@
+# -*- encoding: utf-8 -*-
+
+require File.join(File.dirname(__FILE__), "/spec_helper")
+
+describe MetaInspector do
+  describe "redirections" do
+    describe "safe redirections (HTTP to HTTPS)" do
+      it "allows safe redirections by default" do
+        m = MetaInspector.new("http://facebook.com")
+        m.title.should == "Hello From Facebook"
+        m.should be_ok
+      end
+
+      it "allows safe redirections when specifically set to true" do
+        m = MetaInspector.new("http://facebook.com", :allow_safe_redirections => true)
+        m.title.should == "Hello From Facebook"
+        m.should be_ok
+      end
+
+      it "disallows safe redirections if set to false" do
+        m = MetaInspector.new("http://facebook.com", :allow_safe_redirections => false)
+        m.title.should be_nil
+        m.should_not be_ok
+        m.errors.first.should == "Scraping exception: redirection forbidden: http://facebook.com -> https://www.facebook.com/"
+      end
+    end
+
+    describe "unsafe redirections (HTTPS to HTTP)" do
+      it "disallows unsafe redirections by default" do
+        m = MetaInspector.new("https://unsafe-facebook.com")
+        m.title.should be_nil
+        m.should_not be_ok
+        m.errors.first.should == "Scraping exception: redirection forbidden: https://unsafe-facebook.com -> http://unsafe-facebook.com/"
+      end
+
+      it "disallows unsafe redirections when specifically set to false" do
+        m = MetaInspector.new("https://unsafe-facebook.com", :allow_unsafe_redirections => false)
+        m.title.should be_nil
+        m.should_not be_ok
+        m.errors.first.should == "Scraping exception: redirection forbidden: https://unsafe-facebook.com -> http://unsafe-facebook.com/"
+      end
+
+      it "allows unsafe redirections if set to true" do
+        m = MetaInspector.new("https://unsafe-facebook.com", :allow_unsafe_redirections => true)
+        m.title.should == "Hello From Unsafe Facebook"
+        m.should be_ok
+      end
+    end
+  end
+end

data/spec/spec_helper.rb

@@ -10,4 +10,42 @@ def fixture_file(filename)
   return '' if filename == ''
   file_path = File.expand_path(File.dirname(__FILE__) + '/fixtures/' + filename)
   File.read(file_path)
-end
+end
+
+#######################
+# Faked web responses #
+#######################
+
+FakeWeb.register_uri(:get, "http://pagerankalert.com", :response => fixture_file("pagerankalert.com.response"))
+FakeWeb.register_uri(:get, "pagerankalert.com", :response => fixture_file("pagerankalert.com.response"))
+FakeWeb.register_uri(:get, "http://www.alazan.com", :response => fixture_file("alazan.com.response"))
+FakeWeb.register_uri(:get, "http://alazan.com/websolution.asp", :response => fixture_file("alazan_websolution.response"))
+FakeWeb.register_uri(:get, "http://www.theonion.com/articles/apple-claims-new-iphone-only-visible-to-most-loyal,2772/", :response => fixture_file("theonion.com.response"))
+FakeWeb.register_uri(:get, "http://theonion-no-description.com", :response => fixture_file("theonion-no-description.com.response"))
+FakeWeb.register_uri(:get, "http://www.iteh.at", :response => fixture_file("iteh.at.response"))
+FakeWeb.register_uri(:get, "http://www.tea-tron.com/jbravo/blog/", :response => fixture_file("tea-tron.com.response"))
+FakeWeb.register_uri(:get, "http://www.guardian.co.uk/media/pda/2011/sep/15/techcrunch-arrington-startups", :response => fixture_file("guardian.co.uk.response"))
+FakeWeb.register_uri(:get, "http://protocol-relative.com", :response => fixture_file("protocol_relative.response"))
+FakeWeb.register_uri(:get, "https://protocol-relative.com", :response => fixture_file("protocol_relative.response"))
+FakeWeb.register_uri(:get, "http://example.com/nonhttp", :response => fixture_file("nonhttp.response"))
+FakeWeb.register_uri(:get, "http://example.com/invalid_href", :response => fixture_file("invalid_href.response"))
+FakeWeb.register_uri(:get, "http://www.youtube.com/watch?v=iaGSSrp49uc", :response => fixture_file("youtube.response"))
+FakeWeb.register_uri(:get, "http://markupvalidator.com/faqs", :response => fixture_file("markupvalidator_faqs.response"))
+FakeWeb.register_uri(:get, "https://twitter.com/markupvalidator", :response => fixture_file("twitter_markupvalidator.response"))
+FakeWeb.register_uri(:get, "https://example.com/empty", :response => fixture_file("empty_page.response"))
+FakeWeb.register_uri(:get, "http://international.com", :response => fixture_file("international.response"))
+FakeWeb.register_uri(:get, "http://charset000.com", :response => fixture_file("charset_000.response"))
+FakeWeb.register_uri(:get, "http://charset001.com", :response => fixture_file("charset_001.response"))
+FakeWeb.register_uri(:get, "http://charset002.com", :response => fixture_file("charset_002.response"))
+FakeWeb.register_uri(:get, "http://www.inkthemes.com/", :response => fixture_file("wordpress_site.response"))
+FakeWeb.register_uri(:get, "http://pagerankalert.com/image.png", :body => "Image", :content_type => "image/png")
+FakeWeb.register_uri(:get, "http://pagerankalert.com/file.tar.gz", :body => "Image", :content_type => "application/x-gzip")
+
+# These examples are used to test the redirections from HTTP to HTTPS and vice versa
+# http://facebook.com => https://facebook.com
+FakeWeb.register_uri(:get, "http://facebook.com/",          :response => fixture_file("facebook.com.response"))
+FakeWeb.register_uri(:get, "https://www.facebook.com/",     :response => fixture_file("https.facebook.com.response"))
+
+# https://unsafe-facebook.com => http://unsafe-facebook.com
+FakeWeb.register_uri(:get, "https://unsafe-facebook.com/",  :response => fixture_file("unsafe_https.facebook.com.response"))
+FakeWeb.register_uri(:get, "http://unsafe-facebook.com/",   :response => fixture_file("unsafe_facebook.com.response"))

metadata

@@ -1,13 +1,13 @@
 --- !ruby/object:Gem::Specification 
 name: metainspector
 version: !ruby/object:Gem::Version 
-  hash: 59
+  hash: 39
   prerelease: 
   segments: 
   - 1
-  - 11
+  - 12
   - 0
-  version: 1.11.0
+  version: 1.12.0
 platform: ruby
 authors: 
 - Jaime Iniesta
@@ -15,7 +15,7 @@ autorequire:
 bindir: bin
 cert_chain: []
 
-date: 2012-11-26 00:00:00 Z
+date: 2012-12-01 00:00:00 Z
 dependencies: 
 - !ruby/object:Gem::Dependency 
   version_requirements: &id001 !ruby/object:Gem::Requirement 
@@ -131,6 +131,7 @@ files:
 - README.rdoc
 - Rakefile
 - lib/meta_inspector.rb
+- lib/meta_inspector/open_uri.rb
 - lib/meta_inspector/scraper.rb
 - lib/meta_inspector/version.rb
 - lib/metainspector.rb
@@ -143,7 +144,9 @@ files:
 - spec/fixtures/charset_001.response
 - spec/fixtures/charset_002.response
 - spec/fixtures/empty_page.response
+- spec/fixtures/facebook.com.response
 - spec/fixtures/guardian.co.uk.response
+- spec/fixtures/https.facebook.com.response
 - spec/fixtures/international.response
 - spec/fixtures/invalid_href.response
 - spec/fixtures/iteh.at.response
@@ -155,9 +158,12 @@ files:
 - spec/fixtures/theonion-no-description.com.response
 - spec/fixtures/theonion.com.response
 - spec/fixtures/twitter_markupvalidator.response
+- spec/fixtures/unsafe_facebook.com.response
+- spec/fixtures/unsafe_https.facebook.com.response
 - spec/fixtures/wordpress_site.response
 - spec/fixtures/youtube.response
 - spec/metainspector_spec.rb
+- spec/redirections_spec.rb
 - spec/spec_helper.rb
 homepage: https://github.com/jaimeiniesta/metainspector
 licenses: []