metainspector 2.0.0 → 2.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: f4afd755a0fdc53abb2c3af992bea56021245ec2
-  data.tar.gz: fddcbbb1b151558bf245c9630585553149a93798
+  metadata.gz: fcb1f8adaeb835639f198ee9ede5f54da02211fd
+  data.tar.gz: 40bb733a1e3f2b48011c535951a2a2ac2bb097dc
 SHA512:
-  metadata.gz: e3c4cf8afc4de72cf0432cb2c051f3fc38049f1e1e648033c151644d5a3b38a211f829d88cff33f7813ff6d1d18fb945aa6016df9dea286a7b1d8eed37bbc623
-  data.tar.gz: d18e6647d1187ff115a734d5d9ebca9ab6f87eab4b12b91f0541df6db65cb2c2eb5e31b1be298c7540862fdfbdc3b84ca79eb5769401abfd0de2dac2a701d744
+  metadata.gz: 5f502ee4f24fb04b8d0afb8e369a5da2ac7c7c4a4b4c8a743cfbfd24f0f15125d139c5f35bd65c2fe9bce59d4015305c2fdf05f4fe6e696e691e90bb60e445b2
+  data.tar.gz: 2f9252d0283f815b15bc7e4c43b33e588975b63a935354e3e0c34639a0bad1c96d52b63308f9902120829b3747c796fc43f36d95d5ee78237c9dbc4fb64baa80

data/.travis.yml

@@ -2,3 +2,4 @@ rvm:
   - 1.9.2
   - 1.9.3
   - 2.0.0
+  - 2.1.0

data/README.md

@@ -18,7 +18,7 @@ If you're using it on a Rails application, just add it to your Gemfile and run `
 
     gem 'metainspector'
 
-This gem is tested on Ruby versions 1.9.2, 1.9.3 and 2.0.0.
+This gem is tested on Ruby versions 1.9.2, 1.9.3, 2.0.0 and 2.1.0.
 
 ## Usage
 
@@ -200,35 +200,22 @@ MetaInspector will try to parse all URLs by default. If you want to raise an exc
 This is useful when using MetaInspector on web spidering. Although on the initial URL you'll probably have an HTML URL, following links you may find yourself trying to parse non-html URLs.
 
     page = MetaInspector.new('http://example.com/image.png')
-    page.title         # returns ""
     page.content_type  # "image/png"
-    page.ok?           # true
+    page.description   # will returned a garbled string
 
     page = MetaInspector.new('http://example.com/image.png', :html_content_only => true)
-    page.title         # returns nil
     page.content_type  # "image/png"
-    page.ok?           # false
-    page.exceptions.first.message  # "The url provided contains image/png content instead of text/html content"
+    page.description   # raises an exception
 
 ## Exception handling
 
-You can check if the page has been succesfully parsed with:
+By default, MetaInspector will raise the exceptions found. We think that this is the safest default: in case the URL you're trying to scrape is unreachable, you should clearly be notified, and treat the exception as needed in your app.
 
-    page.ok?     # Will return true if everything looks OK
+However, if you prefer you can also set the `warn_level: :warn` option, so that exceptions found will just be warned on the standard output, instead of being raised.
 
-In case there have been any exceptions, you can check them with:
+You can also set the `warn_level: :store` option so that exceptions found will be silenced, and left for you to inspect on `page.exceptions`. You can also ask for `page.ok?`, wich will return `true` if no exceptions are stored.
 
-    page.exceptions  # Will return an array with the exceptions
-
-You can also specify what to do when encountering an exception. By default it
-will store it, but you can also tell MetaInspector to warn about it on the log
-console, or to raise the exceptions, like this:
-
-    # This will warn about the exception on console
-    page = MetaInspector.new('http://example.com', warn_level: :warn)
-
-    # This will raise the exception
-    page = MetaInspector.new('http://example.com', warn_level: :raise)
+You should avoid using the `:store` option, or use it wisely, as silencing errors can be problematic, it's always better to face the errors and treat them accordingly. 
 
 ## Examples
 

data/lib/meta_inspector/document.rb

@@ -26,13 +26,12 @@ module MetaInspector
         options[:warn_level] = :warn
       end
 
-      @warn_level         = options[:warn_level]
-
-      @exception_log  = MetaInspector::ExceptionLog.new(warn_level: warn_level)
+      @warn_level     = options[:warn_level]
+      @exception_log  = options[:exception_log] || MetaInspector::ExceptionLog.new(warn_level: warn_level)
       @url            = MetaInspector::URL.new(initial_url, exception_log: @exception_log)
       @request        = MetaInspector::Request.new(@url, allow_redirections: @allow_redirections,
                                                          timeout:            @timeout,
-                                                         exception_log:      @exception_log)
+                                                         exception_log:      @exception_log) unless @document
       @parser         = MetaInspector::Parser.new(self,  exception_log:      @exception_log)
     end
 
@@ -66,7 +65,7 @@ module MetaInspector
     private
 
     def defaults
-      { :timeout => 20, :html_content_only => false }
+      { :timeout => 20, :html_content_only => false, :warn_level => :raise }
     end
 
     def document

data/lib/meta_inspector/exception_log.rb

@@ -7,23 +7,34 @@ module MetaInspector
     attr_reader :exceptions, :warn_level
 
     def initialize(options = {})
-      @exceptions = []
+      options     = defaults.merge(options)
       @warn_level = options[:warn_level]
+      @exceptions = []
     end
 
     def <<(exception)
       case warn_level
-      when :warn
-        warn exception
       when :raise
         raise exception
+      when :warn
+        warn exception
+      when :store
+        @exceptions << exception
       end
-
-      @exceptions << exception
     end
 
     def ok?
-      exceptions.empty?
+      if warn_level == :store
+        exceptions.empty?
+      else
+        warn "ExceptionLog#ok? should only be used when warn_level is :store"
+      end
+    end
+
+    private
+
+    def defaults
+      { warn_level: :raise }
     end
   end
 end

data/lib/meta_inspector/request.rb

@@ -17,6 +17,8 @@ module MetaInspector
       @allow_redirections = options[:allow_redirections]
       @timeout            = options[:timeout]
       @exception_log      = options[:exception_log]
+
+      response            # as soon as it is set up, we make the request so we can fail early
     end
 
     extend Forwardable

data/lib/meta_inspector/url.rb

@@ -44,8 +44,7 @@ module MetaInspector
       else
         Addressable::URI.join(base_url, url).normalize.to_s
       end
-    rescue URI::InvalidURIError, Addressable::URI::InvalidURIError => e
-      @exception_log << e
+    rescue Addressable::URI::InvalidURIError => e
       nil
     end
 
@@ -66,9 +65,9 @@ module MetaInspector
     end
 
     def parsed(url)
-      URI.parse(url)
+      Addressable::URI.parse(url)
 
-      rescue URI::InvalidURIError, URI::InvalidComponentError => e
+      rescue Addressable::URI::InvalidURIError => e
         @exception_log << e
         nil
     end

data/lib/meta_inspector/version.rb

@@ -1,5 +1,5 @@
 # -*- encoding: utf-8 -*-
 
 module MetaInspector
-  VERSION = "2.0.0"
+  VERSION = "2.1.0"
 end

data/spec/document_spec.rb

@@ -57,38 +57,36 @@ describe MetaInspector::Document do
   end
 
   describe 'exception handling' do
+    let(:logger) { MetaInspector::ExceptionLog.new }
+
     it "should parse images when parse_html_content_type_only is not specified" do
-      image_url = MetaInspector::Document.new('http://pagerankalert.com/image.png')
-      desc = image_url.description
+      logger.should_not receive(:<<)
 
-      image_url.should be_ok
+      image_url = MetaInspector::Document.new('http://pagerankalert.com/image.png', exception_log: logger)
+      image_url.title
     end
 
     it "should parse images when parse_html_content_type_only is false" do
-      image_url = MetaInspector::Document.new('http://pagerankalert.com/image.png', :html_content_only => false)
-      desc = image_url.description
+      logger.should_not receive(:<<)
 
-      image_url.should be_ok
+      image_url = MetaInspector::Document.new('http://pagerankalert.com/image.png', html_content_only: false, exception_log: logger)
+      image_url.title
     end
 
     it "should handle errors when content is image/jpeg and html_content_type_only is true" do
-      image_url = MetaInspector::Document.new('http://pagerankalert.com/image.png', :html_content_only => true)
+      logger.should_receive(:<<).with(an_instance_of(RuntimeError))
 
-      expect {
-        title = image_url.title
-      }.to change { image_url.exceptions.size }
+      image_url = MetaInspector::Document.new('http://pagerankalert.com/image.png', html_content_only: true, exception_log: logger)
 
-      image_url.exceptions.first.message.should == "The url provided contains image/png content instead of text/html content"
+      image_url.title
     end
 
     it "should handle errors when content is not text/html and html_content_type_only is true" do
-      tar_url = MetaInspector::Document.new('http://pagerankalert.com/file.tar.gz', :html_content_only => true)
+      logger.should_receive(:<<).with(an_instance_of(RuntimeError))
 
-      expect {
-        title = tar_url.title
-      }.to change { tar_url.exceptions.size }
+      tar_url = MetaInspector::Document.new('http://pagerankalert.com/file.tar.gz', html_content_only: true, exception_log: logger)
 
-      tar_url.exceptions.first.message.should == "The url provided contains application/x-gzip content instead of text/html content"
+      tar_url.title
     end
   end
 end

data/spec/exception_log_spec.rb

@@ -3,9 +3,41 @@
 require File.join(File.dirname(__FILE__), "/spec_helper")
 
 describe MetaInspector::ExceptionLog do
-  let(:logger) { MetaInspector::ExceptionLog.new }
+
+  describe "warn_level" do
+    it "should be :raise by default" do
+      MetaInspector::ExceptionLog.new.warn_level.should == :raise
+    end
+
+    it "should raise exceptions when warn_level is :raise" do
+      logger    = MetaInspector::ExceptionLog.new(warn_level: :raise)
+      exception = StandardError.new("this should be raised")
+
+      expect {
+        logger << exception
+      }.to raise_exception(StandardError, "this should be raised")
+    end
+
+    it "should warn about the error if warn_level is :warn" do
+      logger    = MetaInspector::ExceptionLog.new(warn_level: :warn)
+      exception = StandardError.new("an error message")
+
+      logger.should_receive(:warn).with(exception)
+      logger << exception
+    end
+
+    it "should store the error if warn_level is :store" do
+      logger    = MetaInspector::ExceptionLog.new(warn_level: :store)
+      exception = StandardError.new("an error message")
+      expect {
+        logger << exception
+      }.to change { logger.exceptions.size }.by(1)
+    end
+  end
 
   describe "storing exceptions" do
+    let(:logger) { MetaInspector::ExceptionLog.new(warn_level: :store) }
+
     it "should store exceptions" do
       expect {
         logger << StandardError.new("an error message")
@@ -21,39 +53,29 @@ describe MetaInspector::ExceptionLog do
 
       logger.exceptions.should == [first, second]
     end
-  end
-
-  describe "ok?" do
-    it "should be true if no exceptions stored" do
-      logger.should be_ok
-    end
-
-    it "should be false if some exception stored" do
-      logger << StandardError.new("some message")
-      logger.should_not be_ok
-    end
-  end
-
-  describe "warn_level" do
-    it "should be quiet by default" do
-      MetaInspector::ExceptionLog.new.warn_level.should be_nil
-    end
 
-    it "should warn about the error if warn_level is :warn" do
-      verbose_logger = MetaInspector::ExceptionLog.new(warn_level: :warn)
-      exception      = StandardError.new("an error message")
+    describe "ok?" do
+      it "should be true if no exceptions stored" do
+        logger.should be_ok
+      end
 
-      verbose_logger.should_receive(:warn).with(exception)
-      verbose_logger << exception
-    end
+      it "should be false if some exception stored" do
+        logger << StandardError.new("some message")
+        logger.should_not be_ok
+      end
 
-    it "should raise exceptions when warn_level is :raise" do
-      raiser_logger = MetaInspector::ExceptionLog.new(warn_level: :raise)
-      exception     = StandardError.new("this should be raised")
+      it "should warn about misuse if warn_level is :raise" do
+        logger    = MetaInspector::ExceptionLog.new(warn_level: :raise)
+        logger.should_receive(:warn).with("ExceptionLog#ok? should only be used when warn_level is :store")
+        logger.ok?
+      end
 
-      expect {
-        raiser_logger << exception
-      }.to raise_exception(StandardError, "this should be raised")
+      it "should warn about misuse if warn_level is :warn" do
+        logger    = MetaInspector::ExceptionLog.new(warn_level: :warn)
+        logger.should_receive(:warn).with("ExceptionLog#ok? should only be used when warn_level is :store")
+        logger.ok?
+      end
     end
   end
+
 end

data/spec/parser_spec.rb

@@ -3,6 +3,8 @@
 require File.join(File.dirname(__FILE__), "/spec_helper")
 
 describe MetaInspector::Parser do
+  let(:logger) { MetaInspector::ExceptionLog.new }
+
   describe 'Doing a basic scrape' do
 
     before(:each) do
@@ -155,10 +157,7 @@ describe MetaInspector::Parser do
 
         it "should not crash when processing malformed hrefs" do
           m = MetaInspector::Parser.new(doc 'http://example.com/malformed_href')
-          expect {
-            m.internal_links.should == [ "http://example.com/faqs" ]
-            m.should be_ok
-          }.to_not raise_error
+          m.internal_links.should == [ "http://example.com/faqs" ]
         end
       end
 
@@ -174,11 +173,8 @@ describe MetaInspector::Parser do
 
         it "should not crash when processing malformed hrefs" do
           m = MetaInspector::Parser.new(doc 'http://example.com/malformed_href')
-          expect {
-            m.external_links.should == ["skype:joeuser?call", "telnet://telnet.cdrom.com",
-                                        "javascript:alert('ok');", "javascript://", "mailto:email(at)example.com"]
-            m.should be_ok
-          }.to_not raise_error
+          m.external_links.should == ["skype:joeuser?call", "telnet://telnet.cdrom.com", "javascript:alert('ok');",
+                                      "javascript://", "mailto:email(at)example.com"]
         end
       end
     end
@@ -369,7 +365,7 @@ describe MetaInspector::Parser do
 
   private
 
-  def doc(url, options = {})
+  def doc(url, options = { exception_log: logger })
     MetaInspector::Document.new(url, options)
   end
 end

data/spec/redirections_spec.rb

@@ -4,59 +4,63 @@ require File.join(File.dirname(__FILE__), "/spec_helper")
 
 describe MetaInspector do
   describe "redirections" do
+    let(:logger) { MetaInspector::ExceptionLog.new }
+
     describe "safe redirections (HTTP to HTTPS)" do
       it "disallows safe redirections by default" do
-        m = MetaInspector.new("http://facebook.com")
-        m.to_hash
-        m.should_not be_ok
-        m.exceptions.first.message.should == "redirection forbidden: http://facebook.com/ -> https://www.facebook.com/"
+        logger.should receive(:<<).with(an_instance_of(RuntimeError))
+
+        MetaInspector.new("http://facebook.com", exception_log: logger)
       end
 
       it "allows safe redirections when :allow_redirections => :safe" do
-        m = MetaInspector.new("http://facebook.com", :allow_redirections => :safe)
-        m.to_hash
-        m.should be_ok
+        logger.should_not receive(:<<)
+
+        m = MetaInspector.new("http://facebook.com", :allow_redirections => :safe, exception_log: logger)
+
+        m.url.should == "https://www.facebook.com/"
       end
 
       it "allows safe redirections when :allow_redirections => :all" do
-        m = MetaInspector.new("http://facebook.com", :allow_redirections => :all)
-        m.to_hash
-        m.should be_ok
+        logger.should_not receive(:<<)
+
+        m = MetaInspector.new("http://facebook.com", :allow_redirections => :all, exception_log: logger)
+
+        m.url.should == "https://www.facebook.com/"
       end
     end
 
     describe "unsafe redirections (HTTPS to HTTP)" do
       it "disallows unsafe redirections by default" do
-        m = MetaInspector.new("https://unsafe-facebook.com")
-        m.to_hash
-        m.should_not be_ok
-        m.exceptions.first.message.should == "redirection forbidden: https://unsafe-facebook.com/ -> http://unsafe-facebook.com/"
+        logger.should receive(:<<).with(an_instance_of(RuntimeError))
+
+        MetaInspector.new("https://unsafe-facebook.com", exception_log: logger)
       end
 
       it "disallows unsafe redirections when :allow_redirections => :safe" do
-        m = MetaInspector.new("https://unsafe-facebook.com", :allow_redirections => :safe)
-        m.to_hash
-        m.should_not be_ok
-        m.exceptions.first.message.should == "redirection forbidden: https://unsafe-facebook.com/ -> http://unsafe-facebook.com/"
+        logger.should receive(:<<).with(an_instance_of(RuntimeError))
+
+        MetaInspector.new("https://unsafe-facebook.com", :allow_redirections => :safe, exception_log: logger)
       end
 
       it "allows unsafe redirections when :allow_redirections => :all" do
-        m = MetaInspector.new("https://unsafe-facebook.com", :allow_redirections => :all)
-        m.to_hash
-        m.should be_ok
+        logger.should_not receive(:<<)
+
+        m = MetaInspector.new("https://unsafe-facebook.com", :allow_redirections => :all, exception_log: logger)
+
+        m.url.should == "http://unsafe-facebook.com/"
       end
     end
 
     describe "Redirections should update the base_uri" do
       it "updates the base_uri on safe redirections" do
-        m = MetaInspector.new("http://facebook.com", :allow_redirections => :safe) 
-        m.to_hash
+        m = MetaInspector.new("http://facebook.com", :allow_redirections => :safe)
+
         m.url.should == "https://www.facebook.com/"
       end
 
       it "updates the base_uri on all redirections" do
         m = MetaInspector.new("http://facebook.com", :allow_redirections => :all)
-        m.to_hash
 
         m.url.should == "https://www.facebook.com/"
       end

data/spec/request_spec.rb

@@ -27,6 +27,8 @@ describe MetaInspector::Request do
   end
 
   describe 'exception handling' do
+    let(:logger) { MetaInspector::ExceptionLog.new }
+
     before(:each) do
       FakeWeb.allow_net_connect = true
     end
@@ -36,23 +38,15 @@ describe MetaInspector::Request do
     end
 
     it "should handle timeouts" do
-      impatient = MetaInspector::Request.new(url('http://example.com'), timeout: 0.0000000000001)
-
-      expect {
-        impatient.read.should be_nil
-      }.to change { impatient.exceptions.size }
+      logger.should receive(:<<).with(an_instance_of(Timeout::Error))
 
-      impatient.exceptions.first.class.should == Timeout::Error
+      MetaInspector::Request.new(url('http://example.com/timeout'), timeout: 0.0000000000000000001, exception_log: logger)
     end
 
     it "should handle socket errors" do
-      nowhere = MetaInspector::Request.new(url('http://caca232dsdsaer3sdsd-asd343.org'))
-
-      expect {
-        nowhere.read.should be_nil
-      }.to change { nowhere.exceptions.size }
+      logger.should receive(:<<).with(an_instance_of(SocketError))
 
-      nowhere.exceptions.first.class.should == SocketError
+      MetaInspector::Request.new(url('http://caca232dsdsaer3sdsd-asd343.org'), exception_log: logger)
     end
   end
 

data/spec/spec_helper.rb

@@ -16,6 +16,7 @@ end
 # Faked web responses #
 #######################
 
+FakeWeb.register_uri(:get, "http://example.com/", :response => fixture_file("empty_page.response"))
 FakeWeb.register_uri(:get, "http://pagerankalert.com", :response => fixture_file("pagerankalert.com.response"))
 FakeWeb.register_uri(:get, "pagerankalert.com", :response => fixture_file("pagerankalert.com.response"))
 FakeWeb.register_uri(:get, "http://www.alazan.com", :response => fixture_file("alazan.com.response"))

data/spec/url_spec.rb

@@ -53,22 +53,4 @@ describe MetaInspector::URL do
       url.url.should == 'http://second.com/'
     end
   end
-
-  describe "exception handling" do
-    it "should handle URI::InvalidURIError" do
-      expect {
-        @malformed = MetaInspector::URL.new('javascript://')
-      }.to_not raise_error
-
-      @malformed.exceptions.first.class.should == URI::InvalidURIError
-    end
-
-    it "should handle URI::InvalidComponentError" do
-      expect {
-        @malformed = MetaInspector::URL.new('mailto:email(at)example.com')
-      }.to_not raise_error
-
-      @malformed.exceptions.first.class.should == URI::InvalidComponentError
-    end
-  end
 end

metadata

@@ -1,55 +1,55 @@
 --- !ruby/object:Gem::Specification
 name: metainspector
 version: !ruby/object:Gem::Version
-  version: 2.0.0
+  version: 2.1.0
 platform: ruby
 authors:
 - Jaime Iniesta
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-01-20 00:00:00.000000000 Z
+date: 2014-02-16 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: nokogiri
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '1.6'
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: '1.6'
 - !ruby/object:Gem::Dependency
   name: open_uri_redirections
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: 0.1.4
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: 0.1.4
 - !ruby/object:Gem::Dependency
   name: addressable
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: 2.3.5
   type: :runtime
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: 2.3.5
 - !ruby/object:Gem::Dependency
@@ -84,28 +84,28 @@ dependencies:
   name: awesome_print
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: 1.2.0
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: 1.2.0
 - !ruby/object:Gem::Dependency
   name: rake
   requirement: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: 10.1.0
   type: :development
   prerelease: false
   version_requirements: !ruby/object:Gem::Requirement
     requirements:
-    - - ~>
+    - - "~>"
       - !ruby/object:Gem::Version
         version: 10.1.0
 description: MetaInspector lets you scrape a web page and get its title, charset,
@@ -116,9 +116,9 @@ executables: []
 extensions: []
 extra_rdoc_files: []
 files:
-- .gitignore
-- .rspec.example
-- .travis.yml
+- ".gitignore"
+- ".rspec.example"
+- ".travis.yml"
 - Gemfile
 - MIT-LICENSE
 - README.md
@@ -182,17 +182,17 @@ require_paths:
 - lib
 required_ruby_version: !ruby/object:Gem::Requirement
   requirements:
-  - - '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 required_rubygems_version: !ruby/object:Gem::Requirement
   requirements:
-  - - '>='
+  - - ">="
     - !ruby/object:Gem::Version
       version: '0'
 requirements: []
 rubyforge_project: 
-rubygems_version: 2.0.6
+rubygems_version: 2.1.11
 signing_key: 
 specification_version: 4
 summary: MetaInspector is a ruby gem for web scraping purposes, that returns a hash