metainspector 3.0.0 → 3.1.0

checksums.yaml

@@ -1,7 +1,7 @@
 ---
 SHA1:
-  metadata.gz: 19cf1cb2c804ede2e36407d38a150840c5f19de6
-  data.tar.gz: 6e3cc4349b842ee52e5d4f9b993603662a573037
+  metadata.gz: 46f73f718a436065bc4353e279e997afebb0c43b
+  data.tar.gz: ed7aa01afb850aacca6ed942beb7ba9d5e596d3b
 SHA512:
-  metadata.gz: f77d946f82fd31d6683c512cef527370e99b7edef0a84c355d25e0ce2c73c2c12de642438c9f4c28a80b8e82b73d9974e2f52a8b0eab2e0cfc1ba900aa7148d2
-  data.tar.gz: cfa3f9e830fb56a8e3202c26827158d8504ca0ef0b1787953599b282b9d5005aa47275979e7d212a6f8dbed5a56aafd84ca0fd9025c700e819da8fc3bbbd183f
+  metadata.gz: 469d195a8b0f8fb5417bff510f3703b5cfde6ce5dba9866851e629f8c8d64dd73a680da7e3fb7614f556302ad1f12f7a59014df66b10ca1cf4f8cdead00ad67a
+  data.tar.gz: 0d7e3ae71b4beec707e293067c2f76db5698d399ea42071ef8845b551fedee2e7f400a62a066225572dc195ae21fc2504043419ef60b75ef0adf22ae6220034a

data/README.md

@@ -184,12 +184,30 @@ And the full scraped document is accessible from:
 
 ## Options
 
-### Timeout
-
-By default, MetaInspector times out after 20 seconds of waiting for a page to respond.
-You can set a different timeout with a second parameter, like this:
-
-    page = MetaInspector.new('sitevalidator.com', :timeout => 5) # 5 seconds timeout
+### Timeout & Retries
+
+By default, MetaInspector times out after 20 seconds of waiting for a page to respond,
+and it will retry fetching the page 3 times.
+You can specify different values for both of these, like this:
+
+    # timeout after 5 seconds, retry 4 times
+    page = MetaInspector.new('sitevalidator.com', :timeout => 5, :retries => 4)
+
+If MetaInspector fails to fetch the page after it has exhausted its retries,
+it will raise `MetaInspector::Request::TimeoutError`, which you can rescue in your
+application code.
+
+    begin
+      data = MetaInspector.new(url)
+    rescue MetaInspector::Request::TimeoutError
+      enqueue_for_future_fetch_attempt(url)
+      render_simple(url)
+    rescue
+      log_fetch_error($!)
+      render_simple(url)
+    else
+      render_rich(data)
+    end
 
 ### Redirections
 

data/lib/meta_inspector/document.rb

@@ -64,10 +64,12 @@ module MetaInspector
 
     def defaults
       { :timeout => 20,
+        :retries => 3,
         :html_content_only => false,
         :warn_level => :raise,
         :headers => {'User-Agent' => "MetaInspector/#{MetaInspector::VERSION} (+https://github.com/jaimeiniesta/metainspector)"},
-        :allow_redirections => true
+        :allow_redirections => true,
+        :exception_log => MetaInspector::ExceptionLog.new
       }
     end
 

data/lib/meta_inspector/request.rb

@@ -12,12 +12,11 @@ module MetaInspector
     include MetaInspector::Exceptionable
 
     def initialize(initial_url, options = {})
-      options = defaults.merge(options)
-
       @url                = initial_url
       
       @allow_redirections = options[:allow_redirections]
       @timeout            = options[:timeout]
+      @retries            = options[:retries]
       @exception_log      = options[:exception_log]
       @headers            = options[:headers]
 
@@ -38,8 +37,13 @@ module MetaInspector
     private
 
     def response
+      request_count ||= 0
+      request_count += 1
       Timeout::timeout(@timeout) { @response ||= fetch }
-    rescue TimeoutError, Faraday::ConnectionFailed, RuntimeError => e
+    rescue Timeout::Error
+      retry unless @retries == request_count
+      @exception_log << TimeoutError.new("Attempt to fetch #{url} timed out 3 times.")
+    rescue Faraday::ConnectionFailed, RuntimeError => e
       @exception_log << e
       nil
     end
@@ -60,8 +64,7 @@ module MetaInspector
       response
     end
 
-    def defaults
-      { timeout: 20, exception_log: MetaInspector::ExceptionLog.new, allow_redirections: true }
+    class TimeoutError < StandardError
     end
   end
 end

data/lib/meta_inspector/version.rb

@@ -1,5 +1,5 @@
 # -*- encoding: utf-8 -*-
 
 module MetaInspector
-  VERSION = "3.0.0"
+  VERSION = "3.1.0"
 end

data/spec/request_spec.rb

@@ -37,12 +37,6 @@ describe MetaInspector::Request do
       FakeWeb.allow_net_connect = false
     end
 
-    it "should handle timeouts" do
-      logger.should receive(:<<).with(an_instance_of(Timeout::Error))
-
-      MetaInspector::Request.new(url('http://example.com/timeout'), timeout: 0.0000000000000000001, exception_log: logger)
-    end
-
     it "should handle socket errors" do
       TCPSocket.stub(:open).and_raise(SocketError)
       logger.should receive(:<<).with(an_instance_of(Faraday::ConnectionFailed))
@@ -51,6 +45,50 @@ describe MetaInspector::Request do
     end
   end
 
+  describe "retrying on timeouts" do
+    let(:logger) { MetaInspector::ExceptionLog.new }
+    subject do
+      MetaInspector::Request.new(url('http://pagerankalert.com'),
+                                 exception_log: logger, retries: 3)
+     end
+
+    context "when request never succeeds" do
+      before{ Timeout.stub(:timeout).and_raise(Timeout::Error) }
+      it "swallows all the timeout errors and raises MetaInspector::Request::TimeoutError" do
+        logger.should receive(:<<).with(an_instance_of(MetaInspector::Request::TimeoutError))
+        subject
+      end
+    end
+
+    context "when request succeeds on third try" do
+      before do
+        Timeout.stub(:timeout).and_raise(Timeout::Error)
+        Timeout.stub(:timeout).and_raise(Timeout::Error)
+        Timeout.stub(:timeout).and_call_original
+      end
+      it "doesn't raise an exception" do
+        logger.should_not receive(:<<)
+        subject
+      end
+      it "succeeds as normal" do
+        subject.content_type.should == "text/html"
+      end
+    end
+
+    context "when request succeeds on fourth try" do
+      before do
+        Timeout.stub(:timeout).exactly(3).times.and_raise(Timeout::Error)
+        # if it were called a fourth time, rspec would raise an error
+        # so this implicitely tests the correct behavior
+      end
+      it "swallows all the timeout errors and raises MetaInspector::Request::TimeoutError" do
+        logger.should receive(:<<).with(an_instance_of(MetaInspector::Request::TimeoutError))
+        subject
+      end
+    end
+
+  end
+
   private
 
   def url(initial_url)

metadata

@@ -1,14 +1,14 @@
 --- !ruby/object:Gem::Specification
 name: metainspector
 version: !ruby/object:Gem::Version
-  version: 3.0.0
+  version: 3.1.0
 platform: ruby
 authors:
 - Jaime Iniesta
 autorequire: 
 bindir: bin
 cert_chain: []
-date: 2014-10-07 00:00:00.000000000 Z
+date: 2014-10-09 00:00:00.000000000 Z
 dependencies:
 - !ruby/object:Gem::Dependency
   name: nokogiri